From dce29ad3c7db6d6b2009b1527580df3ff274cc5f Mon Sep 17 00:00:00 2001 From: catsout Date: Sat, 13 Aug 2022 07:14:18 +0800 Subject: [PATCH 1/6] node: Support yarn berry --- node/flatpak_node_generator/flatpak-yarn.js | 262 ++++++++++++++++++ node/flatpak_node_generator/manifest.py | 13 +- .../providers/special.py | 1 - node/flatpak_node_generator/providers/yarn.py | 261 +++++++++++++++-- 4 files changed, 509 insertions(+), 28 deletions(-) create mode 100644 node/flatpak_node_generator/flatpak-yarn.js diff --git a/node/flatpak_node_generator/flatpak-yarn.js b/node/flatpak_node_generator/flatpak-yarn.js new file mode 100644 index 00000000..03890e7c --- /dev/null +++ b/node/flatpak_node_generator/flatpak-yarn.js @@ -0,0 +1,262 @@ +const PackageManager = { + Yarn1: `Yarn Classic`, + Yarn2: `Yarn`, + Npm: `npm`, + Pnpm: `pnpm`, +} + +module.exports = { + name: `flatpak-builder`, + factory: require => { + const { BaseCommand } = require(`@yarnpkg/cli`); + const { parseSyml } = require('@yarnpkg/parsers'); + const { Configuration, Manifest, scriptUtils, structUtils, tgzUtils, execUtils, miscUtils, hashUtils } = require('@yarnpkg/core') + const { Filename, ZipFS, npath, ppath, PortablePath, xfs } = require('@yarnpkg/fslib'); + const { getLibzipPromise } = require('@yarnpkg/libzip'); + const { gitUtils } = require('@yarnpkg/plugin-git'); + const { PassThrough, Readable, Writable } = require('stream'); + const { Command, Option } = require(`clipanion`); + const { YarnVersion } = require('@yarnpkg/core'); + const fs = require('fs'); + + // from https://github.com/yarnpkg/berry/blob/%40yarnpkg/shell/3.2.3/packages/plugin-essentials/sources/commands/set/version.ts#L194 + async function setPackageManager(projectCwd) { + const bundleVersion = YarnVersion; + + const manifest = (await Manifest.tryFind(projectCwd)) || new Manifest(); + + if (bundleVersion && miscUtils.isTaggedYarnVersion(bundleVersion)) { + manifest.packageManager = `yarn@${bundleVersion}`; + const data = {}; + manifest.exportTo(data); + + const path = ppath.join(projectCwd, Manifest.fileName); + const content = `${JSON.stringify(data, null, manifest.indent)}\n`; + + await xfs.changeFilePromise(path, content, { + automaticNewlines: true, + }); + } + } + + // func from https://github.com/yarnpkg/berry/blob/%40yarnpkg/shell/3.2.3/packages/yarnpkg-core/sources/scriptUtils.ts#L215 + async function prepareExternalProject(cwd, outputPath, { configuration, locator, stdout, yarn_v1, workspace = null }) { + const devirtualizedLocator = locator && structUtils.isVirtualLocator(locator) + ? structUtils.devirtualizeLocator(locator) + : locator; + + const name = devirtualizedLocator + ? structUtils.stringifyLocator(devirtualizedLocator) + : `an external project`; + + const stderr = stdout; + + stdout.write(`Packing ${name} from sources\n`); + + const packageManagerSelection = await scriptUtils.detectPackageManager(cwd); + let effectivePackageManager; + if (packageManagerSelection !== null) { + stdout.write(`Using ${packageManagerSelection.packageManager} for bootstrap. Reason: ${packageManagerSelection.reason}\n\n`); + effectivePackageManager = packageManagerSelection.packageManager; + } else { + stdout.write(`No package manager configuration detected; defaulting to Yarn\n\n`); + effectivePackageManager = PackageManager.Yarn2; + } + if (effectivePackageManager === PackageManager.Pnpm) { + effectivePackageManager = PackageManager.Npm; + } + + const workflows = new Map([ + [PackageManager.Yarn1, async () => { + const workspaceCli = workspace !== null + ? [`workspace`, workspace] + : []; + + await setPackageManager(cwd); + + await Configuration.updateConfiguration(cwd, { + yarnPath: yarn_v1, + }); + + await xfs.appendFilePromise(ppath.join(cwd, `.npmignore`), `/.yarn\n`); + + const pack = await execUtils.pipevp(`yarn`, [...workspaceCli, `pack`, `--filename`, npath.fromPortablePath(outputPath)], { cwd, stdout, stderr }); + if (pack.code !== 0) + return pack.code; + + return 0; + }], + [PackageManager.Yarn2, async () => { + const workspaceCli = workspace !== null + ? [`workspace`, workspace] + : []; + const lockfilePath = ppath.join(cwd, Filename.lockfile); + if (!(await xfs.existsPromise(lockfilePath))) + await xfs.writeFilePromise(lockfilePath, ``); + + const pack = await execUtils.pipevp(`yarn`, [...workspaceCli, `pack`, `--filename`, npath.fromPortablePath(outputPath)], { cwd, stdout, stderr }); + if (pack.code !== 0) + return pack.code; + return 0; + }], + [PackageManager.Npm, async () => { + const workspaceCli = workspace !== null + ? [`--workspace`, workspace] + : []; + const packStream = new PassThrough(); + const packPromise = miscUtils.bufferStream(packStream); + const pack = await execUtils.pipevp(`npm`, [`pack`, `--silent`, ...workspaceCli], { cwd, stdout: packStream, stderr }); + if (pack.code !== 0) + return pack.code; + + const packOutput = (await packPromise).toString().trim().replace(/^.*\n/s, ``); + const packTarget = ppath.resolve(cwd, npath.toPortablePath(packOutput)); + await xfs.renamePromise(packTarget, outputPath); + return 0; + }], + ]); + const workflow = workflows.get(effectivePackageManager); + const code = await workflow(); + if (code === 0 || typeof code === `undefined`) + return; + else + throw `Packing the package failed (exit code ${code})`; + } + + class convertToZipCommand extends BaseCommand { + static paths = [[`convertToZip`]]; + yarn_v1 = Option.String({ required: true }); + + async execute() { + const configuration = await Configuration.find(this.context.cwd, + this.context.plugins); + const lockfilePath = ppath.join(this.context.cwd, 'yarn.lock'); + const cacheFolder = `${configuration.get('globalFolder')}/cache`; + const locatorFolder = `${cacheFolder}/locator`; + + const compressionLevel = configuration.get(`compressionLevel`); + const stdout = this.context.stdout; + const gitChecksumPatches = []; // {name:, oriHash:, newHash:} + + async function patchLockfileChecksum(lockfilePath, patches) { + let currentContent = ``; + try { + currentContent = await xfs.readFilePromise(lockfilePath, `utf8`); + } catch (error) { + } + const newContent = patches.reduce((acc, item, i) => { + stdout.write(`patch '${item.name}' checksum:\n-${item.oriHash}\n+${item.newHash}\n\n\n`); + const regex = new RegExp(item.oriHash, "g"); + return acc.replace(regex, item.newHash); + }, currentContent); + + await xfs.writeFilePromise(lockfilePath, newContent); + } + + async function getLockFileMeta(lockfilePath) { + const content = await xfs.readFilePromise(lockfilePath, `utf8`); + const parsed = parseSyml(content); + return parsed.__metadata; + } + + const lockMeta = await getLockFileMeta(lockfilePath); + stdout.write(`yarn lock: ${lockfilePath}\n`); + stdout.write(`yarn lock version: ${lockMeta.version}\n`); + stdout.write(`yarn lock cacheKey: ${lockMeta.cacheKey}\n`); + + const convertToZip = async (tgz, target, opts) => { + const tgzBuf = await xfs.readFilePromise(tgz); + const fs = await tgzUtils.convertToZip(tgzBuf, opts); + fs.discardAndClose(); + await xfs.copyFilePromise(fs.path, target); + await xfs.unlinkPromise(fs.path); + } + + stdout.write(`converting tgz to zip: ${cacheFolder}\n`); + + const files = fs.readdirSync(locatorFolder); + const tasks = [] + for (const i in files) { + const file = `${files[i]}`; + let tgzFile = `${locatorFolder}/${file}`; + const match = file.match(/([^-]+)-([^.]{1,10})[.](tgz|git)$/); + if (!match) { + stdout.write(`ignore ${file}\n`); + continue; + } + let resolution, locator; + const entry_type = match[3]; + const sha = match[2]; + let checksum; + + if (entry_type === 'tgz') { + resolution = Buffer.from(match[1], 'base64').toString(); + locator = structUtils.parseLocator(resolution, true); + } + else if (entry_type === 'git') { + const gitJson = JSON.parse(fs.readFileSync(tgzFile, 'utf8')); + + resolution = gitJson.resolution; + locator = structUtils.parseLocator(resolution, true); + checksum = gitJson.checksum; + + const repoPathRel = gitJson.repo_dir_rel; + + const cloneTarget = `${cacheFolder}/${repoPathRel}`; + + const repoUrlParts = gitUtils.splitRepoUrl(locator.reference); + const packagePath = ppath.join(cloneTarget, `package.tgz`); + + await prepareExternalProject(cloneTarget, packagePath, { + configuration: configuration, + stdout, + workspace: repoUrlParts.extra.workspace, + locator, + yarn_v1: this.yarn_v1, + }); + + tgzFile = packagePath; + + } + const filename = + `${structUtils.slugifyLocator(locator)}-${lockMeta.cacheKey}.zip`; + const targetFile = `${cacheFolder}/${filename}` + + tasks.push(async () => { + await convertToZip(tgzFile, targetFile, { + compressionLevel: compressionLevel, + prefixPath: `node_modules/${structUtils.stringifyIdent(locator)}`, + stripComponents: 1, + }); + + if (entry_type === 'git') { + const file_checksum = await hashUtils.checksumFile(targetFile); + + if (file_checksum !== checksum) { + const newSha = file_checksum.slice(0, 10); + const newTarget = `${cacheFolder}/${structUtils.slugifyLocator(locator)}-${lockMeta.cacheKey}.zip`; + fs.renameSync(targetFile, newTarget); + + gitChecksumPatches.push({ + name: locator.name, + oriHash: checksum, + newHash: file_checksum, + }); + } + } + }); + } + + await Promise.all(tasks.map(t => t())); + + patchLockfileChecksum(lockfilePath, gitChecksumPatches); + stdout.write(`converting finished\n`); + } + } + return { + commands: [ + convertToZipCommand + ], + }; + } +}; diff --git a/node/flatpak_node_generator/manifest.py b/node/flatpak_node_generator/manifest.py index 9897dc6f..ad4129ff 100644 --- a/node/flatpak_node_generator/manifest.py +++ b/node/flatpak_node_generator/manifest.py @@ -168,9 +168,18 @@ def add_data_source(self, data: Union[str, bytes], destination: Path) -> None: self._add_source_with_destination(source, destination, is_dir=False) def add_git_source( - self, url: str, commit: str, destination: Optional[Path] = None + self, + url: str, + commit: Optional[str] = None, + destination: Optional[Path] = None, + tag: Optional[str] = None, ) -> None: - source = {'type': 'git', 'url': url, 'commit': commit} + source = {'type': 'git', 'url': url} + assert commit or tag + if commit: + source['commit'] = commit + if tag: + source['tag'] = tag self._add_source_with_destination(source, destination, is_dir=True) def add_script_source(self, commands: List[str], destination: Path) -> None: diff --git a/node/flatpak_node_generator/providers/special.py b/node/flatpak_node_generator/providers/special.py index 2623cba5..cae94309 100644 --- a/node/flatpak_node_generator/providers/special.py +++ b/node/flatpak_node_generator/providers/special.py @@ -349,7 +349,6 @@ async def _handle_playwright(self, package: Package) -> None: url_tp = 'https://playwright.azureedge.net/builds/chromium/%d/%s' dl_file = 'chromium-linux.zip' elif name == 'chromium-headless-shell': - # Shouldn't need the old scheme (didn't exist in 'browsers') url_tp = 'https://playwright.azureedge.net/builds/chromium/%d/%s' dl_file = 'chromium-headless-shell-linux.zip' elif name == 'firefox': diff --git a/node/flatpak_node_generator/providers/yarn.py b/node/flatpak_node_generator/providers/yarn.py index affd2e30..fedad6cd 100644 --- a/node/flatpak_node_generator/providers/yarn.py +++ b/node/flatpak_node_generator/providers/yarn.py @@ -1,10 +1,12 @@ +import base64 +import json import os import re import shlex import types import urllib.parse from pathlib import Path -from typing import Any, Dict, Iterator, List, Optional, Tuple, Type +from typing import Any, Dict, Iterator, List, NamedTuple, Optional, Tuple, Type from ..integrity import Integrity from ..manifest import ManifestGenerator @@ -16,6 +18,7 @@ PackageSource, ResolvedSource, ) +from ..requests import Requests from . import LockfileProvider, ModuleProvider, ProviderFactory, RCFileProvider from .npm import NpmRCFileProvider from .special import SpecialSourceProvider @@ -34,6 +37,10 @@ class YarnLockfileProvider(LockfileProvider): _LOCAL_PKG_RE = re.compile(r'^(?:file|link):') + def __init__(self) -> None: + self.version = 1 + self.cacheKey = str() + @staticmethod def is_git_version(version: str) -> bool: for pattern in GIT_URL_PATTERNS: @@ -56,14 +63,9 @@ def is_yarn_v1_lockfile(path: Path) -> bool: return False def parse_lockfile(self, lockfile: Path) -> Dict[str, Any]: - if not self.is_yarn_v1_lockfile(lockfile): - raise NotImplementedError( - 'Only yarn v1 lockfiles are supported. See https://classic.yarnpkg.com/' - ) - def _iter_lines() -> Iterator[Tuple[int, str]]: indent = ' ' - for line in lockfile.open(): + for line in lockfile.open(encoding='utf-8'): level = 0 current_line = line while current_line.startswith(indent): @@ -88,8 +90,10 @@ def _iter_lines() -> Iterator[Tuple[int, str]]: # to speed up parsing we can use something less robust, e.g. # _key, _value = line.split(' ', 1) # parent_entries[-1][self.unquote(_key)] = self.unquote(_value) - key, value = shlex.split(line) - parent_entries[-1][key] = value + key, *values = shlex.split(line) + if key.endswith(':'): + key = key[:-1] + parent_entries[-1][key] = values[0] if len(values) == 1 else values return root_entry @@ -100,7 +104,7 @@ def unquote(self, string: str) -> str: else: return string - def process_package( + def process_package_v1( self, lockfile: Lockfile, name_line: str, entry: Dict[str, Any] ) -> Package: assert name_line and entry @@ -128,10 +132,60 @@ def process_package( lockfile=lockfile, ) + def process_package( + self, lockfile: Lockfile, name_line: str, entry: Dict[str, Any] + ) -> Optional[Package]: + assert name_line and entry + name = self.unquote(name_line).split(',', 1)[0] + name, _ = name.rsplit('@', 1) + + # ignore patch, it will be generated by yarn + if name.find('@patch:') != -1: + return None + + if entry.get('linkType', None) == 'soft': + return None + + version: str = entry['version'] + resolution: str = entry['resolution'] + resolved: str = f'resolution#{resolution}' + lock_checksum: str = entry.get('checksum', self.cacheKey) + integrity: Integrity = Integrity( + algorithm='sha512', digest=lock_checksum.split('/')[-1] + ) + + source: PackageSource + + if self.is_git_version(resolved): + source = self.parse_git_source(version=resolved) + else: + source = ResolvedSource(resolved=resolved, integrity=integrity) + + return Package(name=name, version=version, source=source, lockfile=lockfile) + def process_lockfile(self, lockfile_path: Path) -> Iterator[Package]: - for name_line, package in self.parse_lockfile(lockfile_path).items(): - # only lockfile v1 supported - yield self.process_package(Lockfile(lockfile_path, 1), name_line, package) + lock_dict: Dict[str, Any] = self.parse_lockfile(lockfile_path) + if '__metadata' in lock_dict: + metadata: Dict[str, Any] = lock_dict['__metadata'] + self.version = int(metadata.get('version', 1)) + assert self.version > 0 + if self.version > 1: + self.cacheKey = metadata['cacheKey'] + + lock_dict.pop('__metadata') + + lockfile = Lockfile(lockfile_path, self.version) + + if self.version == 1: + for name_line, package in lock_dict.items(): + yield self.process_package_v1(lockfile, name_line, package) + else: + for name_line, package in lock_dict.items(): + res_package: Optional[Package] = self.process_package( + lockfile, name_line, package + ) + if res_package: + yield res_package class YarnRCFileProvider(RCFileProvider): @@ -139,6 +193,25 @@ class YarnRCFileProvider(RCFileProvider): class YarnModuleProvider(ModuleProvider): + class Locator(NamedTuple): + scope: str + name: str + reference: str + + _GIT_PROTOCOLS = ['commit', 'head', 'tag', 'semver'] + + class GitRepoUrlParts(NamedTuple): + repo: str + protocol: Optional[str] + request: str + extra: Optional[Dict[str, str]] + + # From https://github.com/yarnpkg/berry/blob/%40yarnpkg/shell%2F3.1.0/packages/yarnpkg-core/sources/structUtils.ts#L412 + _RESOLUTION_RE = re.compile(r'^(?:@([^/]+?)\/)?([^/]+?)(?:@(.+))$') + # From https://github.com/yarnpkg/berry/blob/%40yarnpkg/shell%2F3.1.0/packages/yarnpkg-core/sources/structUtils.ts#L462 + _REFERENCE_RE = re.compile( + r'^([^#:]*:)?((?:(?!::)[^#])*)(?:#((?:(?!::).)*))?(?:::(.*))?$' + ) # From https://github.com/yarnpkg/yarn/blob/v1.22.4/src/fetchers/tarball-fetcher.js _PACKAGE_TARBALL_URL_RE = re.compile( r'(?:(@[^/]+)(?:/|%2f))?[^/]+/(?:-|_attachments)/(?:@[^/]+/)?([^/]+)$' @@ -148,6 +221,10 @@ def __init__(self, gen: ManifestGenerator, special: SpecialSourceProvider) -> No self.gen = gen self.special_source_provider = special self.mirror_dir = self.gen.data_root / 'yarn-mirror' + self.mirror_berry_dir = self.mirror_dir / 'global' / 'cache' + self.mirror_locator_dir = self.mirror_berry_dir / 'locator' + self.registry = 'https://registry.yarnpkg.com' + self.has_resolution = False def __exit__( self, @@ -155,25 +232,148 @@ def __exit__( exc_value: Optional[BaseException], tb: Optional[types.TracebackType], ) -> None: - pass + self._finalize() + + def get_resolution_from_resolved(self, resolved: str) -> str: + assert resolved.startswith('resolution#') + return resolved[len('resolution#') :] + + def get_locator_url(self, locator: Locator) -> str: + if locator.scope: + return f'/@{locator.scope}%2f{locator.name}' + else: + return f'/{locator.name}' + + def get_locator_from_resolution(self, resolution: str) -> Locator: + match = self._RESOLUTION_RE.match(resolution) + assert match + scope, name, ref = [s or '' for s in match.groups()] + return self.Locator(scope=scope, name=name, reference=ref) + + def name_base64_locator(self, locator: Locator, resolution: str) -> str: + return f'{locator.name}-{base64.b64encode(resolution.encode()).decode()}' + + # From https://github.com/yarnpkg/berry/blob/%40yarnpkg/shell%2F3.1.0/packages/plugin-git/sources/gitUtils.ts#L56 + def parse_git_subsequent(self, url: str) -> GitRepoUrlParts: + repo, subsequent = url.split('#', 1) + protocol: Optional[str] = None + request: str = '' + extra: Dict[str, str] = {} + if not subsequent: + return self.GitRepoUrlParts( + repo=repo, protocol='head', request='HEAD', extra=None + ) + if re.match(r'^[a-z]+=', subsequent): + queries = urllib.parse.parse_qs(subsequent) + for q in queries.keys(): + if q in self._GIT_PROTOCOLS: + protocol = q + request = queries[q][0] + else: + extra[q] = queries[q][-1] + if not request: + protocol, request = 'head', 'HEAD' + return self.GitRepoUrlParts( + repo=repo, protocol=protocol, request=request, extra=extra + ) + else: + protocol, request = subsequent.split(':', 1) + if not request: + protocol, request = None, subsequent + return self.GitRepoUrlParts( + repo=repo, protocol=protocol, request=request, extra=None + ) + + async def resolve_source(self, locator: Locator, version: str) -> ResolvedSource: + data_url = f'{self.registry}{self.get_locator_url(locator)}' + # NOTE: Not cachable, because this is an API call. + raw_data = await Requests.instance.read_all(data_url, cachable=False) + data = json.loads(raw_data) + + assert 'versions' in data, f'{data_url} returned an invalid package index' + + versions = data['versions'] + assert ( + version in versions + ), f'{locator.name} versions available are {", ".join(versions)}, not {version}' + + dist = versions[version]['dist'] + assert 'tarball' in dist, f'{locator.name}@{version} has no tarball in dist' + + integrity: Integrity + if 'integrity' in dist: + integrity = Integrity.parse(dist['integrity']) + elif 'shasum' in dist: + integrity = Integrity.from_sha1(dist['shasum']) + else: + assert False, f'{locator.name}@{version} has no integrity in dist' + + return ResolvedSource(resolved=dist['tarball'], integrity=integrity) async def generate_package(self, package: Package) -> None: source = package.source if isinstance(source, ResolvedSource): - integrity = await source.retrieve_integrity() - url_parts = urllib.parse.urlparse(source.resolved) - match = self._PACKAGE_TARBALL_URL_RE.search(url_parts.path) - if match is not None: - scope, filename = match.groups() - if scope: - filename = f'{scope}-{filename}' + if source.resolved.startswith('resolution#'): + if not self.has_resolution: + self.has_resolution = True + assert source.integrity, f'{source.resolved}' + resolution = self.get_resolution_from_resolved(source.resolved) + locator = self.get_locator_from_resolution(resolution) + if YarnLockfileProvider.is_git_version(locator.reference): + filename = f'{self.name_base64_locator(locator, "git")}-{source.integrity.digest[:10]}.git' + git_parts = self.parse_git_subsequent(locator.reference) + repo_dir = self.gen.tmp_root / locator.name + if git_parts.protocol == 'commit' or git_parts.protocol is None: + self.gen.add_git_source( + git_parts.repo, + commit=git_parts.request, + destination=repo_dir, + ) + elif git_parts.protocol == 'tag': + self.gen.add_git_source( + git_parts.repo, tag=git_parts.request, destination=repo_dir + ) + else: + assert ( + False + ), f'Not supported git protocol: {git_parts.protocol}' + repo_dir_rel = os.path.relpath(repo_dir, self.mirror_berry_dir) + self.gen.add_data_source( + json.dumps( + { + 'repo_dir_rel': repo_dir_rel, + 'resolution': resolution, + 'checksum': source.integrity.digest, + } + ), + destination=self.mirror_locator_dir / filename, + ) + else: + filename = f'{self.name_base64_locator(locator, resolution)}-{source.integrity.digest[:10]}.tgz' + resolved_source = await self.resolve_source( + locator, package.version + ) + assert resolved_source.integrity + self.gen.add_url_source( + resolved_source.resolved, + resolved_source.integrity, + self.mirror_locator_dir / filename, + ) else: - filename = os.path.basename(url_parts.path) + integrity = await source.retrieve_integrity() + url_parts = urllib.parse.urlparse(source.resolved) + match = self._PACKAGE_TARBALL_URL_RE.search(url_parts.path) + if match is not None: + scope, filename = match.groups() + if scope: + filename = f'{scope}-{filename}' + else: + filename = os.path.basename(url_parts.path) - self.gen.add_url_source( - source.resolved, integrity, self.mirror_dir / filename - ) + self.gen.add_url_source( + source.resolved, integrity, self.mirror_dir / filename + ) elif isinstance(source, GitSource): repo_name = urllib.parse.urlparse(source.url).path.split('/')[-1] @@ -199,6 +399,17 @@ async def generate_package(self, package: Package) -> None: await self.special_source_provider.generate_special_sources(package) + def _finalize(self) -> None: + if not self.has_resolution: + return + + with open( + Path(__file__).parents[1] / 'flatpak-yarn.js', mode='r', encoding='utf-8' + ) as f: + yarn2_plugin_source = f.read() + js_dest = self.gen.data_root / 'flatpak-yarn.js' + self.gen.add_data_source(yarn2_plugin_source, destination=js_dest) + class YarnProviderFactory(ProviderFactory): def __init__(self) -> None: From 0848ee3b9b569f0f16b047168aca1cdec3bda3e6 Mon Sep 17 00:00:00 2001 From: catsout Date: Sat, 20 Dec 2025 11:40:55 +0800 Subject: [PATCH 2/6] node: drop node 14,16,18 in tests as runtime eof --- node/tests/conftest.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/node/tests/conftest.py b/node/tests/conftest.py index 8946cf4f..d4ccb368 100644 --- a/node/tests/conftest.py +++ b/node/tests/conftest.py @@ -56,7 +56,7 @@ def requests() -> Iterator[RequestsController]: _DEFAULT_MODULE = 'module' -_DEFAULT_NODE = 16 +_DEFAULT_NODE = 20 @dataclass @@ -99,11 +99,9 @@ def build( sdk_extensions = [] build_options = {} NODE_RUNTIME_VERSION_MAP = { - '14': '22.08', - '16': '22.08', - '18': '22.08', - '20': '24.08', - '22': '24.08', + '20': '25.08', + '22': '25.08', + '24': '25.08', } if use_node: @@ -121,9 +119,9 @@ def build( for i, command in enumerate(commands): commands[i] = f'. /usr/lib/sdk/node{use_node}/enable.sh && {command}' - runtime_version = NODE_RUNTIME_VERSION_MAP.get(use_node_str, '24.08') + runtime_version = NODE_RUNTIME_VERSION_MAP.get(use_node_str, '25.08') else: - runtime_version = '22.08' + runtime_version = '25.08' manifest = { 'id': 'com.test.Test', @@ -322,7 +320,7 @@ def provider_factory_spec(request: Any, shared_datadir: Path) -> ProviderFactory return ProviderFactorySpec(datadir=shared_datadir, type=type) -@pytest.fixture(params=[14, 16, 18, 20, 22]) +@pytest.fixture(params=[20, 22, 24]) def node_version(request: Any) -> int: version = request.param assert isinstance(version, int) From d87d4b55065e560f2876ec156c3237f3244b29b6 Mon Sep 17 00:00:00 2001 From: catsout Date: Sat, 20 Dec 2025 11:48:27 +0800 Subject: [PATCH 3/6] node: update python and flatpak runtime in workflow --- .github/workflows/node.yaml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/node.yaml b/.github/workflows/node.yaml index e42ea7a2..acf5b402 100644 --- a/.github/workflows/node.yaml +++ b/.github/workflows/node.yaml @@ -23,11 +23,9 @@ jobs: fail-fast: false matrix: python-version: - - '3.9' - - '3.10' - - '3.11' - '3.12' - '3.13' + - '3.14' runs-on: ubuntu-latest steps: # 4.2.2 @@ -57,9 +55,7 @@ jobs: run: | flatpak --user remote-add flathub https://flathub.org/repo/flathub.flatpakrepo flatpak --user install -y flathub \ - org.freedesktop.{Platform,Sdk{,.Extension.node{14,16,18}}}//22.08 - flatpak --user install -y flathub \ - org.freedesktop.{Platform,Sdk{,.Extension.node{20,22}}}//24.08 + org.freedesktop.{Platform,Sdk{,.Extension.node{20,22,24}}}//25.08 - name: Install dependencies run: poetry install --with=dev From 3511c98fd05e9e8368fd0744e209c2f9083c6f6d Mon Sep 17 00:00:00 2001 From: catsout Date: Sat, 20 Dec 2025 15:42:29 +0800 Subject: [PATCH 4/6] node: add test_yarn_lock_8 --- node/flatpak_node_generator/package.py | 1 + node/flatpak_node_generator/providers/yarn.py | 8 +- node/tests/test_yarn_lock_8.py | 97 +++++++++++++++++++ 3 files changed, 102 insertions(+), 4 deletions(-) create mode 100644 node/tests/test_yarn_lock_8.py diff --git a/node/flatpak_node_generator/package.py b/node/flatpak_node_generator/package.py index d2d9cdef..72077bbb 100644 --- a/node/flatpak_node_generator/package.py +++ b/node/flatpak_node_generator/package.py @@ -140,6 +140,7 @@ class LocalSource(PackageSource): class Lockfile: path: Path version: int + cache_key: Optional[str] = None class Package(NamedTuple): diff --git a/node/flatpak_node_generator/providers/yarn.py b/node/flatpak_node_generator/providers/yarn.py index fedad6cd..4398dd83 100644 --- a/node/flatpak_node_generator/providers/yarn.py +++ b/node/flatpak_node_generator/providers/yarn.py @@ -39,7 +39,6 @@ class YarnLockfileProvider(LockfileProvider): def __init__(self) -> None: self.version = 1 - self.cacheKey = str() @staticmethod def is_git_version(version: str) -> bool: @@ -149,7 +148,7 @@ def process_package( version: str = entry['version'] resolution: str = entry['resolution'] resolved: str = f'resolution#{resolution}' - lock_checksum: str = entry.get('checksum', self.cacheKey) + lock_checksum: str = entry.get('checksum', lockfile.cache_key or '') integrity: Integrity = Integrity( algorithm='sha512', digest=lock_checksum.split('/')[-1] ) @@ -165,16 +164,17 @@ def process_package( def process_lockfile(self, lockfile_path: Path) -> Iterator[Package]: lock_dict: Dict[str, Any] = self.parse_lockfile(lockfile_path) + cache_key = None if '__metadata' in lock_dict: metadata: Dict[str, Any] = lock_dict['__metadata'] self.version = int(metadata.get('version', 1)) assert self.version > 0 if self.version > 1: - self.cacheKey = metadata['cacheKey'] + cache_key = metadata['cacheKey'] lock_dict.pop('__metadata') - lockfile = Lockfile(lockfile_path, self.version) + lockfile = Lockfile(lockfile_path, self.version, cache_key) if self.version == 1: for name_line, package in lock_dict.items(): diff --git a/node/tests/test_yarn_lock_8.py b/node/tests/test_yarn_lock_8.py new file mode 100644 index 00000000..3604d3c7 --- /dev/null +++ b/node/tests/test_yarn_lock_8.py @@ -0,0 +1,97 @@ +from pathlib import Path + +from flatpak_node_generator.integrity import Integrity +from flatpak_node_generator.package import ( + Lockfile, + Package, + ResolvedSource, +) +from flatpak_node_generator.providers.yarn import YarnLockfileProvider + +TEST_LOCKFILE = """ +# This file is generated by running "yarn install" inside your project. +# Manual changes might be lost - proceed with caution! + +__metadata: + version: 8 + cacheKey: 10c0 + +"bling@npm:^0.12.3": + version: 0.12.3 + resolution: "bling@npm:0.12.3" + checksum: 10c0/44a1686ba11075c72721980a3e4b881656497fef93873f900a702f9de297ce8b1caf858711cb3da89cb3121b0b13fd2174b890785a85240e68f9dd0f329f2136 + languageName: node + linkType: hard + +"isexe@https://github.com/isaacs/isexe": + version: 3.1.1 + resolution: "isexe@https://github.com/isaacs/isexe.git#commit=e3078d94e64cb587201a290bdef17916d1342cf8" + checksum: 10c0/cdf2ea84ddec39437ec5595e61d797dd8bb4e39f847f5b1abede325c650533b5053b2b90613a3b7687c46ab120e2565ccb84573f816fa7bf20017479aeb83a9d + languageName: node + linkType: hard + +"test@workspace:.": + version: 0.0.0-use.local + resolution: "test@workspace:." + dependencies: + bling: "npm:^0.12.3" + isexe: "https://github.com/isaacs/isexe" + thing: "npm:^1.0.1" + languageName: unknown + linkType: soft + +"thing@npm:^1.0.1": + version: 1.0.1 + resolution: "thing@npm:1.0.1" + checksum: 10c0/52e193e281a1f45b503e0863278cfece75ce0b188d2d8740def7af7fb87aeed14ee36ff777ba7969a510a23287b7b6c719c6c2bfc1313cb65dfa0a9799a0f3dc + languageName: node + linkType: hard +""" + + +def test_lockfile_parsing(tmp_path: Path) -> None: + lockfile_provider = YarnLockfileProvider() + + yarn_lock = Lockfile(tmp_path / 'yarn.lock', 8, '10c0') + yarn_lock.path.write_text(TEST_LOCKFILE) + + packages = list(lockfile_provider.process_lockfile(yarn_lock.path)) + + assert packages == [ + Package( + lockfile=yarn_lock, + name='bling', + version='0.12.3', + source=ResolvedSource( + resolved='resolution#bling@npm:0.12.3', + integrity=Integrity( + 'sha512', + '44a1686ba11075c72721980a3e4b881656497fef93873f900a702f9de297ce8b1caf858711cb3da89cb3121b0b13fd2174b890785a85240e68f9dd0f329f2136', + ), + ), + ), + Package( + lockfile=yarn_lock, + name='isexe', + version='3.1.1', + source=ResolvedSource( + resolved='resolution#isexe@https://github.com/isaacs/isexe.git#commit=e3078d94e64cb587201a290bdef17916d1342cf8', + integrity=Integrity( + 'sha512', + 'cdf2ea84ddec39437ec5595e61d797dd8bb4e39f847f5b1abede325c650533b5053b2b90613a3b7687c46ab120e2565ccb84573f816fa7bf20017479aeb83a9d', + ), + ), + ), + Package( + lockfile=yarn_lock, + name='thing', + version='1.0.1', + source=ResolvedSource( + resolved='resolution#thing@npm:1.0.1', + integrity=Integrity( + 'sha512', + '52e193e281a1f45b503e0863278cfece75ce0b188d2d8740def7af7fb87aeed14ee36ff777ba7969a510a23287b7b6c719c6c2bfc1313cb65dfa0a9799a0f3dc', + ), + ), + ), + ] From 20f5fbbd4d069f2fc89d2520490c72e5c43c3070 Mon Sep 17 00:00:00 2001 From: catsout Date: Sat, 20 Dec 2025 19:01:44 +0800 Subject: [PATCH 5/6] node: change yarn mirror_berry_dir --- node/flatpak_node_generator/providers/yarn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/flatpak_node_generator/providers/yarn.py b/node/flatpak_node_generator/providers/yarn.py index 4398dd83..138692b8 100644 --- a/node/flatpak_node_generator/providers/yarn.py +++ b/node/flatpak_node_generator/providers/yarn.py @@ -221,7 +221,7 @@ def __init__(self, gen: ManifestGenerator, special: SpecialSourceProvider) -> No self.gen = gen self.special_source_provider = special self.mirror_dir = self.gen.data_root / 'yarn-mirror' - self.mirror_berry_dir = self.mirror_dir / 'global' / 'cache' + self.mirror_berry_dir = self.gen.data_root / 'yarn-berry' / 'cache' self.mirror_locator_dir = self.mirror_berry_dir / 'locator' self.registry = 'https://registry.yarnpkg.com' self.has_resolution = False From c97bfc82cde83511f42f85a214f522bd660926fd Mon Sep 17 00:00:00 2001 From: catsout Date: Sat, 20 Dec 2025 19:02:12 +0800 Subject: [PATCH 6/6] node: add fiddle-yarn-berry showcase --- node/README.md | 5 +- node/fiddle-yarn-berry/.gitignore | 3 + node/fiddle-yarn-berry/README.md | 12 +++ node/fiddle-yarn-berry/gen.sh | 9 ++ .../org.electronjs.fiddle.yaml | 89 +++++++++++++++++++ .../replace/contributors.json | 20 +++++ .../replace/fetch-releases.ts | 22 +++++ 7 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 node/fiddle-yarn-berry/.gitignore create mode 100644 node/fiddle-yarn-berry/README.md create mode 100755 node/fiddle-yarn-berry/gen.sh create mode 100644 node/fiddle-yarn-berry/org.electronjs.fiddle.yaml create mode 100644 node/fiddle-yarn-berry/replace/contributors.json create mode 100644 node/fiddle-yarn-berry/replace/fetch-releases.ts diff --git a/node/README.md b/node/README.md index bef37e7d..d7c40569 100644 --- a/node/README.md +++ b/node/README.md @@ -11,7 +11,7 @@ required, however, you can disable it explicitly via `--no-xdg-layout`. ## Requirements - flatpak-builder 1.1.2 or newer -- Python 3.9+. +- Python 3.11+. - [pipx](https://pypa.github.io/pipx/) (recommended) or [pip](https://pip.pypa.io/en/stable/) (both of these are usually available in your distro repositories, the latter is often included with Python installs). @@ -40,6 +40,9 @@ There are two examples provided for how to use flatpak-node-generator: - `webpack-quick-start` - A Flatpak of [electron-webpack-quick-start](https://github.com/electron-userland/electron-webpack-quick-start). It uses yarn for package management and electron-builder + webpack. +- `fiddle-yarn-berry` - A Flatpak of + [electron-fiddle](https://github.com/electron/fiddle.git). It uses yarn/berry for + package management and a rather basic Electron workflow. Both manifests have comments to highlight their differences, so you can mix and match to e.g. get npm with electron-builder. diff --git a/node/fiddle-yarn-berry/.gitignore b/node/fiddle-yarn-berry/.gitignore new file mode 100644 index 00000000..d079245c --- /dev/null +++ b/node/fiddle-yarn-berry/.gitignore @@ -0,0 +1,3 @@ +/yarn.lock +/generated-sources.json +/build \ No newline at end of file diff --git a/node/fiddle-yarn-berry/README.md b/node/fiddle-yarn-berry/README.md new file mode 100644 index 00000000..1c744b5a --- /dev/null +++ b/node/fiddle-yarn-berry/README.md @@ -0,0 +1,12 @@ +# vanilla-quick-start + +Showcases building a Flatpak of [electron-fiddle](https://github.com/electron/fiddle.git) + +To create generated-sources.json run: + +```sh +./gen.sh +``` + +## Run app in Fiddle +Add `--no-sandbox` to `Prefrences -> Execution -> Electron Flags` diff --git a/node/fiddle-yarn-berry/gen.sh b/node/fiddle-yarn-berry/gen.sh new file mode 100755 index 00000000..675e0755 --- /dev/null +++ b/node/fiddle-yarn-berry/gen.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +dir="$(dirname $(realpath $0))" +commit=98c0605f804df86df8636a477515bbe178cd3943 + +pushd "$dir/.." +curl -L "https://github.com/electron/fiddle/raw/$commit/yarn.lock" > "$dir/yarn.lock" +poetry run flatpak-node-generator yarn -o "$dir/generated-sources.json" "$dir/yarn.lock" +popd diff --git a/node/fiddle-yarn-berry/org.electronjs.fiddle.yaml b/node/fiddle-yarn-berry/org.electronjs.fiddle.yaml new file mode 100644 index 00000000..3bb90dd7 --- /dev/null +++ b/node/fiddle-yarn-berry/org.electronjs.fiddle.yaml @@ -0,0 +1,89 @@ +app-id: org.electronjs.fiddle +runtime: org.freedesktop.Platform +runtime-version: '25.08' +sdk: org.freedesktop.Sdk +# Use the Electron 2 BaseApp, which adds several common libraries we'll need. +base: org.electronjs.Electron2.BaseApp +base-version: '25.08' +separate-locales: false +# Add the Node SDK extension. +sdk-extensions: + - org.freedesktop.Sdk.Extension.node24 +command: fiddle +finish-args: + # These three lines add the permissions needed for graphics. + - --device=dri + - --share=ipc + - --socket=x11 + - --socket=wayland + # Sound access. + - --socket=pulseaudio + # Network access. + - --share=network + # If you need to access the filesystem, also add: + # - --filesystem=home +modules: + # Now is the quickstart module. + - name: fiddle + buildsystem: simple + build-options: + # Add the node bin directory. + append-path: /usr/lib/sdk/node24/bin + env: + XDG_CACHE_HOME: /run/build/fiddle/flatpak-node/cache + npm_config_nodedir: /usr/lib/sdk/node24 + # more verbose + YARN_ENABLE_INLINE_BUILDS: '1' + # disable telemetry + YARN_ENABLE_TELEMETRY: '0' + # disable network access + YARN_ENABLE_NETWORK: '0' + # disable global cache + YARN_ENABLE_GLOBAL_CACHE: '0' + # set yarn global folder + YARN_GLOBAL_FOLDER: /run/build/fiddle/flatpak-node/yarn-berry + build-commands: + # print yarn basic info + - yarn config + # setup yarn for flatpak + - yarn plugin import $FLATPAK_BUILDER_BUILDDIR/flatpak-node/flatpak-yarn.js + # prepare yarn cache + - yarn convertToZip $(which yarn) + # Install the packages from our offline cache. + # --prefix= is the path to our subdirectory (see the electron-quick-start source below). + - yarn install + + - yarn run package + - mv out/Electron* /app/ElectronFiddle + - install -Dm755 fiddle.sh /app/bin/fiddle + sources: + - type: git + url: https://github.com/electron/fiddle.git + commit: 98c0605f804df86df8636a477515bbe178cd3943 + + # Add the flatpak-node-generator generated sources. + - generated-sources.json + # Our runner script. + - type: script + dest-filename: fiddle.sh + commands: + # We need to wrap the main binary with Zypak in order for sandboxing + # to work. Without this, we'll get errors about the "SUID sandbox + # helper binary". + - exec zypak-wrapper /app/ElectronFiddle/electron-fiddle + + # Sources below are only for fiddle, as it wants to fetch these online + # prefetch for offline + - type: file + url: https://releases.electronjs.org/releases.json + sha256: badb8576b81b4e43716aa3314724c9563d42eb4140cf06000201ad7a31067dc4 + # replace to use release.json prefetched + - type: file + path: replace/fetch-releases.ts + dest-filename: fetch-releases.ts + dest: tools + # used embed contributors.json + - type: file + path: replace/contributors.json + dest-filename: contributors.json + dest: static diff --git a/node/fiddle-yarn-berry/replace/contributors.json b/node/fiddle-yarn-berry/replace/contributors.json new file mode 100644 index 00000000..481ab31a --- /dev/null +++ b/node/fiddle-yarn-berry/replace/contributors.json @@ -0,0 +1,20 @@ +[ + { + "url": "https://github.com/felixrieseberg", + "api": "https://api.github.com/users/felixrieseberg", + "login": "felixrieseberg", + "avatar": "https://avatars.githubusercontent.com/u/1426799?v=4", + "name": "Felix Rieseberg", + "bio": "🙇 ✨🌳 ", + "location": "San Francisco" + }, + { + "url": "https://github.com/dsanders11", + "api": "https://api.github.com/users/dsanders11", + "login": "dsanders11", + "avatar": "https://avatars.githubusercontent.com/u/5820654?v=4", + "name": "David Sanders", + "bio": null, + "location": "Santa Barbara, CA" + } +] \ No newline at end of file diff --git a/node/fiddle-yarn-berry/replace/fetch-releases.ts b/node/fiddle-yarn-berry/replace/fetch-releases.ts new file mode 100644 index 00000000..0e7f8f66 --- /dev/null +++ b/node/fiddle-yarn-berry/replace/fetch-releases.ts @@ -0,0 +1,22 @@ +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +const outputFile = path.join(__dirname, '..', 'static', 'releases.json'); + +export async function populateReleases() { + const sourceDir = process.env.FLATPAK_BUILDER_BUILDDIR; + + if (!sourceDir) { + throw new Error('FLATPAK_BUILDER_BUILDDIR is not set.'); + } + + const inputFile = path.resolve(sourceDir, 'releases.json'); + const data = await fs.promises.readFile(inputFile, 'utf-8'); + await fs.promises.writeFile(outputFile, data); +} + +if (require.main === module) { + (async () => { + await populateReleases(); + })(); +}