fix(node): pnpm - support v10 offline tarballs

Author: nukeop

node/flatpak_node_generator/populate_pnpm_store.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import base64
+import contextlib
 import hashlib
 import json
 import os
@@ -37,6 +38,8 @@ def populate_store(manifest_path: str, tarball_dir: str, store_dir: str) -> None
             integrity_hex=info['integrity_hex'],
             store=store,
             now=now,
+            tarball_url=info.get('tarball_url'),
+            store_version=store_version,
         )
 
 
@@ -48,8 +51,12 @@ def _process_tarball(
     integrity_hex: str,
     store: str,
     now: int,
+    tarball_url: str | None = None,
+    store_version: str = 'v3',
 ) -> None:
     index_files: dict[str, dict[str, object]] = {}
+    real_pkg_name = pkg_name
+    real_pkg_version = pkg_version
 
     with tarfile.open(tarball_path, 'r:gz') as tf:
         for member in tf.getmembers():
@@ -60,6 +67,17 @@ def _process_tarball(
                 continue
             data = fobj.read()
 
+            if member.name.endswith('package.json') and member.name.count('/') <= 1:
+                with contextlib.suppress(ValueError, TypeError, UnicodeDecodeError):
+                    pkg_data = json.loads(data.decode('utf-8'))
+                    if isinstance(pkg_data, dict):
+                        if 'name' in pkg_data and isinstance(pkg_data['name'], str):
+                            real_pkg_name = pkg_data['name']
+                        if 'version' in pkg_data and isinstance(
+                            pkg_data['version'], str
+                        ):
+                            real_pkg_version = pkg_data['version']
+
             digest = hashlib.sha512(data).digest()
             file_hex = digest.hex()
             is_exec = bool(member.mode & 0o111)
@@ -86,20 +104,42 @@ def _process_tarball(
                 'size': len(data),
             }
 
+    index_data = {
+        'name': real_pkg_name,
+        'version': real_pkg_version,
+        'requiresBuild': False,
+        'files': index_files,
+    }
+
     idx_prefix = integrity_hex[:2]
     idx_rest = integrity_hex[2:64]
     pkg_id = _SANITIZE_RE.sub('+', f'{pkg_name}@{pkg_version}')
     idx_dir = os.path.join(store, 'index', idx_prefix)
     os.makedirs(idx_dir, exist_ok=True)
     idx_path = os.path.join(idx_dir, f'{idx_rest}-{pkg_id}.json')
-    index_data = {
-        'name': pkg_name,
-        'version': pkg_version,
-        'files': index_files,
-    }
     with open(idx_path, 'w', encoding='utf-8') as out:
         json.dump(index_data, out)
 
+    # For tarball-URL packages, also create an index entry keyed by the URL hash
+    # this is how pnpm looks up tarball deps without integrity
+    if tarball_url:
+        if store_version == 'v3':
+            url_hash = hashlib.sha256(tarball_url.encode()).hexdigest()
+            url_idx_prefix = url_hash[:2]
+            url_idx_rest = url_hash[2:64]
+            url_idx_dir = os.path.join(store, 'index', url_idx_prefix)
+            os.makedirs(url_idx_dir, exist_ok=True)
+            url_idx_path = os.path.join(url_idx_dir, f'{url_idx_rest}-{pkg_id}.json')
+            with open(url_idx_path, 'w', encoding='utf-8') as out:
+                json.dump(index_data, out)
+        else:
+            url_dir_name = re.sub(r'[:/]', '+', tarball_url)
+            url_idx_dir = os.path.join(store, url_dir_name)
+            os.makedirs(url_idx_dir, exist_ok=True)
+            url_idx_path = os.path.join(url_idx_dir, 'integrity.json')
+            with open(url_idx_path, 'w', encoding='utf-8') as out:
+                json.dump(index_data, out)
+
 
 if __name__ == '__main__':
     if len(sys.argv) != 4:

node/flatpak_node_generator/providers/pnpm.py

@@ -189,27 +189,27 @@ async def generate_package(self, package: Package) -> None:
         if isinstance(source, ResolvedSource):
             assert source.resolved is not None
 
-            if source.integrity is None:
+            integrity = source.integrity
+            if integrity is None:
                 print(
-                    f'WARNING: skipping {package.name}@{package.version}: '
-                    'no integrity in lockfile (required for pnpm store)',
+                    f'INFO: {package.name}@{package.version}: '
+                    'no integrity in lockfile, fetching to compute...',
                     file=sys.stderr,
                 )
-                return
+                integrity = await source.retrieve_integrity()
 
-            # Use name-version as filename; replace / in scoped names
             tarball_name = f'{package.name.replace("/", "__")}-{package.version}.tgz'
             self.gen.add_url_source(
                 url=source.resolved,
-                integrity=source.integrity,
+                integrity=integrity,
                 destination=self.tarball_dir / tarball_name,
             )
             self._tarballs.append(
                 self._TarballInfo(
                     tarball_name=tarball_name,
                     name=package.name,
                     version=package.version,
-                    integrity=source.integrity,
+                    integrity=integrity,
                 )
             )
 
@@ -236,11 +236,14 @@ def _finalize(self) -> None:
     def _add_store_population_script(self) -> None:
         packages = {}
         for info in self._tarballs:
-            packages[info.tarball_name] = {
+            entry: dict[str, str] = {
                 'name': info.name,
                 'version': info.version,
                 'integrity_hex': info.integrity.digest,
             }
+            if info.version.startswith(('http://', 'https://')):
+                entry['tarball_url'] = info.version
+            packages[info.tarball_name] = entry
 
         manifest = {
             'store_version': self._store_version,