node: Support v10 offline tarballs for pnpm generator

Author: tonur

node/flatpak_node_generator/populate_pnpm_store.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import base64
+import contextlib
 import hashlib
 import json
 import os
@@ -37,6 +38,8 @@ def populate_store(manifest_path: str, tarball_dir: str, store_dir: str) -> None
             integrity_hex=info['integrity_hex'],
             store=store,
             now=now,
+            tarball_url=info.get('tarball_url'),
+            store_version=store_version,
         )
 
 
@@ -48,8 +51,12 @@ def _process_tarball(
     integrity_hex: str,
     store: str,
     now: int,
+    tarball_url: str | None = None,
+    store_version: str = 'v3',
 ) -> None:
     index_files: dict[str, dict[str, object]] = {}
+    real_pkg_name = pkg_name
+    real_pkg_version = pkg_version
 
     with tarfile.open(tarball_path, 'r:gz') as tf:
         for member in tf.getmembers():
@@ -60,6 +67,17 @@ def _process_tarball(
                 continue
             data = fobj.read()
 
+            if member.name.endswith('package.json') and member.name.count('/') <= 1:
+                with contextlib.suppress(ValueError, TypeError, UnicodeDecodeError):
+                    pkg_data = json.loads(data.decode('utf-8'))
+                    if isinstance(pkg_data, dict):
+                        if 'name' in pkg_data and isinstance(pkg_data['name'], str):
+                            real_pkg_name = pkg_data['name']
+                        if 'version' in pkg_data and isinstance(
+                            pkg_data['version'], str
+                        ):
+                            real_pkg_version = pkg_data['version']
+
             digest = hashlib.sha512(data).digest()
             file_hex = digest.hex()
             is_exec = bool(member.mode & 0o111)
@@ -86,20 +104,42 @@ def _process_tarball(
                 'size': len(data),
             }
 
+    index_data = {
+        'name': real_pkg_name,
+        'version': real_pkg_version,
+        'requiresBuild': False,
+        'files': index_files,
+    }
+
     idx_prefix = integrity_hex[:2]
     idx_rest = integrity_hex[2:64]
     pkg_id = _SANITIZE_RE.sub('+', f'{pkg_name}@{pkg_version}')
     idx_dir = os.path.join(store, 'index', idx_prefix)
     os.makedirs(idx_dir, exist_ok=True)
     idx_path = os.path.join(idx_dir, f'{idx_rest}-{pkg_id}.json')
-    index_data = {
-        'name': pkg_name,
-        'version': pkg_version,
-        'files': index_files,
-    }
     with open(idx_path, 'w', encoding='utf-8') as out:
         json.dump(index_data, out)
 
+    # For tarball-URL packages, also create an index entry keyed by the URL hash
+    # this is how pnpm looks up tarball deps without integrity
+    if tarball_url:
+        if store_version == 'v3':
+            url_hash = hashlib.sha256(tarball_url.encode()).hexdigest()
+            url_idx_prefix = url_hash[:2]
+            url_idx_rest = url_hash[2:64]
+            url_idx_dir = os.path.join(store, 'index', url_idx_prefix)
+            os.makedirs(url_idx_dir, exist_ok=True)
+            url_idx_path = os.path.join(url_idx_dir, f'{url_idx_rest}-{pkg_id}.json')
+            with open(url_idx_path, 'w', encoding='utf-8') as out:
+                json.dump(index_data, out)
+        else:
+            url_dir_name = re.sub(r'[:/]', '+', tarball_url)
+            url_idx_dir = os.path.join(store, url_dir_name)
+            os.makedirs(url_idx_dir, exist_ok=True)
+            url_idx_path = os.path.join(url_idx_dir, 'integrity.json')
+            with open(url_idx_path, 'w', encoding='utf-8') as out:
+                json.dump(index_data, out)
+
 
 if __name__ == '__main__':
     if len(sys.argv) != 4:

node/flatpak_node_generator/providers/pnpm.py

@@ -189,27 +189,27 @@ async def generate_package(self, package: Package) -> None:
         if isinstance(source, ResolvedSource):
             assert source.resolved is not None
 
-            if source.integrity is None:
+            integrity = source.integrity
+            if integrity is None:
                 print(
-                    f'WARNING: skipping {package.name}@{package.version}: '
-                    'no integrity in lockfile (required for pnpm store)',
+                    f'INFO: {package.name}@{package.version}: '
+                    'no integrity in lockfile, fetching to compute...',
                     file=sys.stderr,
                 )
-                return
+                integrity = await source.retrieve_integrity()
 
-            # Use name-version as filename; replace / in scoped names
             tarball_name = f'{package.name.replace("/", "__")}-{package.version}.tgz'
             self.gen.add_url_source(
                 url=source.resolved,
-                integrity=source.integrity,
+                integrity=integrity,
                 destination=self.tarball_dir / tarball_name,
             )
             self._tarballs.append(
                 self._TarballInfo(
                     tarball_name=tarball_name,
                     name=package.name,
                     version=package.version,
-                    integrity=source.integrity,
+                    integrity=integrity,
                 )
             )
 
@@ -236,11 +236,14 @@ def _finalize(self) -> None:
     def _add_store_population_script(self) -> None:
         packages = {}
         for info in self._tarballs:
-            packages[info.tarball_name] = {
+            entry: dict[str, str] = {
                 'name': info.name,
                 'version': info.version,
                 'integrity_hex': info.integrity.digest,
             }
+            if info.version.startswith(('http://', 'https://')):
+                entry['tarball_url'] = info.version
+            packages[info.tarball_name] = entry
 
         manifest = {
             'store_version': self._store_version,

node/tests/test_pnpm.py

@@ -1,16 +1,24 @@
+import hashlib
+import json
 from pathlib import Path
 
 import pytest
+from conftest import RequestsController
 
 from flatpak_node_generator.integrity import Integrity
+from flatpak_node_generator.manifest import ManifestGenerator
 from flatpak_node_generator.package import (
     GitSource,
     LocalSource,
     Lockfile,
     Package,
     ResolvedSource,
 )
-from flatpak_node_generator.providers.pnpm import PnpmLockfileProvider
+from flatpak_node_generator.providers.pnpm import (
+    PnpmLockfileProvider,
+    PnpmModuleProvider,
+)
+from flatpak_node_generator.providers.special import SpecialSourceProvider
 
 TEST_LOCKFILE_V9 = """
 lockfileVersion: '9.0'
@@ -325,3 +333,123 @@ def test_lockfile_v9_git_and_local(tmp_path: Path) -> None:
             source=LocalSource(path='../local-pkg'),
         ),
     ]
+
+
+def test_pnpm_module_provider_tarball_url(tmp_path: Path) -> None:
+    gen = ManifestGenerator()
+    special = SpecialSourceProvider(
+        gen,
+        SpecialSourceProvider.Options(
+            node_chromedriver_from_electron=None,
+            electron_ffmpeg=None,
+            electron_node_headers=False,
+            nwjs_version=None,
+            nwjs_node_headers=False,
+            nwjs_ffmpeg=False,
+            xdg_layout=True,
+            node_sdk_extension=None,
+        ),
+    )
+    provider = PnpmModuleProvider(gen, special, tmp_path)
+
+    provider._store_version = 'v3'
+    provider._tarballs = [
+        PnpmModuleProvider._TarballInfo(
+            tarball_name='normal-pkg-1.0.0.tgz',
+            name='normal-pkg',
+            version='1.0.0',
+            integrity=Integrity('sha512', 'abc123def456'),
+        ),
+        PnpmModuleProvider._TarballInfo(
+            tarball_name='url-pkg-http-123.tgz',
+            name='url-pkg',
+            version='http://example.com/url-pkg.tgz',
+            integrity=Integrity('sha512', 'fedcba654'),
+        ),
+        PnpmModuleProvider._TarballInfo(
+            tarball_name='url-pkg-https-123.tgz',
+            name='url-pkg-2',
+            version='https://example.com/url-pkg-2.tgz',
+            integrity=Integrity('sha512', '99999999'),
+        ),
+    ]
+
+    provider._add_store_population_script()
+
+    # Manifest data source should have been added to gen._sources
+    manifest_source_dict = next(
+        dict(s)
+        for s in gen._sources
+        if dict(s).get('dest-filename') == 'pnpm-manifest.json'
+    )
+    assert manifest_source_dict is not None
+
+    manifest_data = json.loads(manifest_source_dict['contents'])
+    packages = manifest_data['packages']
+
+    # Check each package based on original tarballs for correct handling
+    for tarball in provider._tarballs:
+        pkg = packages[tarball.tarball_name]
+        assert pkg['version'] == tarball.version
+        if tarball.version.startswith(('http://', 'https://')):
+            assert pkg['tarball_url'] == tarball.version
+        else:
+            assert 'tarball_url' not in pkg
+
+
+@pytest.mark.asyncio
+async def test_pnpm_module_provider_missing_integrity(
+    tmp_path: Path, requests: RequestsController
+) -> None:
+
+    gen = ManifestGenerator()
+    special = SpecialSourceProvider(
+        gen,
+        SpecialSourceProvider.Options(
+            node_chromedriver_from_electron=None,
+            electron_ffmpeg=None,
+            electron_node_headers=False,
+            nwjs_version=None,
+            nwjs_node_headers=False,
+            nwjs_ffmpeg=False,
+            xdg_layout=True,
+            node_sdk_extension=None,
+        ),
+    )
+
+    provider = PnpmModuleProvider(gen, special, tmp_path)
+    provider._store_version = 'v3'
+
+    lockfile = Lockfile(tmp_path / 'pnpm-lock.yaml', 9)
+
+    test_data = b'dummy tarball content'
+    test_digest = hashlib.sha256(test_data).hexdigest()
+    expected_integrity = Integrity('sha256', test_digest)
+
+    requests.server.expect_oneshot_request(
+        '/test-pkg-1.0.0.tgz', 'GET'
+    ).respond_with_data(test_data)
+
+    source = ResolvedSource(
+        resolved=requests.url_for('/test-pkg-1.0.0.tgz'),
+        integrity=None,
+    )
+
+    pkg = Package(
+        lockfile=lockfile,
+        name='test-pkg',
+        version='1.0.0',
+        source=source,
+    )
+
+    await provider.generate_package(pkg)
+
+    # Assert tarball was added with computed integrity
+    assert len(provider._tarballs) == 1
+    assert provider._tarballs[0].integrity == expected_integrity
+
+    # Assert it was added to manifest generator with the right integrity
+    tarball_source = next(
+        dict(s) for s in gen._sources if dict(s).get('url') == source.resolved
+    )
+    assert tarball_source['sha256'] == expected_integrity.digest