Release 0.4.3

Author: dannote

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## 0.4.3
+
+- Fix `mix npm.exec` to resolve binaries via `NPM.Exec.which/2` and run them through Node instead of shell string spawning
+- Preserve `optional_dependencies` in `npm.lock`
+- Skip linking missing optional packages instead of crashing during install
+- Add focused test coverage for exec environment, cached Node runner execution, optional runtime linking, and resolver optional dependency handling
+
 ## 0.4.2
 
 - Speculative parallel prefetch of transitive dependencies before solving — fetches the full dep tree breadth-first with 16 concurrent requests

README.md

@@ -107,6 +107,10 @@ mix npm.config
 9. Warns about unmet peer dependencies and deprecated packages
 10. Retries failed downloads with exponential backoff
 
+## Why `npm.lock` instead of `package-lock.json`?
+
+`npm_ex` is not npm, so it keeps its own lockfile. `package.json` is the shared manifest; `npm.lock` is the reproducibility file for the `npm_ex` installer.
+
 ## Configuration
 
 Set environment variables to customize behavior:

lib/mix/tasks/npm.exec.ex

@@ -16,13 +16,13 @@ defmodule Mix.Tasks.Npm.Exec do
   def run([command | args]) do
     Application.ensure_all_started(:req)
 
-    bin_path = Path.join("node_modules/.bin", command)
+    case NPM.Exec.which(command, "node_modules") do
+      {:ok, bin_path} ->
+        execute(bin_path, args)
 
-    if File.exists?(bin_path) do
-      execute(bin_path, args)
-    else
-      Mix.shell().error("Binary #{command} not found in node_modules/.bin/")
-      Mix.shell().info("Run `mix npm.install` to install packages.")
+      {:error, :not_found} ->
+        Mix.shell().error("Binary #{command} not found in node_modules/.bin/")
+        Mix.shell().info("Run `mix npm.install` to install packages.")
     end
   end
 
@@ -44,30 +44,20 @@ defmodule Mix.Tasks.Npm.Exec do
   end
 
   defp execute(bin_path, args) do
-    full_command = Enum.join([bin_path | args], " ")
-
-    port =
-      Port.open({:spawn, full_command}, [
-        :binary,
-        :exit_status,
-        :stderr_to_stdout
-      ])
-
-    stream_port(port)
-  end
-
-  defp stream_port(port) do
-    receive do
-      {^port, {:data, data}} ->
-        IO.write(data)
-        stream_port(port)
+    {output, status} =
+      NPM.NodeRunner.run(Path.expand(bin_path), args,
+        node_modules_dir: "node_modules",
+        cd: File.cwd!()
+      )
 
-      {^port, {:exit_status, 0}} ->
-        :ok
+    if output != "", do: IO.write(output)
 
-      {^port, {:exit_status, code}} ->
+    case status do
+      0 -> :ok
+      code ->
         Mix.shell().error("Exited with code #{code}")
         {:error, {:exit, code}}
     end
   end
+
 end

lib/npm.ex

@@ -277,7 +277,8 @@ defmodule NPM do
            version: version_str,
            integrity: info.dist.integrity,
            tarball: info.dist.tarball,
-           dependencies: info.dependencies
+           dependencies: info.dependencies,
+           optional_dependencies: info.optional_dependencies
          }}
       end
 

lib/npm/cache.ex

@@ -33,17 +33,23 @@ defmodule NPM.Cache do
   Downloads and extracts the tarball if not already cached.
   Returns `{:ok, cache_path}` or `{:error, reason}`.
   """
-  @spec ensure(String.t(), String.t(), String.t(), String.t()) ::
+  @spec ensure(String.t(), String.t(), String.t(), String.t(), keyword()) ::
           {:ok, String.t()} | {:error, term()}
-  def ensure(name, version, tarball_url, integrity) do
+  def ensure(name, version, tarball_url, integrity, opts \\ []) do
     dest = package_dir(name, version)
 
     if cached?(name, version) do
       {:ok, dest}
     else
       case NPM.Tarball.fetch_and_extract(tarball_url, integrity, dest) do
         {:ok, _count} -> {:ok, dest}
-        error -> error
+        {:error, reason} ->
+          if Keyword.get(opts, :optional?, false) do
+            File.rm_rf(dest)
+            {:ok, :missing_optional}
+          else
+            {:error, reason}
+          end
       end
     end
   end

lib/npm/exec.ex

@@ -72,8 +72,16 @@ defmodule NPM.Exec do
   @spec env(String.t()) :: [{String.t(), String.t()}]
   def env(node_modules_dir \\ "node_modules") do
     bin_dir = Path.expand(Path.join(node_modules_dir, ".bin"))
+    node_modules_dir = Path.expand(node_modules_dir)
     current_path = System.get_env("PATH") || ""
-    [{"PATH", "#{bin_dir}:#{current_path}"}]
+    node_path = [node_modules_dir, System.get_env("NODE_PATH")]
+    |> Enum.reject(&is_nil/1)
+    |> Enum.join(":")
+
+    [
+      {"PATH", "#{bin_dir}:#{current_path}"},
+      {"NODE_PATH", node_path}
+    ]
   end
 
   defp find_in_packages(command, node_modules_dir) do

lib/npm/linker.ex

@@ -21,8 +21,8 @@ defmodule NPM.Linker do
   """
   @spec link(resolved(), String.t(), strategy()) :: :ok | {:error, term()}
   def link(lockfile, node_modules_dir \\ "node_modules", strategy \\ default_strategy()) do
-    with :ok <- populate_cache(lockfile) do
-      create_node_modules(lockfile, node_modules_dir, strategy)
+    with {:ok, skipped} <- populate_cache(lockfile) do
+      create_node_modules(lockfile, node_modules_dir, strategy, skipped)
     end
   end
 
@@ -49,21 +49,32 @@ defmodule NPM.Linker do
     lockfile
     |> Task.async_stream(
       fn {name, entry} ->
-        NPM.Cache.ensure(name, entry.version, entry.tarball, entry.integrity)
+        optional? = optional_dependency?(name, lockfile)
+
+        case NPM.Cache.ensure(name, entry.version, entry.tarball, entry.integrity, optional?: optional?) do
+          {:ok, :missing_optional} -> {:skip, name}
+          other -> other
+        end
       end,
       max_concurrency: 8,
       timeout: 60_000
     )
-    |> Enum.reduce(:ok, fn
-      {:ok, {:ok, _}}, acc -> acc
-      {:ok, {:error, reason}}, _ -> {:error, reason}
-      {:exit, reason}, _ -> {:error, reason}
+    |> Enum.reduce({:ok, MapSet.new()}, fn
+      {:ok, {:ok, _}}, {status, skipped} -> {status, skipped}
+      {:ok, {:skip, name}}, {status, skipped} -> {status, MapSet.put(skipped, name)}
+      {:ok, {:error, reason}}, {_status, _skipped} -> {{:error, reason}, MapSet.new()}
+      {:exit, reason}, {_status, _skipped} -> {{:error, reason}, MapSet.new()}
     end)
   end
 
-  defp create_node_modules(lockfile, node_modules_dir, strategy) do
+  defp create_node_modules(lockfile, node_modules_dir, strategy, skipped) do
     File.mkdir_p!(node_modules_dir)
-    tree = hoist(lockfile)
+
+    tree =
+      lockfile
+      |> hoist()
+      |> Enum.reject(fn {name, _version} -> MapSet.member?(skipped, name) end)
+
     expected_names = MapSet.new(tree, &elem(&1, 0))
 
     prune(node_modules_dir, expected_names)
@@ -259,17 +270,24 @@ defmodule NPM.Linker do
     end
   end
 
+  defp optional_dependency?(name, lockfile) do
+    Enum.any?(lockfile, fn {_pkg, entry} ->
+      Map.has_key?(Map.get(entry, :optional_dependencies, %{}), name)
+    end)
+  end
+
   defp install_single_nested(_pkg, nil, _parent, _nm_dir, _strategy), do: :ok
 
   defp install_single_nested(pkg, version, parent, nm_dir, strategy) do
     with {:ok, packument} <- NPM.Registry.get_packument(pkg),
-         %{} = info <- Map.get(packument.versions, version) do
-      tarball = info.tarball
-      integrity = info.integrity
-      NPM.Cache.ensure(pkg, version, tarball, integrity)
-      cache_path = NPM.Cache.package_dir(pkg, version)
-      target = Path.join([nm_dir, parent, "node_modules", pkg])
-      link_package(cache_path, target, strategy)
+         %{} = info <- Map.get(packument.versions, version),
+         {:ok, cache_result} <-
+           NPM.Cache.ensure(pkg, version, info.dist.tarball, info.dist.integrity) do
+      if cache_result != :missing_optional do
+        cache_path = NPM.Cache.package_dir(pkg, version)
+        target = Path.join([nm_dir, parent, "node_modules", pkg])
+        link_package(cache_path, target, strategy)
+      end
     end
 
     :ok

lib/npm/lockfile.ex

@@ -12,7 +12,8 @@ defmodule NPM.Lockfile do
           version: String.t(),
           integrity: String.t(),
           tarball: String.t(),
-          dependencies: %{String.t() => String.t()}
+          dependencies: %{String.t() => String.t()},
+          optional_dependencies: %{String.t() => String.t()}
         }
 
   @type t :: %{String.t() => entry()}
@@ -52,7 +53,8 @@ defmodule NPM.Lockfile do
          version: Map.get(info, "version", ""),
          integrity: Map.get(info, "integrity", ""),
          tarball: Map.get(info, "tarball", ""),
-         dependencies: Map.get(info, "dependencies", %{})
+         dependencies: Map.get(info, "dependencies", %{}),
+         optional_dependencies: Map.get(info, "optional_dependencies", %{})
        }}
     end
   end
@@ -106,7 +108,8 @@ defmodule NPM.Lockfile do
          "version" => entry.version,
          "integrity" => entry.integrity,
          "tarball" => entry.tarball,
-         "dependencies" => entry.dependencies
+         "dependencies" => entry.dependencies,
+         "optional_dependencies" => Map.get(entry, :optional_dependencies, %{})
        }}
     end
   end

lib/npm/node_runner.ex

@@ -0,0 +1,36 @@
+defmodule NPM.NodeRunner do
+  @moduledoc false
+
+  @spec run(String.t(), [String.t()], keyword()) :: {String.t(), non_neg_integer()}
+  def run(entrypoint, args, opts \\ []) do
+    node_modules_dir = Keyword.get(opts, :node_modules_dir, "node_modules")
+    env = Keyword.get(opts, :env, []) ++ NPM.Exec.env(node_modules_dir)
+
+    loader_path = write_loader(Path.expand(node_modules_dir), Path.expand(entrypoint))
+
+    try do
+      System.cmd("node", [loader_path | args],
+        env: env,
+        stderr_to_stdout: true,
+        cd: Keyword.get(opts, :cd, File.cwd!())
+      )
+    after
+      File.rm(loader_path)
+    end
+  end
+
+  defp write_loader(node_modules_dir, entrypoint) do
+    path = Path.join(System.tmp_dir!(), "npm-node-runner-#{System.unique_integer([:positive])}.mjs")
+
+    File.write!(path, """
+    import { createRequire } from 'node:module'
+    const require = createRequire(import.meta.url)
+    globalThis.require = require
+    process.env.NODE_PATH = #{inspect(node_modules_dir)}
+    require('node:module').Module._initPaths()
+    await import(#{inspect(entrypoint)})
+    """)
+
+    path
+  end
+end

lib/npm/registry.ex

@@ -16,7 +16,21 @@ defmodule NPM.Registry do
 
   @type version_info :: %{
           dependencies: %{String.t() => String.t()},
-          dist: %{tarball: String.t(), integrity: String.t()}
+          optional_dependencies: %{String.t() => String.t()},
+          peer_dependencies: %{String.t() => String.t()},
+          peer_dependencies_meta: %{String.t() => map()},
+          bin: %{optional(String.t()) => String.t()},
+          engines: %{String.t() => String.t()},
+          os: [String.t()],
+          cpu: [String.t()],
+          has_install_script: boolean(),
+          deprecated: String.t() | nil,
+          dist: %{
+            tarball: String.t(),
+            integrity: String.t(),
+            file_count: integer() | nil,
+            unpacked_size: integer() | nil
+          }
         }
 
   @doc "Get the configured registry URL."