Skip to content

Commit 6cf041c

Browse files
committed
Speculative parallel prefetch of transitive dependencies
Before running the solver, prefetch the entire dependency tree breadth-first with 16 concurrent HTTP requests. Uses only the latest stable version's deps to avoid fetching every version.
1 parent dadfc48 commit 6cf041c

1 file changed

Lines changed: 52 additions & 0 deletions

File tree

lib/npm/resolver.ex

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ defmodule NPM.Resolver do
4949
end
5050

5151
defp resolve_with_nesting(root_deps, excluded, nested, depth) do
52+
prefetch_tree(Map.keys(root_deps))
5253
dependencies = build_dependencies(root_deps)
5354

5455
case run_solver(dependencies, excluded) do
@@ -329,4 +330,55 @@ defmodule NPM.Resolver do
329330
error
330331
end
331332
end
333+
334+
defp prefetch_tree(packages, depth \\ 0)
335+
defp prefetch_tree(_packages, depth) when depth > 10, do: :ok
336+
337+
defp prefetch_tree(packages, depth) do
338+
to_fetch = Enum.reject(packages, &cached?/1)
339+
340+
if to_fetch != [] do
341+
to_fetch
342+
|> Task.async_stream(&fetch_and_cache/1, max_concurrency: 16, timeout: 30_000)
343+
|> Stream.run()
344+
345+
next_level =
346+
to_fetch
347+
|> Enum.flat_map(&dep_names_from_cache/1)
348+
|> Enum.uniq()
349+
|> Enum.reject(&cached?/1)
350+
351+
if next_level != [], do: prefetch_tree(next_level, depth + 1)
352+
end
353+
end
354+
355+
defp dep_names_from_cache(package) do
356+
case :ets.lookup(@table, package) do
357+
[{_, packument}] ->
358+
case latest_version_info(packument) do
359+
nil -> []
360+
info -> Map.keys(info.dependencies)
361+
end
362+
363+
[] ->
364+
[]
365+
end
366+
end
367+
368+
defp latest_version_info(packument) do
369+
packument.versions
370+
|> Map.keys()
371+
|> Enum.flat_map(fn v ->
372+
case Version.parse(v) do
373+
{:ok, ver} -> [{v, ver}]
374+
:error -> []
375+
end
376+
end)
377+
|> Enum.reject(fn {_, ver} -> ver.pre != [] end)
378+
|> Enum.sort_by(&elem(&1, 1), {:desc, Version})
379+
|> case do
380+
[{latest_str, _} | _] -> Map.get(packument.versions, latest_str)
381+
[] -> nil
382+
end
383+
end
332384
end

0 commit comments

Comments
 (0)