Merge pull request #1267 from elixir-lsp/toxic2-integration

Integrate the toxic2 tolerant parser across providers

Author: lukaszsamson

apps/elixir_ls_utils/lib/completion_engine.ex

@@ -97,6 +97,25 @@ defmodule ElixirLS.Utils.CompletionEngine do
     %{type: :bitstring_option, name: "utf32"}
   ]
 
+  # Bitstring modifier validity rules (UTS of `<<x::type-sign-endianness-size-unit>>`). Used to keep
+  # `:bitstring_modifier` completion from offering combinations Elixir rejects (e.g. `signed` after
+  # `binary-`, `size`/`unit` after `utf8-`). Inlined here rather than pulled from a separate module so
+  # the completion engine stays self-contained.
+  @bitstring_types [:integer, :float, :bitstring, :binary, :utf8, :utf16, :utf32]
+  @bitstring_utf_types [:utf8, :utf16, :utf32]
+  @bitstring_type_aliases [bits: :bitstring, bytes: :binary]
+  @bitstring_sign_modifiers [:signed, :unsigned]
+  @bitstring_endianness_modifiers [:little, :big, :native]
+  # types each endianness modifier is compatible with, when the type is not yet fixed
+  @bitstring_endianness_types [:integer, :float, :utf16, :utf32]
+  @bitstring_default_state %{
+    type: nil,
+    sign_modifier: nil,
+    endianness_modifier: nil,
+    size: nil,
+    unit: nil
+  }
+
   @alias_only_atoms ~w(alias import require)a
   @alias_only_charlists ~w(alias import require)c
 
@@ -722,18 +741,37 @@ defmodule ElixirLS.Utils.CompletionEngine do
   end
 
   defp struct_module_filter(true, %State.Env{} = _env, %Metadata{} = metadata) do
+    # Build the loaded/application module-name list ONCE for the whole filter pass
+    # rather than recomputing `:code.all_loaded()` (and the application module scan)
+    # for every candidate module - that turned the filter into O(N^2) work.
+    all_module_names = all_module_names()
+
     fn module ->
       Struct.is_struct(module, metadata.structs) or
-        has_struct_submodule?(module, metadata.structs)
+        has_struct_submodule?(module, metadata.structs, all_module_names)
     end
   end
 
   defp struct_module_filter(false, %State.Env{} = _env, %Metadata{} = _metadata) do
     fn _ -> true end
   end
 
+  # The loaded (and, in interactive mode, application) module names as strings.
+  # Computed once per struct-module filter pass, not per candidate module.
+  defp all_module_names do
+    modules = Enum.map(:code.all_loaded(), &Atom.to_string(elem(&1, 0)))
+
+    case :code.get_mode() do
+      :interactive ->
+        modules ++ Enum.map(Applications.get_modules_from_applications(), &Atom.to_string/1)
+
+      _ ->
+        modules
+    end
+  end
+
   # Check if a module has any direct submodules that are structs
-  defp has_struct_submodule?(module, structs) do
+  defp has_struct_submodule?(module, structs, all_module_names) do
     module_str = Atom.to_string(module)
 
     # Check metadata structs (from current buffer)
@@ -747,30 +785,14 @@ defmodule ElixirLS.Utils.CompletionEngine do
     if metadata_result do
       true
     else
-      # Get all modules and check if any direct submodule is a struct
+      # Check if any direct submodule (from the precomputed list) is a struct
       module_str_with_dot = module_str <> "."
 
-      # Get all loaded modules
-      modules = Enum.map(:code.all_loaded(), &Atom.to_string(elem(&1, 0)))
-
-      # Add modules from applications if in interactive mode
-      modules =
-        case :code.get_mode() do
-          :interactive ->
-            modules ++
-              Enum.map(Applications.get_modules_from_applications(), &Atom.to_string/1)
-
-          _ ->
-            modules
-        end
-
-      # Find submodules
       submodules =
-        for mod <- modules,
+        for mod <- all_module_names,
             String.starts_with?(mod, module_str_with_dot),
             do: String.to_atom(mod)
 
-      # Check if any submodule is a struct
       Enum.any?(submodules, fn mod ->
         Code.ensure_loaded?(mod) and function_exported?(mod, :__struct__, 1)
       end)
@@ -793,16 +815,22 @@ defmodule ElixirLS.Utils.CompletionEngine do
         {container_context_map_fields(pairs, {:struct, alias}, map, hint, metadata), continue?}
 
       :bitstring_modifier ->
-        existing =
+        # Parse the modifiers already typed after `::` into a state and keep only the modifiers that
+        # may still legally follow (honoring the type / sign / endianness / utf / size / unit rules),
+        # instead of merely excluding names already present - which would offer invalid combinations
+        # such as `signed`/`little` after `binary-`, or `size`/`unit`/`signed` after `utf8-`.
+        available_names =
           code
           |> List.to_string()
           |> String.split("::")
           |> List.last()
-          |> String.split("-")
+          |> bitstring_parse()
+          |> bitstring_available_options()
+          |> Enum.map(&Atom.to_string/1)
 
         results =
           @bitstring_modifiers
-          |> Enum.filter(&(Matcher.match?(&1.name, hint) and &1.name not in existing))
+          |> Enum.filter(&(&1.name in available_names and Matcher.match?(&1.name, hint)))
           |> format_expansion()
 
         {results, false}
@@ -812,6 +840,92 @@ defmodule ElixirLS.Utils.CompletionEngine do
     end
   end
 
+  # Parse the modifier text already typed after `::` (e.g. `binary-siz`) into a
+  # `%{type, sign_modifier, endianness_modifier, size, unit}` state. Unknown/partial characters (the
+  # in-progress hint) are skipped one at a time, so a complete parse is not required.
+  defp bitstring_parse(text), do: bitstring_parse(text, @bitstring_default_state)
+
+  for type <- @bitstring_types do
+    defp bitstring_parse(<<unquote(Atom.to_string(type)), rest::binary>>, acc),
+      do: bitstring_parse(rest, %{acc | type: unquote(type)})
+  end
+
+  for {type_alias, type} <- @bitstring_type_aliases do
+    defp bitstring_parse(<<unquote(Atom.to_string(type_alias)), rest::binary>>, acc),
+      do: bitstring_parse(rest, %{acc | type: unquote(type)})
+  end
+
+  for sign <- @bitstring_sign_modifiers do
+    defp bitstring_parse(<<unquote(Atom.to_string(sign)), rest::binary>>, acc),
+      do: bitstring_parse(rest, %{acc | sign_modifier: unquote(sign)})
+  end
+
+  for endianness <- @bitstring_endianness_modifiers do
+    defp bitstring_parse(<<unquote(Atom.to_string(endianness)), rest::binary>>, acc),
+      do: bitstring_parse(rest, %{acc | endianness_modifier: unquote(endianness)})
+  end
+
+  defp bitstring_parse(<<"-", rest::binary>>, acc), do: bitstring_parse(rest, acc)
+
+  defp bitstring_parse(<<"size", rest::binary>>, acc),
+    do: bitstring_parse(rest, %{acc | size: true})
+
+  defp bitstring_parse(<<"unit", rest::binary>>, acc),
+    do: bitstring_parse(rest, %{acc | unit: true})
+
+  defp bitstring_parse(<<_::binary-size(1), rest::binary>>, acc), do: bitstring_parse(rest, acc)
+  defp bitstring_parse(<<>>, acc), do: acc
+
+  # The modifiers that may still legally follow, given the parsed state.
+  defp bitstring_available_options(state) do
+    bitstring_available_types(state) ++
+      bitstring_available_sign_modifiers(state) ++
+      bitstring_available_endianness_modifiers(state) ++
+      bitstring_available_size(state) ++
+      bitstring_available_unit(state)
+  end
+
+  defp bitstring_available_types(
+         %{type: nil, sign_modifier: nil, endianness_modifier: nil} = state
+       ),
+       do: bitstring_filter_utf(state, @bitstring_types)
+
+  defp bitstring_available_types(%{type: nil, sign_modifier: nil, endianness_modifier: e} = state)
+       when not is_nil(e),
+       do: bitstring_filter_utf(state, @bitstring_endianness_types)
+
+  defp bitstring_available_types(%{type: nil}), do: [:integer]
+  defp bitstring_available_types(_), do: []
+
+  defp bitstring_available_sign_modifiers(%{type: type, sign_modifier: nil})
+       when type in [nil, :integer],
+       do: @bitstring_sign_modifiers
+
+  defp bitstring_available_sign_modifiers(_), do: []
+
+  defp bitstring_available_endianness_modifiers(%{type: type, endianness_modifier: nil})
+       when type in [nil, :integer, :utf16, :utf32],
+       do: @bitstring_endianness_modifiers
+
+  defp bitstring_available_endianness_modifiers(%{type: :float, endianness_modifier: nil}),
+    do: [:little, :big]
+
+  defp bitstring_available_endianness_modifiers(_), do: []
+
+  # size and unit are unsupported on utf types (they fail to compile).
+  defp bitstring_available_size(%{size: nil, type: type}) when type not in @bitstring_utf_types,
+    do: [:size]
+
+  defp bitstring_available_size(_), do: []
+
+  defp bitstring_available_unit(%{unit: nil, type: type}) when type not in @bitstring_utf_types,
+    do: [:unit]
+
+  defp bitstring_available_unit(_), do: []
+
+  defp bitstring_filter_utf(%{size: nil, unit: nil}, list), do: list
+  defp bitstring_filter_utf(_, list), do: list -- @bitstring_utf_types
+
   defp container_context_map_fields(pairs, kind, map, hint, metadata) do
     {keys, types, alias, doc, meta} =
       case kind do

apps/elixir_ls_utils/test/complete_test.exs

@@ -2033,6 +2033,32 @@ defmodule ElixirLS.Utils.CompletionEngineTest do
     refute Enum.any?(entries, &(&1.name == "integer"))
     refute Enum.any?(entries, &(&1.name == "little"))
     assert Enum.any?(entries, &(&1.name == "size" and &1.arity == 1))
+
+    # state-based validity: a `binary` type only allows size/unit to follow
+    names = fn code ->
+      expand(code) |> Enum.filter(&(&1[:type] == :bitstring_option)) |> Enum.map(& &1.name)
+    end
+
+    binary = names.('<<foo::binary-')
+    assert "size" in binary
+    assert "unit" in binary
+    refute "signed" in binary
+    refute "little" in binary
+    refute "native" in binary
+    refute "utf8" in binary
+
+    # utf types take no further modifiers
+    assert names.('<<foo::utf8-') == []
+
+    # float allows little/big (not native), size and unit - but not sign or another type
+    float = names.('<<foo::float-')
+    assert "little" in float
+    assert "big" in float
+    assert "size" in float
+    assert "unit" in float
+    refute "native" in float
+    refute "signed" in float
+    refute "binary" in float
   end
 
   test "completion for aliases in special forms" do