Represent bitstring and binary as distinct types (#15038)

Author: josevalim

lib/elixir/lib/module/types/apply.ex

@@ -168,13 +168,13 @@ defmodule Module.Types.Apply do
         {:erlang, :binary_to_integer, [{[binary()], integer()}]},
         {:erlang, :binary_to_integer, [{[binary(), integer()], integer()}]},
         {:erlang, :binary_to_float, [{[binary()], float()}]},
-        {:erlang, :bit_size, [{[binary()], integer()}]},
+        {:erlang, :bit_size, [{[bitstring()], integer()}]},
         {:erlang, :bnot, [{[integer()], integer()}]},
         {:erlang, :bor, [{[integer(), integer()], integer()}]},
         {:erlang, :bsl, [{[integer(), integer()], integer()}]},
         {:erlang, :bsr, [{[integer(), integer()], integer()}]},
         {:erlang, :bxor, [{[integer(), integer()], integer()}]},
-        {:erlang, :byte_size, [{[binary()], integer()}]},
+        {:erlang, :byte_size, [{[bitstring()], integer()}]},
         {:erlang, :ceil, [{[union(integer(), float())], integer()}]},
         {:erlang, :div, [{[integer(), integer()], integer()}]},
         {:erlang, :error, [{[term()], none()}]},
@@ -288,7 +288,7 @@ defmodule Module.Types.Apply do
   is_guards = [
     is_atom: atom(),
     is_binary: binary(),
-    is_bitstring: binary(),
+    is_bitstring: bitstring(),
     is_boolean: boolean(),
     is_float: float(),
     is_function: fun(),
@@ -303,13 +303,10 @@ defmodule Module.Types.Apply do
   ]
 
   for {guard, type} <- is_guards do
-    # is_binary can actually fail for binaries if they are bitstrings
-    return = if guard == :is_binary, do: boolean(), else: atom([true])
-
     domain_clauses =
       {:strong, [term()],
        [
-         {[type], return},
+         {[type], atom([true])},
          {[negation(type)], atom([false])}
        ]}
 
@@ -1282,6 +1279,17 @@ defmodule Module.Types.Apply do
     end
   end
 
+  @doc """
+  Computes the return type of an application.
+  """
+  def return(type, args_types, stack) do
+    cond do
+      stack.mode == :static -> type
+      Enum.any?(args_types, &gradual?/1) -> dynamic(type)
+      true -> type
+    end
+  end
+
   ## Map helpers
 
   defp map_put_new(map, key, value, name, args_types, stack) do
@@ -1325,14 +1333,6 @@ defmodule Module.Types.Apply do
 
   ## Application helpers
 
-  defp return(type, args_types, stack) do
-    cond do
-      stack.mode == :static -> type
-      Enum.any?(args_types, &gradual?/1) -> dynamic(type)
-      true -> type
-    end
-  end
-
   defp domain(nil, [{domain, _}]), do: domain
   defp domain(domain, _clauses), do: domain
 

lib/elixir/lib/module/types/descr.ex

@@ -16,14 +16,23 @@ defmodule Module.Types.Descr do
 
   import Bitwise
 
-  @bit_binary 1 <<< 0
-  @bit_empty_list 1 <<< 1
-  @bit_integer 1 <<< 2
-  @bit_float 1 <<< 3
-  @bit_pid 1 <<< 4
-  @bit_port 1 <<< 5
-  @bit_reference 1 <<< 6
-  @bit_top (1 <<< 7) - 1
+  @bit_binary 0b1
+  @bit_bitstring_no_binary 0b10
+  @bit_empty_list 0b100
+  @bit_integer 0b1000
+  @bit_float 0b10000
+  @bit_pid 0b100000
+  @bit_port 0b1000000
+  @bit_reference 0b10000000
+  @bit_top 0b11111111
+
+  # We use two bits to represent bitstrings and binaries,
+  # which must be looked at together
+  # bitstring = 0b11
+  # bitstring and not binary = 0b10
+  # binary = 0b01
+  # none = 0b00
+  @bit_bitstring @bit_binary ||| @bit_bitstring_no_binary
   @bit_number @bit_integer ||| @bit_float
 
   defmacro bdd_leaf(arg1, arg2), do: {arg1, arg2}
@@ -80,6 +89,8 @@ defmodule Module.Types.Descr do
   def atom(as), do: %{atom: atom_new(as)}
   def atom(), do: %{atom: @atom_top}
   def binary(), do: %{bitmap: @bit_binary}
+  def bitstring(), do: %{bitmap: @bit_bitstring}
+  def bitstring_no_binary(), do: %{bitmap: @bit_bitstring_no_binary}
   def closed_map(pairs), do: map_descr(:closed, pairs)
   def empty_list(), do: %{bitmap: @bit_empty_list}
   def empty_map(), do: %{map: @map_empty}
@@ -843,29 +854,14 @@ defmodule Module.Types.Descr do
   @doc """
   Optimized version of `not empty?(intersection(binary(), type))`.
   """
-  def binary_type?(:term), do: true
-  def binary_type?(%{dynamic: :term}), do: true
-  def binary_type?(%{dynamic: %{bitmap: bitmap}}) when (bitmap &&& @bit_binary) != 0, do: true
-  def binary_type?(%{bitmap: bitmap}) when (bitmap &&& @bit_binary) != 0, do: true
-  def binary_type?(_), do: false
+  def bitstring_type?(:term), do: true
+  def bitstring_type?(%{dynamic: :term}), do: true
 
-  @doc """
-  Optimized version of `not empty?(intersection(integer(), type))`.
-  """
-  def integer_type?(:term), do: true
-  def integer_type?(%{dynamic: :term}), do: true
-  def integer_type?(%{dynamic: %{bitmap: bitmap}}) when (bitmap &&& @bit_integer) != 0, do: true
-  def integer_type?(%{bitmap: bitmap}) when (bitmap &&& @bit_integer) != 0, do: true
-  def integer_type?(_), do: false
+  def bitstring_type?(%{dynamic: %{bitmap: bitmap}}) when (bitmap &&& @bit_bitstring) != 0,
+    do: true
 
-  @doc """
-  Optimized version of `not empty?(intersection(float(), type))`.
-  """
-  def float_type?(:term), do: true
-  def float_type?(%{dynamic: :term}), do: true
-  def float_type?(%{dynamic: %{bitmap: bitmap}}) when (bitmap &&& @bit_float) != 0, do: true
-  def float_type?(%{bitmap: bitmap}) when (bitmap &&& @bit_float) != 0, do: true
-  def float_type?(_), do: false
+  def bitstring_type?(%{bitmap: bitmap}) when (bitmap &&& @bit_bitstring) != 0, do: true
+  def bitstring_type?(_), do: false
 
   @doc """
   Optimized version of `not empty?(intersection(integer() or float(), type))`.
@@ -881,7 +877,6 @@ defmodule Module.Types.Descr do
   defp bitmap_to_quoted(val) do
     pairs =
       [
-        binary: @bit_binary,
         empty_list: @bit_empty_list,
         integer: @bit_integer,
         float: @bit_float,
@@ -890,9 +885,17 @@ defmodule Module.Types.Descr do
         reference: @bit_reference
       ]
 
-    for {type, mask} <- pairs,
-        (mask &&& val) !== 0,
-        do: {type, [], []}
+    quoted =
+      for {type, mask} <- pairs,
+          (mask &&& val) !== 0,
+          do: {type, [], []}
+
+    case val &&& @bit_bitstring do
+      0 -> quoted
+      1 -> [{:binary, [], []} | quoted]
+      2 -> [{:and, [], [{:bitstring, [], []}, {:not, [], [{:binary, [], []}]}]} | quoted]
+      3 -> [{:bitstring, [], []} | quoted]
+    end
   end
 
   ## Atoms
@@ -2368,7 +2371,7 @@ defmodule Module.Types.Descr do
 
   defp indivisible_bitmap(descr, opts) do
     with true <- Keyword.get(opts, :skip_dynamic_for_indivisible, true),
-         %{bitmap: bitmap} when map_size(descr) == 1 <- descr,
+         %{bitmap: bitmap} when map_size(descr) == 1 and bitmap != @bit_bitstring <- descr,
          [single] <- bitmap_to_quoted(bitmap) do
       single
     else
@@ -2423,6 +2426,7 @@ defmodule Module.Types.Descr do
 
   defp bitmap_to_domain_keys(bitmap, acc) do
     acc = if (bitmap &&& @bit_binary) != 0, do: [:binary | acc], else: acc
+    acc = if (bitmap &&& @bit_bitstring_no_binary) != 0, do: [:bitstring | acc], else: acc
     acc = if (bitmap &&& @bit_empty_list) != 0, do: [:list | acc], else: acc
     acc = if (bitmap &&& @bit_integer) != 0, do: [:integer | acc], else: acc
     acc = if (bitmap &&& @bit_float) != 0, do: [:float | acc], else: acc
@@ -2433,6 +2437,7 @@ defmodule Module.Types.Descr do
   end
 
   defp domain_key_to_descr(:atom), do: atom()
+  defp domain_key_to_descr(:bitstring), do: bitstring_no_binary()
   defp domain_key_to_descr(:binary), do: binary()
   defp domain_key_to_descr(:integer), do: integer()
   defp domain_key_to_descr(:float), do: float()

lib/elixir/lib/module/types/expr.ex

@@ -122,9 +122,8 @@ defmodule Module.Types.Expr do
   end
 
   # <<...>>>
-  def of_expr({:<<>>, _meta, args}, _expected, _expr, stack, context) do
-    context = Of.binary(args, :expr, stack, context)
-    {binary(), context}
+  def of_expr({:<<>>, meta, args}, _expected, _expr, stack, context) do
+    Of.bitstring(meta, args, :expr, stack, context)
   end
 
   def of_expr({:__CALLER__, _meta, var_context}, _expected, _expr, _stack, context)
@@ -315,7 +314,7 @@ defmodule Module.Types.Expr do
     |> dynamic_unless_static(stack)
   end
 
-  # TODO: fn pat -> expr end
+  # fn pat -> expr end
   def of_expr({:fn, _meta, clauses}, _expected, _expr, stack, context) do
     [{:->, _, [head, _]} | _] = clauses
     {patterns, _guards} = extract_head(head)
@@ -416,20 +415,26 @@ defmodule Module.Types.Expr do
     else
       # TODO: Use the collectable protocol for the output
       into = Keyword.get(opts, :into, [])
-      {into_wrapper, gradual?, context} = for_into(into, meta, stack, context)
+      {into_type, into_kind, context} = for_into(into, meta, stack, context)
       {block_type, context} = of_expr(block, @pending, block, stack, context)
 
-      for_type =
-        for type <- into_wrapper do
-          case type do
-            :binary -> binary()
-            :list -> list(block_type)
-            :term -> term()
+      case into_kind do
+        :bitstring ->
+          case compatible_intersection(block_type, bitstring()) do
+            {:ok, intersection} ->
+              {return_union(into_type, intersection, stack), context}
+
+            {:error, _} ->
+              error = {:badbitbody, block_type, block, context}
+              {error_type(), error(__MODULE__, error, meta, stack, context)}
           end
-        end
-        |> Enum.reduce(&union/2)
 
-      {if(gradual?, do: dynamic(for_type), else: for_type), context}
+        :non_empty_list ->
+          {return_union(into_type, non_empty_list(block_type), stack), context}
+
+        :none ->
+          {into_type, context}
+      end
     end
   end
 
@@ -469,7 +474,7 @@ defmodule Module.Types.Expr do
     apply_many(mods, name, args, expected, call, stack, context)
   end
 
-  # TODO: &Foo.bar/1
+  # &Foo.bar/1
   def of_expr(
         {:&, _, [{:/, _, [{{:., _, [remote, name]}, meta, []}, arity]}]} = call,
         _expected,
@@ -483,7 +488,7 @@ defmodule Module.Types.Expr do
     Apply.remote_capture(mods, name, arity, meta, stack, context)
   end
 
-  # TODO: &foo/1
+  # &foo/1
   def of_expr({:&, _meta, [{:/, _, [{fun, meta, _}, arity]}]}, _expected, _expr, stack, context) do
     Apply.local_capture(fun, arity, meta, stack, context)
   end
@@ -577,13 +582,13 @@ defmodule Module.Types.Expr do
   end
 
   defp for_clause({:<<>>, _, [{:<-, meta, [left, right]}]} = expr, stack, context) do
-    {right_type, context} = of_expr(right, binary(), expr, stack, context)
-    context = Pattern.of_generator(left, [], binary(), :for, expr, stack, context)
+    {right_type, context} = of_expr(right, bitstring(), expr, stack, context)
+    context = Pattern.of_generator(left, [], bitstring(), :for, expr, stack, context)
 
-    if compatible?(right_type, binary()) do
+    if compatible?(right_type, bitstring()) do
       context
     else
-      error = {:badbinary, right_type, right, context}
+      error = {:badbitgenerator, right_type, right, context}
       error(__MODULE__, error, meta, stack, context)
     end
   end
@@ -593,13 +598,13 @@ defmodule Module.Types.Expr do
     context
   end
 
-  @into_compile union(binary(), empty_list())
+  @into_compile union(bitstring(), empty_list())
 
   defp for_into([], _meta, _stack, context),
-    do: {[:list], false, context}
+    do: {empty_list(), :non_empty_list, context}
 
   defp for_into(binary, _meta, _stack, context) when is_binary(binary),
-    do: {[:binary], false, context}
+    do: {binary(), :bitstring, context}
 
   defp for_into(into, meta, stack, context) do
     meta =
@@ -616,21 +621,33 @@ defmodule Module.Types.Expr do
     {type, context} = of_expr(into, domain, expr, stack, context)
 
     # We use subtype? instead of compatible because we want to handle
-    # only binary/list, even if a dynamic with something else is given.
+    # only bitstring/list, even if a dynamic with something else is given.
     if subtype?(type, @into_compile) do
-      case {binary_type?(type), empty_list_type?(type)} do
-        {false, true} -> {[:list], gradual?(type), context}
-        {true, false} -> {[:binary], gradual?(type), context}
-        {_, _} -> {[:binary, :list], gradual?(type), context}
+      case {bitstring_type?(type), empty_list_type?(type)} do
+        # If they can be both be true, then we don't know
+        # what the contents of the block are for
+        {true, true} ->
+          type = union(bitstring(), list(term()))
+          {if(gradual?(type), do: dynamic(type), else: type), :none, context}
+
+        {false, true} ->
+          {type, :non_empty_list, context}
+
+        {true, false} ->
+          {type, :bitstring, context}
       end
     else
       {_type, context} =
         Apply.remote_apply(info, Collectable, :into, [type], expr, stack, context)
 
-      {[:term], true, context}
+      {dynamic(), :none, context}
     end
   end
 
+  defp return_union(left, right, stack) do
+    Apply.return(union(left, right), [left, right], stack)
+  end
+
   ## With
 
   defp with_clause({:<-, _meta, [left, right]} = expr, stack, context) do
@@ -866,15 +883,36 @@ defmodule Module.Types.Expr do
     }
   end
 
-  def format_diagnostic({:badbinary, type, expr, context}) do
+  def format_diagnostic({:badbitgenerator, type, expr, context}) do
+    traces = collect_traces(expr, context)
+
+    %{
+      details: %{typing_traces: traces},
+      message:
+        IO.iodata_to_binary([
+          """
+          expected the right side of <- in a binary generator to be a binary (or bitstring):
+
+              #{expr_to_string(expr) |> indent(4)}
+
+          but got type:
+
+              #{to_quoted_string(type) |> indent(4)}
+          """,
+          format_traces(traces)
+        ])
+    }
+  end
+
+  def format_diagnostic({:badbitbody, type, expr, context}) do
     traces = collect_traces(expr, context)
 
     %{
       details: %{typing_traces: traces},
       message:
         IO.iodata_to_binary([
           """
-          expected the right side of <- in a binary generator to be a binary:
+          expected the body of a for-comprehension with into: binary() (or bitstring()) to be a binary (or bitstring):
 
               #{expr_to_string(expr) |> indent(4)}