Validate normalized tokenizer identifiers

Ensure identifiers that require normalization are checked again
after normalization so decomposed source cannot normalize into
unsupported codepoints.

Closes #15419.

Author: josevalim

lib/elixir/test/elixir/kernel/parser_test.exs

@@ -1359,6 +1359,14 @@ defmodule Kernel.ParserTest do
         ["nofile:1:4:", ~s/unexpected token: "#{"\u01DC"}" (column 4, code point U+01DC)/],
         ~c":fooǜ"
       )
+
+      assert_syntax_error(
+        [
+          "nofile:1:7:",
+          ~s/unexpected token: "ǜ" (column 7, code point U+01DC)/
+        ],
+        ~c":cōng_lǜ_green"
+      )
     end
 
     test "keyword missing space" do

lib/elixir/test/elixir/kernel/string_tokenizer_test.exs

@@ -43,6 +43,18 @@ defmodule Kernel.StringTokenizerTest do
     assert {:error, _} = Code.string_to_quoted(":@123")
   end
 
+  test "rejects identifiers that normalize to unsupported codepoints" do
+    decomposed_u_with_diaeresis_and_grave = [?u, 0x0308, 0x0300]
+
+    assert String.Tokenizer.tokenize([0x01DC]) == {:error, :empty}
+
+    assert String.Tokenizer.tokenize(decomposed_u_with_diaeresis_and_grave) ==
+             {:error, {:unexpected_token, decomposed_u_with_diaeresis_and_grave}}
+
+    assert String.Tokenizer.tokenize(~c"fooǜlul") ==
+             {:error, {:unexpected_token, ~c"fooǜ"}}
+  end
+
   test "tokenizes keywords" do
     assert Code.string_to_quoted!("[_12: 0]") == [_12: 0]
     assert Code.string_to_quoted!("[ola: 0]") == [ola: 0]

lib/elixir/unicode/tokenizer.ex

@@ -473,50 +473,86 @@ defmodule String.Tokenizer do
     original_acc = :lists.reverse(original_acc)
     acc = :unicode.characters_to_nfc_list(original_acc)
 
-    special =
+    {different?, special} =
       if original_acc == acc do
-        special
+        {false, special}
       else
-        [:nfkc | List.delete(special, :nfkc)]
+        {true, [:nfkc | List.delete(special, :nfkc)]}
       end
 
-    if scriptset != @bottom or chunks_single?(acc) do
-      {kind, acc, rest, length, false, special}
-    else
-      breakdown =
-        for codepoint <- acc do
-          scriptsets =
-            case codepoint_to_scriptset(codepoint) do
-              @top ->
-                ""
+    cond do
+      invalid_prefix = different? && invalid_normalized(original_acc, acc, kind) ->
+        {:error, {:unexpected_token, invalid_prefix}}
 
-              scriptset ->
-                scriptset
-                |> ScriptSet.to_indexes()
-                |> Enum.map(&Map.fetch!(@indexed_scriptsets, &1))
-                |> then(&(" {" <> Enum.join(&1, ",") <> "}"))
-            end
+      scriptset != @bottom or chunks_single?(acc) ->
+        {kind, acc, rest, length, false, special}
 
-          hex = :io_lib.format(~c"~4.16.0B", [codepoint])
-          "  \\u#{hex} #{[codepoint]}#{scriptsets}\n"
-        end
+      true ->
+        breakdown =
+          for codepoint <- acc do
+            scriptsets =
+              case codepoint_to_scriptset(codepoint) do
+                @top ->
+                  ""
+
+                scriptset ->
+                  scriptset
+                  |> ScriptSet.to_indexes()
+                  |> Enum.map(&Map.fetch!(@indexed_scriptsets, &1))
+                  |> then(&(" {" <> Enum.join(&1, ",") <> "}"))
+              end
+
+            hex = :io_lib.format(~c"~4.16.0B", [codepoint])
+            "  \\u#{hex} #{[codepoint]}#{scriptsets}\n"
+          end
 
-      prefix = ~c"invalid mixed-script identifier found: "
+        prefix = ~c"invalid mixed-script identifier found: "
 
-      suffix = ~c"""
+        suffix = ~c"""
 
 
-      Mixed-script identifiers are not supported for security reasons. \
-      '#{acc}' is made of the following scripts:\n
-      #{breakdown}
-      Characters in identifiers from different scripts must be separated \
-      by underscore (_).
-      """
+        Mixed-script identifiers are not supported for security reasons. \
+        '#{acc}' is made of the following scripts:\n
+        #{breakdown}
+        Characters in identifiers from different scripts must be separated \
+        by underscore (_).
+        """
 
-      {:error, {:mixed_script, acc, {prefix, suffix}}}
+        {:error, {:mixed_script, acc, {prefix, suffix}}}
     end
   end
 
+  defp invalid_normalized(original_acc, acc, kind) do
+    case tokenize(acc) do
+      {^kind, _acc, [], _length, _ascii?, _special} ->
+        false
+
+      {:error, {:unexpected_token, invalid_prefix}} ->
+        invalid_original_prefix(original_acc, invalid_prefix)
+
+      _other ->
+        invalid_original_prefix(original_acc, acc)
+    end
+  end
+
+  defp invalid_original_prefix(original_acc, invalid_normalized_prefix) do
+    invalid_original_prefix(original_acc, invalid_normalized_prefix, [])
+  end
+
+  defp invalid_original_prefix([head | tail], invalid_normalized_prefix, prefix) do
+    prefix = [head | prefix]
+
+    if :unicode.characters_to_nfc_list(:lists.reverse(prefix)) == invalid_normalized_prefix do
+      :lists.reverse(prefix)
+    else
+      invalid_original_prefix(tail, invalid_normalized_prefix, prefix)
+    end
+  end
+
+  defp invalid_original_prefix([], _invalid_normalized_prefix, prefix) do
+    :lists.reverse(prefix)
+  end
+
   # Support script mixing via chunked identifiers (UTS 55-5's strong recommends).
   # Each chunk in an ident like foo_bar_baz should pass checks.
   defp chunks_single?(acc),