Disallow raw CR line ending in strings, comments and after ? (#15307)

Author: lukaszsamson

lib/elixir/src/elixir_interpolation.erl

@@ -33,6 +33,9 @@ extract([$\\, $\r, $\n | Rest], Buffer, Output, Line, _Column, Scope, Interpol,
 extract([$\\, $\n | Rest], Buffer, Output, Line, _Column, Scope, Interpol, Last) ->
   extract_nl(Rest, [$\n, $\\ | Buffer], Output, Line, Scope, Interpol, Last);
 
+extract([$\r, $\n | Rest], Buffer, Output, Line, _Column, Scope, Interpol, Last) ->
+  extract_nl(Rest, [$\n, $\r | Buffer], Output, Line, Scope, Interpol, Last);
+
 extract([$\n | Rest], Buffer, Output, Line, _Column, Scope, Interpol, Last) ->
   extract_nl(Rest, [$\n | Buffer], Output, Line, Scope, Interpol, Last);
 

lib/elixir/src/elixir_tokenizer.erl

@@ -216,6 +216,11 @@ tokenize([$~, H | _T] = Original, Line, Column, Scope, Tokens) when ?is_upcase(H
 % elixir_errors.erl as by default {char, _, _} tokens are "hijacked" by Erlang
 % and printed with Erlang syntax ($a) in the parser's error messages.
 
+%% Reject bare carriage return after ?\ (CR is only valid as part of CRLF line ending).
+tokenize([$?, $\\, $\r | _Rest] = Original, Line, Column, Scope, Tokens) ->
+  Reason = {?LOC(Line, Column), "invalid bare carriage return after ?\\, use ?\\r instead: ", "\\u000D"},
+  error(Reason, Original, Scope, Tokens);
+
 tokenize([$?, $\\, H | T], Line, Column, Scope, Tokens) ->
   Char = elixir_interpolation:unescape_map(H),
 
@@ -248,6 +253,11 @@ tokenize([$?, $\\, H | T], Line, Column, Scope, Tokens) ->
       tokenize(T, Line, Column + 3, NewScope, [Token | Tokens])
   end;
 
+%% Reject bare carriage return after ? (CR is only valid as part of CRLF line ending).
+tokenize([$?, $\r | _Rest] = Original, Line, Column, Scope, Tokens) ->
+  Reason = {?LOC(Line, Column), "invalid bare carriage return after ?, use ?\\r instead: ", "\\u000D"},
+  error(Reason, Original, Scope, Tokens);
+
 tokenize([$?, Char | T], Line, Column, Scope, Tokens) ->
   NewScope = case handle_char(Char) of
     {Escape, Name} ->

lib/elixir/src/elixir_tokenizer.hrl

@@ -36,8 +36,11 @@
 
 %% Unsupported newlines
 %% https://www.unicode.org/reports/tr55/
+%% CR (0x000D) is included: a bare CR not followed by LF is rejected.
+%% Call sites must handle CRLF explicitly before the ?break check.
 -define(break(C), C =:= 16#000B orelse
                   C =:= 16#000C orelse
+                  C =:= 16#000D orelse
                   C =:= 16#0085 orelse
                   C =:= 16#2028 orelse
                   C =:= 16#2029).

lib/elixir/test/elixir/kernel/parser_test.exs

@@ -1149,6 +1149,105 @@ defmodule Kernel.ParserTest do
       )
     end
 
+    test "invalid bare carriage return in source" do
+      # Bare CR at top level (already produces an unexpected-token error)
+      assert_syntax_error(
+        ["nofile:1:1:", "unexpected token: carriage return (column 1, code point U+000D)"],
+        ~c"\r"
+      )
+
+      # CRLF is still a valid line ending at the top level
+      assert Code.string_to_quoted!("x = 1\r\ny = 2") ==
+               {:__block__, [],
+                [
+                  {:=, [line: 1], [{:x, [line: 1], nil}, 1]},
+                  {:=, [line: 2], [{:y, [line: 2], nil}, 2]}
+                ]}
+
+      # Bare CR inside a comment (Trojan Source via comment camouflage)
+      assert_syntax_error(
+        ["nofile:1:1:", "invalid line break character in comment: \\u000D"],
+        ~c"# safe comment" ++ [13] ++ ~c"hidden_code()"
+      )
+
+      # CRLF correctly terminates a comment (still valid)
+      assert Code.string_to_quoted!("# comment\r\nx = 1") ==
+               {:=, [line: 2], [{:x, [line: 2], nil}, 1]}
+
+      # Bare CR inside a string
+      assert_syntax_error(
+        [
+          "nofile:1:12:",
+          "invalid line break character in string: \\u000D. If you want to use such character, use it in its escaped \\u000D form instead"
+        ],
+        [34] ++ ~c"this is a " ++ [13, 34]
+      )
+
+      # Bare CR after backslash inside a string
+      assert_syntax_error(
+        [
+          "nofile:1:13:",
+          "invalid line break character in string: \\u000D. If you want to use such character, use it in its escaped \\u000D form instead"
+        ],
+        [34] ++ ~c"this is a " ++ [?\\, 13, 34]
+      )
+
+      # CRLF inside a string is preserved as content (same as before)
+      assert Code.string_to_quoted!([34] ++ ~c"hello" ++ [13, 10] ++ ~c"world" ++ [34]) ==
+               "hello\r\nworld"
+
+      # Bare CR inside a charlist
+      assert_syntax_error(
+        ["invalid line break character in string: \\u000D"],
+        [39] ++ ~c"this is a " ++ [13, 39]
+      )
+
+      # Bare CR inside a heredoc
+      assert_syntax_error(
+        ["invalid line break character in string: \\u000D"],
+        ~c"\"\"\"\nhello" ++ [13] ++ ~c"world\n\"\"\""
+      )
+
+      # Bare CR inside a sigil
+      assert_syntax_error(
+        ["invalid line break character in string: \\u000D"],
+        ~c"~s(hello" ++ [13] ++ ~c"world)"
+      )
+
+      # Bare CR inside a quoted atom
+      assert_syntax_error(
+        ["invalid line break character in string: \\u000D"],
+        ~c":\"foo" ++ [13] ++ ~c"bar\""
+      )
+
+      # Bare CR inside a quoted keyword
+      assert_syntax_error(
+        ["invalid line break character in string: \\u000D"],
+        ~c"[\"foo" ++ [13] ++ ~c"bar\": 1]"
+      )
+
+      # Bare CR inside a quoted call (quoted identifier)
+      assert_syntax_error(
+        ["invalid line break character in string: \\u000D"],
+        ~c"x.\"foo" ++ [13] ++ ~c"bar\""
+      )
+
+      # Bare CR after ? (char literal)
+      assert_syntax_error(
+        ["nofile:1:1:", "invalid bare carriage return after ?"],
+        ~c"?" ++ [13]
+      )
+
+      # Bare CR after ?\ (char literal escape)
+      assert_syntax_error(
+        ["nofile:1:1:", "invalid bare carriage return after ?\\"],
+        ~c"?\\" ++ [13]
+      )
+
+      # ?\r (the proper escape) is still valid
+      assert Code.string_to_quoted!(~c"?\\r") == ?\r
+    end
+
     test "reserved tokens" do
       assert_syntax_error(["nofile:1:1:", "reserved token: __aliases__"], ~c"__aliases__")
       assert_syntax_error(["nofile:1:1:", "reserved token: __block__"], ~c"__block__")