Add tests for #21

Author: d0rianb

Cargo.toml

@@ -25,6 +25,10 @@ debug = true
 default = ["jsbindings"]
 jsbindings = ["dep:wasm-bindgen", "dep:tsify"]
 
+[lints.clippy]
+needless_return = "allow"
+
+
 [dependencies]
 serde = { version = "1.0", features = ["derive"] }
 

src/lexer.rs

@@ -313,23 +313,27 @@ if (a == b) \{\
     }
 
     #[test]
-    fn should_handle_whitspace_group() {
-        let rtf = r"{\cf1  }"; // two whitespaces : one should be ignored, the other should be treated as plain text
+    fn should_lex_unicode() {
+        let rtf = r#"{\u21834  \u21834 }"#;
         let tokens = Lexer::scan(rtf).unwrap();
         assert_eq!(
             tokens,
-            [OpeningBracket, ControlSymbol((ColorNumber, Value(1))), PlainText(" "), ClosingBracket]
+            vec![OpeningBracket, ControlSymbol((Unicode, Value(21834))), PlainText(" "), ControlSymbol((Unicode, Value(21834))), ClosingBracket]
         );
     }
 
+    #[test]
+    fn should_handle_whitespace_group() {
+        let rtf = r"{\cf1  }"; // two whitespaces : one should be ignored, the other should be treated as plain text
+        let tokens = Lexer::scan(rtf).unwrap();
+        assert_eq!(tokens, [OpeningBracket, ControlSymbol((ColorNumber, Value(1))), PlainText(" "), ClosingBracket]);
+    }
+
     #[test]
     fn should_handle_google_docs_files() {
         use crate::include_test_file;
         let rtf = include_test_file!("google-docs.rtf");
         let tokens = Lexer::scan(rtf).unwrap();
-        assert_eq!(
-            tokens,
-            []
-        );
+        assert_eq!(tokens, []);
     }
 }