fix: Preserve whitespace in Google Docs RTF exports (#17)

Author: d0rianb

src/lexer.rs

@@ -119,9 +119,8 @@ impl Lexer {
                     let mut ret = vec![Token::ControlSymbol(control_word)];
                     recursive_tokenize!(tail, ret);
 
-                    // \u1234 \u1234 is ok, but \u1234  \u1234 is lost a space, \u1234   \u1234 lost two spaces, and so on
-                    // \u1234  1 -> No need to walk in here, it will enter plain text
-                    if control_word.0 == ControlWord::Unicode && tail.len() > 0 && tail.trim() == "" {
+                    // The first whitespace delimits the control word, the remaining ones are plain text
+                    if tail.len() > 0 && tail.is_only_whitespace() {
                         ret.push(Token::PlainText(tail));
                     }
                     return Ok(ret);
@@ -338,11 +337,4 @@ if (a == b) \{\
         assert_eq!(tokens, [OpeningBracket, ControlSymbol((ColorNumber, Value(1))), PlainText(" "), ClosingBracket]);
     }
 
-    #[test]
-    fn should_handle_google_docs_files() {
-        use crate::include_test_file;
-        let rtf = include_test_file!("google-docs.rtf");
-        let tokens = Lexer::scan(rtf).unwrap();
-        assert_eq!(tokens, []);
-    }
 }

src/parser.rs

@@ -625,6 +625,14 @@ pub mod tests {
         );
     }
 
+    #[test]
+    fn parse_google_docs_whitespaces() {
+        let rtf_content = include_test_file!("google-docs.rtf");
+        let tokens = Lexer::scan(rtf_content).unwrap();
+        let document = Parser::new(tokens).parse().unwrap();
+        assert_eq!(document.get_text(), "Lorem ipsum odor amet");
+    }
+
     #[test]
     fn parse_image_data() {
         // Try to parse without error

src/utils.rs

@@ -17,8 +17,7 @@ impl StrUtils for str {
     }
 
     fn is_only_whitespace(&self) -> bool {
-        // TODO
-        false
+        self.chars().all(|c| c.is_ascii_whitespace())
     }
 }