feat(header): Add support for none-ANSI character set & code page

Author: d0rianb

src/header.rs

@@ -39,6 +39,7 @@ pub struct Style {
 #[cfg_attr(feature = "jsbindings", derive(Tsify), tsify(into_wasm_abi, from_wasm_abi))]
 pub struct RtfHeader {
     pub character_set: CharacterSet,
+    pub code_page: Option<u16>,
     pub font_table: FontTable,
     pub color_table: ColorTable,
     pub stylesheet: StyleSheet,
@@ -66,17 +67,18 @@ pub struct Color {
 pub enum CharacterSet {
     #[default]
     Ansi,
-    Mac,
-    Pc,
-    Pca,
-    Ansicpg(u16),
+    Mac, // Apple Macintosh
+    Pc,  // IBM PC code page 437
+    Pca, // IBM PC code page 850
 }
 
 impl CharacterSet {
     pub fn from(token: &Token) -> Option<Self> {
         match token {
             Token::ControlSymbol((ControlWord::Ansi, _)) => Some(Self::Ansi),
-            // TODO: implement the rest
+            Token::ControlSymbol((ControlWord::Mac, _)) => Some(Self::Mac),
+            Token::ControlSymbol((ControlWord::Pc, _)) => Some(Self::Pc),
+            Token::ControlSymbol((ControlWord::Pca, _)) => Some(Self::Pca),
             _ => None,
         }
     }

src/lib.rs

@@ -1,6 +1,6 @@
 // RTF parser for Text Editor
 // This library supports RTF version 1.9.1
-// Specification is available here : https://dokumen.tips/documents/rtf-specification.html
+// Specification is available here : https://dokumen.tips/documents/rtf-specification.html and https://github.com/GoodNotes/tree-sitter-rtf/blob/main/specs/rtf-specs-1.9.1.pdf
 // Explanations on specification here : https://www.oreilly.com/library/view/rtf-pocket-guide/9781449302047/ch01.html
 
 #![allow(irrefutable_let_patterns)]

src/parser.rs

@@ -397,8 +397,15 @@ impl<'a> Parser<'a> {
                 }
                 // Check and consume token
                 (token, _) => {
-                    if let Some(charset) = CharacterSet::from(token) {
-                        header.character_set = charset;
+                    match token {
+                        Token::ControlSymbol((ControlWord::AnsiCpg, code_page)) => {
+                            header.code_page = Some(code_page.get_value_as::<u16>()?);
+                        }
+                        _ => {
+                            if let Some(character_set) = CharacterSet::from(token) {
+                                header.character_set = character_set;
+                            }
+                        }
                     }
                     self.cursor += 1;
                 }
@@ -497,12 +504,13 @@ pub mod tests {
 
     #[test]
     fn parser_header() {
-        let tokens = Lexer::scan(r#"{ \rtf1\ansi{\fonttbl\f0\fswiss Helvetica;}\f0\pard Voici du texte en {\b gras}.\par }"#).unwrap();
+        let tokens = Lexer::scan(r#"{ \rtf1\ansi\ansicpg1252{\fonttbl\f0\fswiss Helvetica;}\f0\pard Voici du texte en {\b gras}.\par }"#).unwrap();
         let doc = Parser::new(tokens).parse().unwrap();
         assert_eq!(
             doc.header,
             RtfHeader {
                 character_set: Ansi,
+                code_page: Some(1252),
                 font_table: FontTable::from([(
                     0,
                     Font {
@@ -562,6 +570,7 @@ pub mod tests {
             doc.header,
             RtfHeader {
                 character_set: Ansi,
+                code_page: Some(1252),
                 font_table: FontTable::from([
                     (
                         0,

src/tokens.rs

@@ -91,7 +91,14 @@ impl Property {
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
 pub enum ControlWord<'a> {
     Rtf,
-    Ansi,
+
+    // Character Set
+    Ansi, // default
+    Mac,  // Apple Macintosh
+    Pc,   // IBM PC code page 437
+    Pca,  // BM PC code page 850
+    AnsiCpg, // ANSI code page used to perform the Unicode to ANSI conversion
+
 
     Unicode,
     UnicodeIgnoreCount,
@@ -187,7 +194,12 @@ impl<'a> ControlWord<'a> {
         #[rustfmt::skip]
         let control_word = match prefix {
             r"\rtf"           => ControlWord::Rtf,
+            // Character set
             r"\ansi"          => ControlWord::Ansi,
+            r"\mac"           => ControlWord::Mac,
+            r"\pc"            => ControlWord::Pc,
+            r"\pca"           => ControlWord::Pca,
+            r"\ansicpg"       => ControlWord::AnsiCpg,
             // Unicode
             r"\u"             => ControlWord::Unicode,
             r"\uc"            => ControlWord::UnicodeIgnoreCount,
@@ -258,7 +270,9 @@ mod tests {
     #[test]
     fn control_word_from_input_test() {
         let input = r"\rtf1";
-        assert_eq!(ControlWord::from(input).unwrap(), (ControlWord::Rtf, Property::Value(1)))
+        assert_eq!(ControlWord::from(input).unwrap(), (ControlWord::Rtf, Property::Value(1)));
+        let cpg_input = r"\ansicpg1252";
+        assert_eq!(ControlWord::from(cpg_input).unwrap(), (ControlWord::AnsiCpg, Property::Value(1252)))
     }
 
     #[test]