From dea6cabe12b821cfc5a8ea777283bfb3a8cf9f1e Mon Sep 17 00:00:00 2001 From: Duarte David Date: Thu, 29 Sep 2016 21:43:22 +0200 Subject: [PATCH 1/6] Add simple scalar parser implementation --- examples/dump_yaml.rs | 2 +- examples/include_file_tag.rs | 61 ++++++++++++++++++++++++ src/yaml.rs | 89 +++++++++++++++++++++++++++++++++--- 3 files changed, 144 insertions(+), 8 deletions(-) create mode 100644 examples/include_file_tag.rs diff --git a/examples/dump_yaml.rs b/examples/dump_yaml.rs index 8fce0f3c..8558519f 100644 --- a/examples/dump_yaml.rs +++ b/examples/dump_yaml.rs @@ -11,7 +11,7 @@ fn print_indent(indent: usize) { } } -fn dump_node(doc: &yaml::Yaml, indent: usize) { +pub fn dump_node(doc: &yaml::Yaml, indent: usize) { match *doc { yaml::Yaml::Array(ref v) => { for x in v { diff --git a/examples/include_file_tag.rs b/examples/include_file_tag.rs new file mode 100644 index 00000000..e08e6931 --- /dev/null +++ b/examples/include_file_tag.rs @@ -0,0 +1,61 @@ +extern crate yaml_rust; + +mod dump_yaml; + +use std::path::Path; +use std::fs::File; +use std::io::Read; +use std::env; +use yaml_rust::yaml; +use yaml_rust::scanner; + +struct IncludeParser<'a> { + root: &'a Path +} + +impl<'a> IncludeParser<'a> { + fn new(root: &'a Path) -> IncludeParser { + IncludeParser { + root: root + } + } +} + +impl<'a> yaml::YamlScalarParser for IncludeParser<'a> { + fn parse_scalar(&self, tag: &scanner::TokenType, value: &str) -> Option { + if let scanner::TokenType::Tag(ref handle, ref suffix) = *tag { + if *handle == "!" && *suffix == "include" { + let mut content = String::new(); + return Some(match File::open(self.root.join(value)){ + Ok(mut f) => { + let _ = f.read_to_string(&mut content); + match yaml::YamlLoader::load_from_str(&content.to_owned()) { + Ok(mut docs) => docs.pop().unwrap(), + Err(_) => yaml::Yaml::BadValue + } + } + Err(_) => yaml::Yaml::BadValue + }) + } + } + None + } +} + +fn main() { + let args: Vec<_> = env::args().collect(); + let mut f = File::open(&args[1]).unwrap(); + let mut s = String::new(); + f.read_to_string(&mut s).unwrap(); + + let p = env::current_dir().unwrap(); + let parser = IncludeParser::new(p.as_path()); + let mut loader = yaml::YamlLoader::new(); + loader.register_scalar_parser(&parser); + + let docs = loader.parse_from_str(&s).unwrap(); + for doc in &docs { + println!("---"); + dump_yaml::dump_node(doc, 0); + } +} \ No newline at end of file diff --git a/src/yaml.rs b/src/yaml.rs index fe112cc3..779f35b8 100644 --- a/src/yaml.rs +++ b/src/yaml.rs @@ -66,16 +66,44 @@ fn parse_f64(v: &str) -> Option { } } -pub struct YamlLoader { +/// A `YamlScalarParser` is a parser that change the parsing of a yaml scalar value +/// like a tag +/// +/// # Examples +/// +/// ``` +///use yaml_rust::yaml; +///use yaml_rust::scanner; +/// +///struct HelloTagParser; +/// +///impl yaml::YamlScalarParser for HelloTagParser { +/// fn parse_scalar(&self, tag: &scanner::TokenType, value: &str) -> Option { +/// if let scanner::TokenType::Tag(ref handle, ref suffix) = *tag { +/// if *handle == "!" && *suffix == "hello" { +/// return Some(yaml::Yaml::String("Hello ".to_string() + value)) +/// } +/// } +/// None +/// } +///} +/// ``` +pub trait YamlScalarParser { + fn parse_scalar(&self, tag: &TokenType, value: &str) -> Option; +} + +#[derive(Default)] +pub struct YamlLoader<'a> { docs: Vec, // states // (current node, anchor_id) tuple doc_stack: Vec<(Yaml, usize)>, key_stack: Vec, anchor_map: BTreeMap, + scalar_parser: Vec<&'a YamlScalarParser>, } -impl MarkedEventReceiver for YamlLoader { +impl<'a> MarkedEventReceiver for YamlLoader<'a> { fn on_event(&mut self, ev: Event, _: Marker) { // println!("EV {:?}", ev); match ev { @@ -107,6 +135,17 @@ impl MarkedEventReceiver for YamlLoader { self.insert_new_node(node); } Event::Scalar(v, style, aid, tag) => { + if let Some(ref tag) = tag { + let mut yaml = None; + for parser in &self.scalar_parser { + yaml = parser.parse_scalar(tag, &v); + } + if let Some(yaml) = yaml { + self.insert_new_node((yaml, aid)); + return; + } + } + let node = if style != TScalarStyle::Plain { Yaml::String(v) } else if let Some(TokenType::Tag(ref handle, ref suffix)) = tag { @@ -157,7 +196,7 @@ impl MarkedEventReceiver for YamlLoader { } } -impl YamlLoader { +impl<'a> YamlLoader<'a> { fn insert_new_node(&mut self, node: (Yaml, usize)) { // valid anchor id starts from 1 if node.1 > 0 { @@ -186,16 +225,28 @@ impl YamlLoader { } } + pub fn register_scalar_parser(&mut self, parser: &'a YamlScalarParser) { + self.scalar_parser.push(parser); + } + pub fn load_from_str(source: &str) -> Result, ScanError> { - let mut loader = YamlLoader { + YamlLoader::new().parse_from_str(source) + } + + pub fn new() -> YamlLoader<'a> { + YamlLoader { docs: Vec::new(), doc_stack: Vec::new(), key_stack: Vec::new(), anchor_map: BTreeMap::new(), - }; + scalar_parser: Vec::new(), + } + } + + pub fn parse_from_str(mut self, source: &str) -> Result, ScanError> { let mut parser = Parser::new(source.chars()); - parser.load(&mut loader, true)?; - Ok(loader.docs) + parser.load(&mut self, true)?; + Ok(self.docs) } } @@ -367,6 +418,7 @@ impl Iterator for YamlIter { #[cfg(test)] mod test { + use scanner::*; use std::f64; use yaml::*; #[test] @@ -736,4 +788,27 @@ subcommands3: let s = "[".repeat(10_000) + &"]".repeat(10_000); assert!(YamlLoader::load_from_str(&s).is_err()); } + + struct HelloTagParser; + + impl YamlScalarParser for HelloTagParser { + fn parse_scalar(&self, tag: &TokenType, value: &str) -> Option { + if let TokenType::Tag(ref handle, ref suffix) = *tag { + if *handle == "!" && *suffix == "hello" { + return Some(Yaml::String("Hello ".to_string() + value)); + } + } + None + } + } + + #[test] + fn test_scalar_parser() { + let parser = HelloTagParser; + let mut loader = YamlLoader::new(); + loader.register_scalar_parser(&parser); + let out = loader.parse_from_str("- !hello world").unwrap(); + let doc = &out[0]; + assert_eq!(doc[0].as_str().unwrap(), "Hello world") + } } From dcb859f7ad5119f4db70573bbecc902b7eb68d7f Mon Sep 17 00:00:00 2001 From: Duarte David Date: Thu, 27 Oct 2016 23:35:48 +0200 Subject: [PATCH 2/6] Recursive !include parsing --- examples/include_file_tag.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/include_file_tag.rs b/examples/include_file_tag.rs index e08e6931..4876b872 100644 --- a/examples/include_file_tag.rs +++ b/examples/include_file_tag.rs @@ -29,7 +29,9 @@ impl<'a> yaml::YamlScalarParser for IncludeParser<'a> { return Some(match File::open(self.root.join(value)){ Ok(mut f) => { let _ = f.read_to_string(&mut content); - match yaml::YamlLoader::load_from_str(&content.to_owned()) { + let mut loader = yaml::YamlLoader::new(); + loader.register_scalar_parser(self); + match loader.parse_from_str(&content.to_owned()) { Ok(mut docs) => docs.pop().unwrap(), Err(_) => yaml::Yaml::BadValue } From ac176780db30f3a7da6eb27902f8b04d3a73eb9d Mon Sep 17 00:00:00 2001 From: Duarte David Date: Sat, 5 Nov 2016 18:33:13 +0100 Subject: [PATCH 3/6] Parse tag directives --- examples/include_file_tag.rs | 2 +- src/parser.rs | 29 +++++++++++++++++++++++++---- src/yaml.rs | 23 ++++++++++++++++++++--- 3 files changed, 46 insertions(+), 8 deletions(-) diff --git a/examples/include_file_tag.rs b/examples/include_file_tag.rs index 4876b872..6472434a 100644 --- a/examples/include_file_tag.rs +++ b/examples/include_file_tag.rs @@ -24,7 +24,7 @@ impl<'a> IncludeParser<'a> { impl<'a> yaml::YamlScalarParser for IncludeParser<'a> { fn parse_scalar(&self, tag: &scanner::TokenType, value: &str) -> Option { if let scanner::TokenType::Tag(ref handle, ref suffix) = *tag { - if *handle == "!" && *suffix == "include" { + if (*handle == "!" || *handle == "yaml-rust.include.prefix") && *suffix == "include" { let mut content = String::new(); return Some(match File::open(self.root.join(value)){ Ok(mut f) => { diff --git a/src/parser.rs b/src/parser.rs index 22692ffb..fef13fa5 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -72,6 +72,7 @@ pub struct Parser { current: Option<(Event, Marker)>, anchors: HashMap, anchor_id: usize, + tag_directives: HashMap, } pub trait EventReceiver { @@ -103,6 +104,7 @@ impl> Parser { anchors: HashMap::new(), // valid anchor_id starts from 1 anchor_id: 1, + tag_directives: HashMap::new(), } } @@ -374,14 +376,16 @@ impl> Parser { // "found incompatible YAML document")); //} } - TokenType::TagDirective(..) => { - // TODO add tag directive + TokenType::TagDirective(ref handle, ref prefix) => { + let handle = String::from(handle); + let mut prefix = String::from(prefix); + prefix.pop(); + self.tag_directives.insert(handle, prefix); } _ => break, } self.skip(); } - // TODO tag directive Ok(()) } @@ -502,7 +506,24 @@ impl> Parser { Token(_, TokenType::Scalar(..)) => { self.pop_state(); if let Token(mark, TokenType::Scalar(style, v)) = self.fetch_token() { - Ok((Event::Scalar(v, style, anchor_id, tag), mark)) + Ok(( + if let Some(TokenType::Tag(handle, suffix)) = tag { + let t = self.tag_directives.get(&handle); + Event::Scalar( + v, + style, + anchor_id, + Some(if let Some(s) = t { + TokenType::Tag((*s).clone(), suffix) + } else { + TokenType::Tag(handle, suffix) + }), + ) + } else { + Event::Scalar(v, style, anchor_id, tag) + }, + mark, + )) } else { unreachable!() } diff --git a/src/yaml.rs b/src/yaml.rs index 779f35b8..0a8dbefe 100644 --- a/src/yaml.rs +++ b/src/yaml.rs @@ -794,7 +794,7 @@ subcommands3: impl YamlScalarParser for HelloTagParser { fn parse_scalar(&self, tag: &TokenType, value: &str) -> Option { if let TokenType::Tag(ref handle, ref suffix) = *tag { - if *handle == "!" && *suffix == "hello" { + if (*handle == "!" || *handle == "yaml-rust.hello.prefix") && *suffix == "hello" { return Some(Yaml::String("Hello ".to_string() + value)); } } @@ -807,8 +807,25 @@ subcommands3: let parser = HelloTagParser; let mut loader = YamlLoader::new(); loader.register_scalar_parser(&parser); - let out = loader.parse_from_str("- !hello world").unwrap(); + let out = loader.parse_from_str("!hello world").unwrap(); let doc = &out[0]; - assert_eq!(doc[0].as_str().unwrap(), "Hello world") + assert_eq!(doc.as_str().unwrap(), "Hello world") + } + + #[test] + fn test_tag_directive() { + let parser = HelloTagParser; + let mut loader = YamlLoader::new(); + loader.register_scalar_parser(&parser); + let out = loader + .parse_from_str( + "%YAML 1.2 +%TAG ! yaml-rust.hello.prefix: +--- +!hello world", + ) + .unwrap(); + let doc = &out[0]; + assert_eq!(doc.as_str().unwrap(), "Hello world") } } From 9c1977187c8f9e9eb94863148dfd5185a2899705 Mon Sep 17 00:00:00 2001 From: Alexandre Bury Date: Tue, 6 Aug 2019 16:07:36 -0700 Subject: [PATCH 4/6] Fix lints --- src/emitter.rs | 15 +++++++-------- src/scanner.rs | 10 +++++----- src/yaml.rs | 4 ++-- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/src/emitter.rs b/src/emitter.rs index 09e9f876..3438b8f1 100644 --- a/src/emitter.rs +++ b/src/emitter.rs @@ -17,7 +17,7 @@ impl Error for EmitError { } } - fn cause(&self) -> Option<&Error> { + fn cause(&self) -> Option<&dyn Error> { None } } @@ -38,7 +38,7 @@ impl From for EmitError { } pub struct YamlEmitter<'a> { - writer: &'a mut fmt::Write, + writer: &'a mut dyn fmt::Write, best_indent: usize, compact: bool, @@ -48,7 +48,7 @@ pub struct YamlEmitter<'a> { pub type EmitResult = Result<(), EmitError>; // from serialize::json -fn escape_str(wr: &mut fmt::Write, v: &str) -> Result<(), fmt::Error> { +fn escape_str(wr: &mut dyn fmt::Write, v: &str) -> Result<(), fmt::Error> { wr.write_str("\"")?; let mut start = 0; @@ -111,7 +111,7 @@ fn escape_str(wr: &mut fmt::Write, v: &str) -> Result<(), fmt::Error> { } impl<'a> YamlEmitter<'a> { - pub fn new(writer: &'a mut fmt::Write) -> YamlEmitter { + pub fn new(writer: &'a mut dyn fmt::Write) -> YamlEmitter { YamlEmitter { writer, best_indent: 2, @@ -316,12 +316,12 @@ fn need_quotes(string: &str) -> bool { | '\"' | '\'' | '\\' - | '\0'...'\x06' + | '\0'..='\x06' | '\t' | '\n' | '\r' - | '\x0e'...'\x1a' - | '\x1c'...'\x1f' => true, + | '\x0e'..='\x1a' + | '\x1c'..='\x1f' => true, _ => false, }) || [ @@ -637,5 +637,4 @@ a: assert_eq!(s, writer); } - } diff --git a/src/scanner.rs b/src/scanner.rs index 6f4fa587..b2ce148b 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -67,7 +67,7 @@ impl Error for ScanError { self.info.as_ref() } - fn cause(&self) -> Option<&Error> { + fn cause(&self) -> Option<&dyn Error> { None } } @@ -199,7 +199,7 @@ fn is_digit(c: char) -> bool { #[inline] fn is_alpha(c: char) -> bool { match c { - '0'...'9' | 'a'...'z' | 'A'...'Z' => true, + '0'..='9' | 'a'..='z' | 'A'..='Z' => true, '_' | '-' => true, _ => false, } @@ -211,9 +211,9 @@ fn is_hex(c: char) -> bool { #[inline] fn as_hex(c: char) -> u32 { match c { - '0'...'9' => (c as u32) - ('0' as u32), - 'a'...'f' => (c as u32) - ('a' as u32) + 10, - 'A'...'F' => (c as u32) - ('A' as u32) + 10, + '0'..='9' => (c as u32) - ('0' as u32), + 'a'..='f' => (c as u32) - ('a' as u32) + 10, + 'A'..='F' => (c as u32) - ('A' as u32) + 10, _ => unreachable!(), } } diff --git a/src/yaml.rs b/src/yaml.rs index 0a8dbefe..15a750d3 100644 --- a/src/yaml.rs +++ b/src/yaml.rs @@ -100,7 +100,7 @@ pub struct YamlLoader<'a> { doc_stack: Vec<(Yaml, usize)>, key_stack: Vec, anchor_map: BTreeMap, - scalar_parser: Vec<&'a YamlScalarParser>, + scalar_parser: Vec<&'a dyn YamlScalarParser>, } impl<'a> MarkedEventReceiver for YamlLoader<'a> { @@ -225,7 +225,7 @@ impl<'a> YamlLoader<'a> { } } - pub fn register_scalar_parser(&mut self, parser: &'a YamlScalarParser) { + pub fn register_scalar_parser(&mut self, parser: &'a dyn YamlScalarParser) { self.scalar_parser.push(parser); } From 9e75f8b519968050726a07a78cdf7be819bb2a39 Mon Sep 17 00:00:00 2001 From: Alexandre Bury Date: Wed, 7 Aug 2019 10:34:00 -0700 Subject: [PATCH 5/6] Remove need for NLL - now compiles on 1.28.0 --- .travis.yml | 4 +--- appveyor.yml | 4 ++-- src/parser.rs | 23 +++++++++++++++++++---- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 908b9253..a640b6e8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,9 +5,7 @@ matrix: - rust: stable - rust: beta - rust: nightly - - rust: 1.17.0 - script: cargo build - - rust: 1.24.1 + - rust: 1.28.0 - rust: nightly env: CLIPPY script: | diff --git a/appveyor.yml b/appveyor.yml index d6707aa4..9d0a4f42 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,6 +1,6 @@ install: - - ps: Start-FileDownload 'https://static.rust-lang.org/dist/rust-1.24.1-i686-pc-windows-gnu.exe' - - rust-1.24.1-i686-pc-windows-gnu.exe /VERYSILENT /NORESTART /DIR="C:\Program Files (x86)\Rust" + - ps: Start-FileDownload 'https://static.rust-lang.org/dist/rust-1.28.0-i686-pc-windows-gnu.exe' + - rust-1.28.0-i686-pc-windows-gnu.exe /VERYSILENT /NORESTART /DIR="C:\Program Files (x86)\Rust" - SET PATH=%PATH%;C:\Program Files (x86)\Rust\bin - SET PATH=%PATH%;C:\MinGW\bin - rustc -V diff --git a/src/parser.rs b/src/parser.rs index fef13fa5..a23dfad0 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -367,23 +367,38 @@ impl> Parser { } fn parser_process_directives(&mut self) -> Result<(), ScanError> { + enum DirectiveAction { + None, + Tag { handle: String, prefix: String }, + } + loop { - match self.peek_token()?.1 { + // Without NLL, split the peek and the action + let action = match self.peek_token()?.1 { TokenType::VersionDirective(_, _) => { // XXX parsing with warning according to spec //if major != 1 || minor > 2 { // return Err(ScanError::new(tok.0, // "found incompatible YAML document")); //} + DirectiveAction::None } TokenType::TagDirective(ref handle, ref prefix) => { - let handle = String::from(handle); - let mut prefix = String::from(prefix); + let handle = String::clone(handle); + let mut prefix = String::clone(prefix); prefix.pop(); - self.tag_directives.insert(handle, prefix); + DirectiveAction::Tag { handle, prefix } } _ => break, + }; + + match action { + DirectiveAction::Tag { handle, prefix } => { + self.tag_directives.insert(handle, prefix); + } + _ => (), } + self.skip(); } Ok(()) From 25423d227a8fb49a07af81253619564608f73a44 Mon Sep 17 00:00:00 2001 From: Alexandre Bury Date: Wed, 7 Aug 2019 11:31:35 -0700 Subject: [PATCH 6/6] Run CI build (but not tests) on 1.27.0 --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index a640b6e8..6d99c958 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,9 @@ matrix: - rust: stable - rust: beta - rust: nightly + - rust: 1.27.0 + script: + cargo build - rust: 1.28.0 - rust: nightly env: CLIPPY