Merge #354: Rewrite the lexer and parser in human_encoding

56a2e06 migrate to logos for human_encoding (ivanlele)

Pull request description:

  This PR replaces the toolkit used in the human_encoding module with a logos lexer and a custom parser.


ACKs for top commit:
  apoelstra:
    ACK 56a2e06; successfully ran local tests


Tree-SHA512: 748654202f6a9d4a5239818e4c4a1f6a857e2593b0ba0d360cd859d156b39b9ae5d8ba70e5ada05719c7b329de93c81563c1c922bd616db490b38459734c229e

Author: apoelstra

Cargo-recent.lock

@@ -51,6 +51,12 @@ version = "0.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32637268377fc7b10a8c6d51de3e7fba1ce5dd371a96e342b34e6078db558e7f"
 
+[[package]]
+name = "beef"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1"
+
 [[package]]
 name = "bitcoin"
 version = "0.32.8"
@@ -160,6 +166,12 @@ version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844"
 
+[[package]]
+name = "fnv"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
+
 [[package]]
 name = "getrandom"
 version = "0.2.16"
@@ -238,6 +250,12 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "lazy_static"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
+
 [[package]]
 name = "libc"
 version = "0.2.178"
@@ -254,6 +272,40 @@ dependencies = [
  "cc",
 ]
 
+[[package]]
+name = "logos"
+version = "0.15.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff472f899b4ec2d99161c51f60ff7075eeb3097069a36050d8037a6325eb8154"
+dependencies = [
+ "logos-derive",
+]
+
+[[package]]
+name = "logos-codegen"
+version = "0.15.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "192a3a2b90b0c05b27a0b2c43eecdb7c415e29243acc3f89cc8247a5b693045c"
+dependencies = [
+ "beef",
+ "fnv",
+ "lazy_static",
+ "proc-macro2",
+ "quote",
+ "regex-syntax",
+ "rustc_version",
+ "syn",
+]
+
+[[package]]
+name = "logos-derive"
+version = "0.15.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "605d9697bcd5ef3a42d38efc51541aa3d6a4a25f7ab6d1ed0da5ac632a26b470"
+dependencies = [
+ "logos-codegen",
+]
+
 [[package]]
 name = "memchr"
 version = "2.7.6"
@@ -368,6 +420,15 @@ version = "0.8.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58"
 
+[[package]]
+name = "rustc_version"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
+dependencies = [
+ "semver",
+]
+
 [[package]]
 name = "rustversion"
 version = "1.0.22"
@@ -427,6 +488,12 @@ dependencies = [
  "secp256k1-sys",
 ]
 
+[[package]]
+name = "semver"
+version = "1.0.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2"
+
 [[package]]
 name = "serde"
 version = "1.0.228"
@@ -519,8 +586,8 @@ dependencies = [
  "getrandom 0.2.16",
  "ghost-cell",
  "hex-conservative 0.2.2",
+ "logos",
  "miniscript",
- "santiago",
  "serde",
  "serde_json",
  "simplicity-sys 0.6.2",

Cargo.toml

@@ -29,7 +29,7 @@ elements = { version = "0.25.0", optional = true, default-features = false }
 ghost-cell = { version = "0.2.6", default-features = false }
 hashes = { package = "bitcoin_hashes", version = "0.14" }
 hex = { package = "hex-conservative", version = "0.2.1" }
-santiago = "1.3"
+logos = "0.15"
 simplicity-sys = { version = "0.6.2", path = "./simplicity-sys" }
 serde = { version = "1.0.103", features = ["derive"], optional = true }
 

src/human_encoding/README.md

@@ -15,9 +15,7 @@ With that said, the rest of the document defines the encoding.
 
 ## Syntax
 
-The syntax is defined in `src/human_encoding/parse/ast.rs`. It currently uses the
-`santiago` parser generator, but we would like to move away from this, probably to
-an ad-hoc parser, to avoid poor asymptotic behavior and to get better error messages.
+The syntax is defined in `src/human_encoding/parse/ast.rs`.
 
 Comments are started by `--` and end at the next newline. This is the only aspect
 in which whitespace is significant.

src/human_encoding/error.rs

@@ -2,7 +2,6 @@
 
 //! Parsing Errors
 
-use santiago::lexer::Lexeme;
 use std::collections::BTreeMap;
 use std::sync::{Arc, Mutex};
 use std::{error, fmt, iter};
@@ -19,22 +18,6 @@ pub struct ErrorSet {
     errors: BTreeMap<Option<Position>, Vec<Error>>,
 }
 
-impl<T> From<santiago::parser::ParseError<T>> for ErrorSet {
-    fn from(e: santiago::parser::ParseError<T>) -> Self {
-        let lex = e.at.map(|rc| (*rc).clone());
-        match lex.as_ref().map(|lex| &lex.position).map(Position::from) {
-            Some(pos) => ErrorSet::single(pos, Error::ParseFailed(lex)),
-            None => ErrorSet::single_no_position(Error::ParseFailed(lex)),
-        }
-    }
-}
-
-impl From<santiago::lexer::LexerError> for ErrorSet {
-    fn from(e: santiago::lexer::LexerError) -> Self {
-        ErrorSet::single(e.position, Error::LexFailed(e.message))
-    }
-}
-
 impl ErrorSet {
     /// Constructs a new empty error set.
     pub fn new() -> Self {
@@ -260,10 +243,10 @@ pub enum Error {
     NameRepeated(Arc<str>),
     /// Program did not have a `main` expression
     NoMain,
-    /// Parsing failed (the parser provides us some extra information, but beyond
-    /// the line and column, it does not seem very useful to a user, so we drop it).
-    ParseFailed(Option<Lexeme>),
-    /// Lexing failed; here santiago provides us an error message which is useful
+    /// Parsing failed; the string (if any) is a description of the token at which
+    /// parsing failed.
+    ParseFailed(Option<String>),
+    /// Lexing failed
     LexFailed(String),
     /// A number was parsed in some context but was out of range.
     NumberOutOfRange(String),
@@ -327,7 +310,7 @@ impl fmt::Display for Error {
                 write!(f, "number {} was out of allowable range", n)
             }
             Error::ParseFailed(None) => f.write_str("could not parse"),
-            Error::ParseFailed(Some(ref lex)) => write!(f, "could not parse `{}`", lex.raw),
+            Error::ParseFailed(Some(ref raw)) => write!(f, "could not parse `{}`", raw),
             Error::LexFailed(ref msg) => write!(f, "could not parse: {}", msg),
             Error::TypeCheck(ref e) => fmt::Display::fmt(e, f),
             Error::Undefined(ref s) => write!(f, "reference to undefined symbol `{}`", s),

src/human_encoding/mod.rs

@@ -24,31 +24,16 @@ pub use self::error::{Error, ErrorSet};
 pub use self::named_node::NamedCommitNode;
 
 /// Line/column pair
-///
-/// There is a similar type provided by the `santiago` library but it does not implement
-/// `Copy`, among many other traits, which makes it unergonomic to use. Santiago positions
-/// can be converted using `.into()`.
 #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Default, Hash)]
 pub struct Position {
     line: usize,
     column: usize,
 }
 
-impl<'a> From<&'a santiago::lexer::Position> for Position {
-    fn from(position: &'a santiago::lexer::Position) -> Position {
-        Position {
-            line: position.line,
-            column: position.column,
-        }
-    }
-}
-
-impl From<santiago::lexer::Position> for Position {
-    fn from(position: santiago::lexer::Position) -> Position {
-        Position {
-            line: position.line,
-            column: position.column,
-        }
+impl Position {
+    /// Create a new position from line and column.
+    pub fn new(line: usize, column: usize) -> Self {
+        Position { line, column }
     }
 }