Merge #251: Move jets documentation inside compiler

57597c7 add README.md to `codegen` crate (Volodymyr Herashchenko)
03bc3c7 implement clap parser for codegen and json generation (Volodymyr Herashchenko)
2ea3a12 add dependencies for codegen binary (Volodymyr Herashchenko)
f8f7163 add helpers for Jet documentation (Volodymyr Herashchenko)
5d0e9d2 docs: fix typos in jet documentation (Volodymyr Herashchenko)
3a71244 move jet documentation to SimplicityHL (Volodymyr Herashchenko)

Pull request description:

  Right now we have 3 different sources in which we are getting information about Jet: LSP, `codegen` crate (which would generate modules for [SimplicityHL-as-Rust](https://github.com/BlockstreamResearch/SimplicityHL-as-rust)) and [simplicity-lang-org](https://github.com/BlockstreamResearch/simplicity-lang-org).

  This PR moves Jet documentation from `codegen` crate to SimplicityHL compiler, which is gated behind `docs` feature. Also, I had added `codegen` executable to be distributed alongside with `simplicityhl` crate, also gated behind `docs` feature. This executable supports old behavior of generating modules, and also extends to generating JSON file with jet documentation (see BlockstreamResearch/simplicity-lang-org#42 for more context). There are launch instructions in codegen/README.md.

  In future, I would like to add `JetInfo` as a trait for `Jet`, after [this issue](#224) resolved, so it could also load in runtime jets documentation.

ACKs for top commit:
  apoelstra:
    ACK 57597c7; successfully ran local tests
  KyrylR:
    ACK 57597c7; code review

Tree-SHA512: ff171e55477c841517e0449ccbe42313029c7ddb29f7cc469ccdfec168f77d17824382111a38de3cba35f02f55dd771224aa566814e4ff98152c53e1281fa109

Author: KyrylR

Cargo.lock

@@ -8,6 +8,7 @@ repository = "https://github.com/BlockstreamResearch/SimplicityHL"
 description = "Rust-like language that compiles to Simplicity bytecode."
 edition = "2021"
 rust-version = "1.79.0"
+default-run = "simc"
 
 [lib]
 name = "simplicityhl"
@@ -17,9 +18,15 @@ path = "src/lib.rs"
 name = "simc"
 path = "src/main.rs"
 
+[[bin]]
+name = "simplicityhl-codegen"
+path = "codegen/src/main.rs"
+required-features = ["docs", "serde"]
+
 [features]
 default = [ "serde" ]
 serde = ["dep:serde", "dep:serde_json"]
+docs = []
 
 [dependencies]
 base64 = "0.21.2"

Cargo.toml

@@ -6,4 +6,8 @@ description = "Generator of Rust code as interface between SimplicityHL and Rust
 publish = false
 
 [dependencies]
-simplicityhl = { path = ".." }
+simplicityhl = { path = ".." , features = ["docs"]}
+
+serde = { version = "1.0.188", features = ["derive"]}
+serde_json = { version = "1.0.105"}
+clap = "4.5.37"

codegen/Cargo.toml

@@ -0,0 +1,43 @@
+# Code and documentation generation
+
+## Generate Rust code
+
+Generate Rust modules, like in this [repository](https://github.com/BlockstreamResearch/SimplicityHL-as-rust):
+
+```bash
+cargo run --bin codegen -- modules --out-dir modules/
+```
+
+## Generate Jet documentation in a JSON file
+
+Generate JSON file containing jet information:
+
+```bash
+cargo run --bin codegen -- docs elements.json
+```
+
+This JSON file contains an array of jets, each of which includes the following information:
+- "haskell_name" -- The name of the jet used in the Haskell and Rust code.
+- "simplicityhl_name" -- The name of the jet as it is used in SimplicityHL.
+- "section" -- The category to which the jet belongs.
+- "input_type" -- The input types represented as a vector of `AliasedType`s in SimplicityHL, separated by comma.
+- "output_type" -- The output type represented as an `AliasedType` in SimplicityHL.
+- "description" -- A description of the jet's functionality.
+- "deprecated" (optional) -- Indicates if jet is deprecated.
+
+> [!NOTE]
+> Structure of generated JSON similar to file in [`simplicity-lang-org`](https://github.com/BlockstreamResearch/simplicity-lang-org/blob/28c67437c6cef3b111339443293a672d237d72a3/jets.json) repo, but it contains only array, without an object `elements` above.
+
+## Installation
+
+Install `simplicityhl` via `cargo` with `docs` feature -- `simplicityhl-codegen` would be installed alongside `simc`:
+
+```bash
+cargo install simplicityhl --features docs
+```
+
+Or install from local repository:
+
+```bash
+cargo install --path ./ --features docs
+```

codegen/README.md

@@ -1,14 +1,30 @@
-use std::fs::File;
+use std::fs::{self, File};
 use std::io;
+use std::path::PathBuf;
 
+use clap::{Arg, Command};
+use serde::Serialize;
+
+use simplicityhl::docs::jet;
+use simplicityhl::docs::jet::JetInfo;
 use simplicityhl::simplicity::jet::{Elements, Jet};
 use simplicityhl::types::TypeDeconstructible;
 
-mod jet;
+#[derive(Serialize)]
+struct JetObject {
+    pub haskell_name: String,
+    pub simplicityhl_name: String,
+    pub section: String,
+    pub input_type: String,
+    pub output_type: String,
+    pub description: String,
+    #[serde(skip_serializing_if = "std::ops::Not::not")]
+    pub deprecated: bool,
+}
 
 /// Write a SimplicityHL jet as a Rust function to the sink.
 fn write_jet<W: io::Write>(jet: Elements, w: &mut W) -> io::Result<()> {
-    for line in jet::documentation(jet).lines() {
+    for line in jet.documentation().lines() {
         match line.is_empty() {
             true => writeln!(w, "///")?,
             false => writeln!(w, "/// {line}")?,
@@ -49,20 +65,108 @@ fn write_module<W: io::Write>(category: jet::Category, w: &mut W) -> io::Result<
     writeln!(w)?;
     writeln!(w, "use super::*;")?;
 
-    for jet in category.iter().cloned() {
+    for jet in category.iter().copied() {
         writeln!(w)?;
         write_jet(jet, w)?;
     }
 
     Ok(())
 }
 
-fn main() -> io::Result<()> {
+/// Generate Rust modules divided by category.
+fn generate_modules(out_dir: PathBuf) -> Result<(), Box<dyn std::error::Error>> {
+    fs::create_dir_all(&out_dir)?;
+
     for category in jet::Category::ALL {
         let file_name = format!("{category}.rs");
-        let mut file = File::create(file_name)?;
+        let file_path = out_dir.join(file_name);
+        let mut file = File::create(&file_path)?;
         write_module(category, &mut file)?;
     }
 
+    println!(
+        "Successfully generated Rust modules in: {}",
+        out_dir.display()
+    );
+    Ok(())
+}
+
+/// Generate JSON file with jet documentation.
+fn generate_docs(output_path: PathBuf) -> Result<(), Box<dyn std::error::Error>> {
+    let generated_elements: Vec<JetObject> = jet::Category::ALL
+        .into_iter()
+        .flat_map(|category| {
+            let section_name = category.to_pretty_string();
+
+            category
+                .iter()
+                .map(move |&jet| JetObject {
+                    haskell_name: format!("{:?}", jet),
+                    simplicityhl_name: jet.to_string(),
+                    section: section_name.clone(),
+                    input_type: simplicityhl::jet::source_type(jet)
+                        .iter()
+                        .map(|ty| ty.to_string())
+                        .collect::<Vec<_>>()
+                        .join(", "),
+                    output_type: simplicityhl::jet::target_type(jet).to_string(),
+                    description: jet.documentation().to_string(),
+                    deprecated: jet.is_deprecated(),
+                })
+                .collect::<Vec<_>>()
+        })
+        .collect();
+
+    let json_string = serde_json::to_string_pretty(&generated_elements)?
+        // Replacing Rust documentation links to render normally in MarkDown.
+        .replace("[`", "`")
+        .replace("`]", "`");
+
+    fs::write(&output_path, json_string)?;
+
+    println!("Successfully wrote JSON data to: {}", output_path.display());
+
+    Ok(())
+}
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let matches = Command::new("simplicityhl-gen")
+        .about("Generates SimplicityHL Rust modules and JSON documentation for jets")
+        .subcommand_required(true)
+        .arg_required_else_help(true)
+        .subcommand(
+            Command::new("modules")
+                .about("Generates SimplicityHL jets as Rust modules")
+                .arg(
+                    Arg::new("out_dir")
+                        .short('o')
+                        .long("out-dir")
+                        .default_value(".")
+                        .help("Optional directory to output the .rs files (defaults to current directory)"),
+                ),
+        )
+        .subcommand(
+            Command::new("docs")
+                .about("Generates SimplicityHL documentation for jets as a JSON file")
+                .arg(
+                    Arg::new("output_path")
+                        .required(true)
+                        .help("Path to write the JSON documentation file"),
+                ),
+        )
+        .get_matches();
+
+    match matches.subcommand() {
+        Some(("modules", sub_matches)) => {
+            let out_dir = sub_matches.get_one::<String>("out_dir").unwrap();
+            generate_modules(PathBuf::from(out_dir))?;
+        }
+        Some(("docs", sub_matches)) => {
+            let output_path = sub_matches.get_one::<String>("output_path").unwrap();
+            generate_docs(PathBuf::from(output_path))?;
+        }
+        _ => unreachable!("Exhausted list of subcommands and subcommand_required is true"),
+    }
+
     Ok(())
 }