Port website dev docs generated content scripts from JS to Rust to avoid intermediate parsing (#3909)

* Remove the crate dependency graph website generation intermediate generation step

* Remove the message system tree website generation intermediate generation step

* Code cleanup

* Remove cache system

* Fix Windows comment URL error

* Fix incorrect artifact download URLs

* Add Flatpak to comment

* Make Flatpak use debug/release mode choice instead of always release mode

Author: Keavon

.github/workflows/build.yml

@@ -145,7 +145,7 @@ jobs:
             gh api \
               -X POST \
               -H "Accept: application/vnd.github+json" \
-              /repos/${{ github.repository }}/commits/$(git rev-parse HEAD)/comments \
+              repos/${{ github.repository }}/commits/$(git rev-parse HEAD)/comments \
               -f body="$COMMENT_BODY"
           else
             # Comment on the PR (use provided PR number from !build, or look it up by branch name)
@@ -172,41 +172,16 @@ jobs:
           name: graphite-web-bundle
           path: frontend/dist
 
-      - name: 📃 Generate code documentation info for website
+      - name: 📃 Trigger website rebuild if auto-generated code docs are stale
         if: github.event_name == 'push'
-        run: |
-          mkdir -p website/generated-new
-          cargo run -p crate-hierarchy-viz -- website/generated-new/crate_hierarchy.dot
-          cargo run -p editor-message-tree -- website/generated-new/hierarchical_message_system_tree.txt
-
-      - name: 💿 Obtain cache of auto-generated code docs artifacts, to check if they've changed
-        if: github.event_name == 'push'
-        id: cache-website-code-docs
-        uses: actions/cache/restore@v5
-        with:
-          path: website/generated
-          key: website-code-docs
-
-      - name: 🔍 Check if auto-generated code docs artifacts changed
-        if: github.event_name == 'push'
-        id: website-code-docs-changed
-        run: |
-          diff --brief --recursive website/generated-new website/generated || echo "changed=true" >> $GITHUB_OUTPUT
-          rm -rf website/generated
-          mv website/generated-new website/generated
-
-      - name: 💾 Save cache of auto-generated code docs artifacts
-        if: github.event_name == 'push' && steps.website-code-docs-changed.outputs.changed == 'true'
-        uses: actions/cache/save@v5
-        with:
-          path: website/generated
-          key: ${{ steps.cache-website-code-docs.outputs.cache-primary-key }}
-
-      - name: ♻️ Trigger website rebuild if the auto-generated code docs artifacts have changed
-        if: github.event_name == 'push' && steps.website-code-docs-changed.outputs.changed == 'true'
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: gh workflow run website.yml --ref master
+        run: |
+          cargo run -p editor-message-tree -- website/generated
+          TREE=volunteer/guide/codebase-overview/hierarchical-message-system-tree
+          curl -sf "https://graphite.art/$TREE.txt" -o "website/static/$TREE.live.txt" \
+            && diff -q "website/static/$TREE.txt" "website/static/$TREE.live.txt" > /dev/null \
+            || gh workflow run website.yml --ref master
 
   windows:
     if: github.event_name == 'push' || inputs.windows
@@ -302,16 +277,18 @@ jobs:
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          ARTIFACT_URL=$(gh api /repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts --jq '.artifacts[] | select(.name == "graphite-windows-bundle") | .archive_download_url')
+          ARTIFACT_ID=$(gh api repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts --jq '.artifacts[] | select(.name == "graphite-windows-bundle") | .id')
+          ARTIFACT_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts/$ARTIFACT_ID"
           PR_NUMBER="${{ inputs.pr_number }}"
           if [ -z "$PR_NUMBER" ]; then
             BRANCH=$(git rev-parse --abbrev-ref HEAD)
             PR_NUMBER=$(gh pr list --repo ${{ github.repository }} --head "$BRANCH" --json number --jq '.[0].number' 2>/dev/null || true)
           fi
-          if [ -n "$PR_NUMBER" ] && [ -n "$ARTIFACT_URL" ]; then
-            gh pr comment "$PR_NUMBER" --repo ${{ github.repository }} --body "| 📦 **Windows Build Complete for** $(git rev-parse HEAD) |
-          |-|
-          | [Download artifact]($ARTIFACT_URL) |"
+          if [ -n "$PR_NUMBER" ] && [ -n "$ARTIFACT_ID" ]; then
+            BODY="| 📦 **Windows Build Complete for** $(git rev-parse HEAD) |"$'\n'
+            BODY+="|-|"$'\n'
+            BODY+="| [Download binary]($ARTIFACT_URL) |"
+            gh pr comment "$PR_NUMBER" --repo ${{ github.repository }} --body "$BODY"
           fi
 
       - name: 🔑 Azure login
@@ -488,16 +465,18 @@ jobs:
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          ARTIFACT_URL=$(gh api /repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts --jq '.artifacts[] | select(.name == "graphite-mac-bundle") | .archive_download_url')
+          ARTIFACT_ID=$(gh api repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts --jq '.artifacts[] | select(.name == "graphite-mac-bundle") | .id')
+          ARTIFACT_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts/$ARTIFACT_ID"
           PR_NUMBER="${{ inputs.pr_number }}"
           if [ -z "$PR_NUMBER" ]; then
             BRANCH=$(git rev-parse --abbrev-ref HEAD)
             PR_NUMBER=$(gh pr list --repo ${{ github.repository }} --head "$BRANCH" --json number --jq '.[0].number' 2>/dev/null || true)
           fi
-          if [ -n "$PR_NUMBER" ] && [ -n "$ARTIFACT_URL" ]; then
-            gh pr comment "$PR_NUMBER" --repo ${{ github.repository }} --body "| 📦 **Mac Build Complete for** $(git rev-parse HEAD) |
-          |-|
-          | [Download artifact]($ARTIFACT_URL) |"
+          if [ -n "$PR_NUMBER" ] && [ -n "$ARTIFACT_ID" ]; then
+            BODY="| 📦 **Mac Build Complete for** $(git rev-parse HEAD) |"$'\n'
+            BODY+="|-|"$'\n'
+            BODY+="| [Download binary]($ARTIFACT_URL) |"
+            gh pr comment "$PR_NUMBER" --repo ${{ github.repository }} --body "$BODY"
           fi
 
       - name: 🔏 Sign and notarize (preparation)
@@ -616,20 +595,24 @@ jobs:
           compression-level: 0
 
       - name: 💬 Comment artifact link on PR
+        id: linux-comment
         if: github.event_name != 'push'
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          ARTIFACT_URL=$(gh api /repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts --jq '.artifacts[] | select(.name == "graphite-linux-bundle") | .archive_download_url')
+          ARTIFACT_ID=$(gh api repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts --jq '.artifacts[] | select(.name == "graphite-linux-bundle") | .id')
+          ARTIFACT_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts/$ARTIFACT_ID"
           PR_NUMBER="${{ inputs.pr_number }}"
           if [ -z "$PR_NUMBER" ]; then
             BRANCH=$(git rev-parse --abbrev-ref HEAD)
             PR_NUMBER=$(gh pr list --repo ${{ github.repository }} --head "$BRANCH" --json number --jq '.[0].number' 2>/dev/null || true)
           fi
-          if [ -n "$PR_NUMBER" ] && [ -n "$ARTIFACT_URL" ]; then
-            gh pr comment "$PR_NUMBER" --repo ${{ github.repository }} --body "| 📦 **Linux Build Complete for** $(git rev-parse HEAD) |
-          |-|
-          | [Download artifact]($ARTIFACT_URL) |"
+          if [ -n "$PR_NUMBER" ] && [ -n "$ARTIFACT_ID" ]; then
+            BODY="| 📦 **Linux Build Complete for** $(git rev-parse HEAD) |"$'\n'
+            BODY+="|-|"$'\n'
+            BODY+="| [Download binary]($ARTIFACT_URL) |"
+            COMMENT_ID=$(gh api repos/${{ github.repository }}/issues/$PR_NUMBER/comments -f body="$BODY" --jq '.id')
+            echo "comment_id=$COMMENT_ID" >> "$GITHUB_OUTPUT"
           fi
 
       - name: 🔧 Install Flatpak tooling
@@ -640,7 +623,7 @@ jobs:
 
       - name: 🏗 Build Flatpak
         run: |
-          nix build .#graphite-flatpak-manifest
+          nix build .#graphite${{ inputs.debug && '-dev' || '' }}-flatpak-manifest
 
           rm -rf .flatpak
           mkdir -p .flatpak
@@ -660,3 +643,18 @@ jobs:
           name: graphite-flatpak
           path: .flatpak/Graphite.flatpak
           compression-level: 0
+
+      - name: 💬 Update PR comment with Flatpak artifact link
+        if: github.event_name != 'push' && steps.linux-comment.outputs.comment_id
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          ARTIFACT_ID=$(gh api repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts --jq '.artifacts[] | select(.name == "graphite-flatpak") | .id')
+          ARTIFACT_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts/$ARTIFACT_ID"
+          COMMENT_ID="${{ steps.linux-comment.outputs.comment_id }}"
+          if [ -n "$ARTIFACT_ID" ]; then
+            EXISTING_BODY=$(gh api repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
+            BODY="$EXISTING_BODY"$'\n'
+            BODY+="| [Download Flatpak]($ARTIFACT_URL) |"
+            gh api repos/${{ github.repository }}/issues/comments/$COMMENT_ID -X PATCH -f body="$BODY"
+          fi

.github/workflows/website.yml

@@ -49,30 +49,16 @@ jobs:
           # Remove the INDEX_HTML_HEAD_INCLUSION environment variable for build links (not master deploys)
           git rev-parse --abbrev-ref HEAD | grep master > /dev/null || export INDEX_HTML_HEAD_INCLUSION=""
 
-      - name: 💿 Obtain cache of auto-generated code docs artifacts
-        id: cache-website-code-docs
-        uses: actions/cache/restore@v5
-        with:
-          path: website/generated
-          key: website-code-docs
-
-      - name: 📁 Fallback in case auto-generated code docs artifacts weren't cached
-        if: steps.cache-website-code-docs.outputs.cache-hit != 'true'
+      - name: 🦀 Produce auto-generated code docs data
         run: |
-          echo "🦀 Initial system version of Rust:"
-          rustc --version
           rustup update stable
-          echo "🦀 Latest updated version of Rust:"
-          rustc --version
-          cargo run -p crate-hierarchy-viz -- website/generated/crate_hierarchy.dot
-          cargo run -p editor-message-tree -- website/generated/hierarchical_message_system_tree.txt
+          cargo run -p crate-hierarchy-viz -- website/generated
+          cargo run -p editor-message-tree -- website/generated
 
-      - name: 🔧 Build auto-generated code docs artifacts into HTML/SVG
+      - name: 🔧 Install website npm dependencies
         run: |
           cd website
           npm ci
-          npm run generate-editor-structure
-          npm run generate-crate-hierarchy
 
       - name: 📃 Generate node catalog documentation
         run: cargo run -p node-docs -- website/content/learn/node-catalog

.nix/default.nix

@@ -61,7 +61,8 @@ in
       graphite-branding = lib.call ./pkgs/graphite-branding.nix;
       graphite-bundle = (lib.call ./pkgs/graphite-bundle.nix) { };
       graphite-dev-bundle = (lib.call ./pkgs/graphite-bundle.nix) { graphite = graphite-dev; };
-      graphite-flatpak-manifest = lib.call ./pkgs/graphite-flatpak-manifest.nix;
+      graphite-flatpak-manifest = (lib.call ./pkgs/graphite-flatpak-manifest.nix) { };
+      graphite-dev-flatpak-manifest = (lib.call ./pkgs/graphite-flatpak-manifest.nix) { graphite-bundle = graphite-dev-bundle; };
 
       # TODO: graphene-cli = lib.call ./pkgs/graphene-cli.nix;
 

.nix/pkgs/graphite-flatpak-manifest.nix

@@ -4,6 +4,9 @@
   system,
   ...
 }:
+{
+  graphite-bundle ? self.packages.${system}.graphite-bundle,
+}:
 
 (pkgs.formats.json { }).generate "art.graphite.Graphite.json" {
   app-id = "art.graphite.Graphite";
@@ -30,7 +33,7 @@
       sources = [
         {
           type = "archive";
-          path = self.packages.${system}.graphite-bundle.tar;
+          path = graphite-bundle.tar;
           strip-components = 0;
         }
       ];

tools/crate-hierarchy-viz/src/main.rs

@@ -2,7 +2,9 @@ use anyhow::{Context, Result};
 use serde::Deserialize;
 use std::collections::{HashMap, HashSet};
 use std::fs;
+use std::io::Write;
 use std::path::PathBuf;
+use std::process::Command;
 
 #[derive(Debug, Deserialize)]
 struct WorkspaceToml {
@@ -83,12 +85,13 @@ fn collect_all_dependencies(crate_name: &str, dep_map: &HashMap<String, HashSet<
 }
 
 fn main() -> Result<()> {
-	let output_path = std::env::args_os()
+	let output_dir = std::env::args_os()
 		.nth(1)
 		.map(PathBuf::from)
-		.ok_or_else(|| anyhow::anyhow!("Usage: crate-hierarchy-viz <output-file>"))?;
+		.ok_or_else(|| anyhow::anyhow!("Usage: crate-hierarchy-viz <output-directory>"))?;
+	let output_path = output_dir.join("crate-hierarchy.svg");
 
-	let workspace_root = std::env::current_dir().unwrap();
+	let workspace_root = std::env::current_dir()?;
 	let workspace_toml_path = workspace_root.join("Cargo.toml");
 
 	// Parse workspace Cargo.toml
@@ -173,17 +176,79 @@ fn main() -> Result<()> {
 
 	remove_transitive_dependencies(&mut crates);
 
-	// Generate DOT format and write to output file
+	// Generate DOT format, convert to SVG, and write to output file
 	let dot_content = generate_dot(&crates);
+	let svg_content = dot_to_svg(&dot_content)?;
 
-	if let Some(parent) = output_path.parent() {
-		fs::create_dir_all(parent).with_context(|| format!("Failed to create directory {:?}", parent))?;
-	}
-	fs::write(&output_path, &dot_content).with_context(|| format!("Failed to write to {:?}", output_path))?;
+	fs::create_dir_all(&output_dir).with_context(|| format!("Failed to create directory {:?}", output_dir))?;
+	fs::write(&output_path, &svg_content).with_context(|| format!("Failed to write to {:?}", output_path))?;
 
 	Ok(())
 }
 
+/// Convert a DOT graph string to SVG by shelling out to @viz-js/viz via Node.js
+fn dot_to_svg(dot: &str) -> Result<String> {
+	let temp_dir = std::env::temp_dir().join("crate-hierarchy-viz");
+	fs::create_dir_all(&temp_dir).with_context(|| "Failed to create temp directory")?;
+
+	// Install @viz-js/viz into the temp directory if not already present
+	let viz_package = temp_dir.join("node_modules").join("@viz-js").join("viz");
+	if !viz_package.exists() {
+		let npm = if cfg!(target_os = "windows") { "npm.cmd" } else { "npm" };
+		let status = Command::new(npm)
+			.args(["install", "--prefix", &temp_dir.to_string_lossy(), "@viz-js/viz"])
+			.stdout(std::process::Stdio::null())
+			.stderr(std::process::Stdio::piped())
+			.status()
+			.with_context(|| "Failed to run `npm install`. Is Node.js installed?")?;
+		if !status.success() {
+			anyhow::bail!("Executing `npm install @viz-js/viz` failed");
+		}
+	}
+
+	// Write a small script that reads DOT from stdin and outputs SVG
+	let script_path = temp_dir.join("convert.mjs");
+	fs::write(
+		&script_path,
+		r#"
+		import { instance } from "@viz-js/viz";
+		let dot = "";
+		for await (const chunk of process.stdin) dot += chunk;
+		const viz = await instance();
+		process.stdout.write(viz.renderString(dot, { format: "svg" }));
+		"#
+		.trim(),
+	)?;
+
+	let mut child = Command::new("node")
+		.arg(&script_path)
+		.stdin(std::process::Stdio::piped())
+		.stdout(std::process::Stdio::piped())
+		.stderr(std::process::Stdio::piped())
+		.spawn()
+		.with_context(|| "Failed to spawn `node`. Is Node.js installed?")?;
+
+	// Write DOT content to stdin then close the pipe
+	child
+		.stdin
+		.take()
+		.context("Failed to get stdin handle for node process")?
+		.write_all(dot.as_bytes())
+		.with_context(|| "Failed to write DOT content to stdin")?;
+
+	let output = child.wait_with_output().with_context(|| "Failed to wait for `node` process")?;
+
+	// Clean up the temp script (node_modules is intentionally kept as a cache)
+	let _ = fs::remove_file(&script_path);
+
+	if !output.status.success() {
+		let stderr = String::from_utf8_lossy(&output.stderr);
+		anyhow::bail!("DOT to SVG conversion failed (exit code {:?}):\n{}", output.status.code(), stderr);
+	}
+
+	String::from_utf8(output.stdout).with_context(|| "SVG output was not valid UTF-8")
+}
+
 fn generate_dot(crates: &[CrateInfo]) -> String {
 	let mut out = String::new();
 	out.push_str("digraph CrateHierarchy {\n");