split out a source detection method

Author: ealmloff

Cargo.lock

@@ -44,7 +44,7 @@ rustdoc-args = ["--cfg", "docsrs"]
 default = ["macro"]
 macro = ["dep:dioxus-code-macro", "dioxus-code-macro/lang-rust"]
 runtime = ["arborium/lang-rust", "dep:arborium-tree-sitter"]
-detection = ["runtime", "dep:betlang"]
+detection = ["runtime", "dep:betlang", "dioxus-code-macro?/detection"]
 all-languages = [
     "runtime",
     "arborium/all-languages",

Cargo.toml

@@ -15,6 +15,7 @@ proc-macro = true
 
 [features]
 default = []
+detection = ["dep:betlang"]
 all-languages = ["arborium/all-languages"]
 lang-rust = ["arborium/lang-rust"]
 lang-ada = ["arborium/lang-ada"]
@@ -123,6 +124,7 @@ lang-zsh = ["arborium/lang-zsh"]
 [dependencies]
 arborium = { version = "2.16.0", default-features = false }
 arborium-theme = "2.16.0"
+betlang = { version = "0.1.0", optional = true }
 macro-string = "0.1.4"
 proc-macro-crate = "3.5.0"
 proc-macro2 = "1.0.103"

dioxus-code-macro/Cargo.toml

@@ -50,6 +50,18 @@ let _tree = code!(
 );
 ```
 
+## Inline source detection
+
+With the `detection` feature enabled, `code_str!` can infer the language from
+inline source contents when no explicit language is provided. The detected
+language still needs its matching `lang-*` feature or `all-languages` enabled.
+
+```rust
+use dioxus_code::code_str;
+
+let _tree = code_str!("fn main() { println!(\"hi\"); }");
+```
+
 ## License
 
 MIT.

dioxus-code-macro/README.md

@@ -40,9 +40,10 @@ pub fn code(input: TokenStream) -> TokenStream {
 ///
 /// Parses a string literal containing source code with [`arborium`] and
 /// expands to the resulting span tree. Pass the source as a string literal,
-/// `concat!(...)`, `include_str!(...)`, or `env!(...)`. The language must be
-/// supplied via [`CodeOptions::builder`] with [`CodeOptions::with_language`]
-/// since there is no file extension to infer from.
+/// `concat!(...)`, `include_str!(...)`, or `env!(...)`. Pass
+/// [`CodeOptions::builder`] with [`CodeOptions::with_language`] to name the
+/// language explicitly; otherwise, with the macro crate's `detection` feature
+/// enabled, the language is inferred from the source contents.
 ///
 /// To highlight a file on disk instead, use [`code!`].
 ///
@@ -106,7 +107,7 @@ fn parse_string_and_options(
     Ok((value, options))
 }
 
-fn try_extract_language(expr: &Expr) -> Option<String> {
+fn try_extract_language(expr: &Expr) -> Option<LanguageSpec> {
     match expr {
         Expr::Group(group) => try_extract_language(&group.expr),
         Expr::Paren(paren) => try_extract_language(&paren.expr),
@@ -123,15 +124,15 @@ fn try_extract_language(expr: &Expr) -> Option<String> {
     }
 }
 
-fn try_parse_language_arg(expr: &Expr) -> Option<String> {
+fn try_parse_language_arg(expr: &Expr) -> Option<LanguageSpec> {
     match expr {
         Expr::Group(group) => try_parse_language_arg(&group.expr),
         Expr::Paren(paren) => try_parse_language_arg(&paren.expr),
         Expr::Call(call) if is_some_call(call) && call.args.len() == 1 => {
             try_parse_language_arg(call.args.first().unwrap())
         }
         Expr::Path(path) if is_none_path(path) => None,
-        Expr::Path(path) => language_slug_from_path(path).map(str::to_string),
+        Expr::Path(path) => language_spec_from_path(path),
         _ => None,
     }
 }
@@ -153,6 +154,12 @@ fn is_none_path(path: &syn::ExprPath) -> bool {
         .is_some_and(|segment| segment.ident == "None")
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+struct LanguageSpec {
+    variant: &'static str,
+    slug: &'static str,
+}
+
 const LANGUAGE_VARIANTS: &[(&str, &str)] = &[
     ("Rust", "rust"),
     ("Ada", "ada"),
@@ -259,19 +266,97 @@ const LANGUAGE_VARIANTS: &[(&str, &str)] = &[
     ("Zsh", "zsh"),
 ];
 
-fn language_slug_from_path(path: &syn::ExprPath) -> Option<&'static str> {
+fn language_spec_from_path(path: &syn::ExprPath) -> Option<LanguageSpec> {
     let variant = path.path.segments.last()?.ident.to_string();
+    language_spec_for_variant(&variant)
+}
+
+fn language_spec_for_variant(variant: &str) -> Option<LanguageSpec> {
     LANGUAGE_VARIANTS
         .iter()
         .find(|(name, _)| *name == variant)
-        .map(|(_, slug)| *slug)
+        .map(|(variant, slug)| LanguageSpec {
+            variant: *variant,
+            slug: *slug,
+        })
 }
 
-fn language_variant_for_slug(slug: &str) -> Option<&'static str> {
+fn language_spec_for_slug(slug: &str) -> Option<LanguageSpec> {
     LANGUAGE_VARIANTS
         .iter()
         .find(|(_, s)| *s == slug)
-        .map(|(name, _)| *name)
+        .map(|(variant, slug)| LanguageSpec {
+            variant: *variant,
+            slug: *slug,
+        })
+}
+
+#[cfg(feature = "detection")]
+fn detect_source_language(source: &str) -> Option<LanguageSpec> {
+    betlang::detect(source)
+        .language()
+        .and_then(language_spec_for_betlang)
+}
+
+#[cfg(not(feature = "detection"))]
+fn detect_source_language(_source: &str) -> Option<LanguageSpec> {
+    None
+}
+
+#[cfg(feature = "detection")]
+fn language_spec_for_betlang(language: betlang::Language) -> Option<LanguageSpec> {
+    match language {
+        betlang::Language::Asm => language_spec_for_variant("Asm"),
+        betlang::Language::Batch => language_spec_for_variant("Batch"),
+        betlang::Language::C => language_spec_for_variant("C"),
+        betlang::Language::Clojure => language_spec_for_variant("Clojure"),
+        betlang::Language::CMake => language_spec_for_variant("CMake"),
+        betlang::Language::Cobol => language_spec_for_variant("Cobol"),
+        betlang::Language::Cpp => language_spec_for_variant("Cpp"),
+        betlang::Language::Cs => language_spec_for_variant("CSharp"),
+        betlang::Language::Css => language_spec_for_variant("Css"),
+        betlang::Language::Dart => language_spec_for_variant("Dart"),
+        betlang::Language::Dockerfile => language_spec_for_variant("Dockerfile"),
+        betlang::Language::Elixir => language_spec_for_variant("Elixir"),
+        betlang::Language::Erlang => language_spec_for_variant("Erlang"),
+        betlang::Language::Gemfile | betlang::Language::Gemspec | betlang::Language::Ruby => {
+            language_spec_for_variant("Ruby")
+        }
+        betlang::Language::Go => language_spec_for_variant("Go"),
+        betlang::Language::Gradle | betlang::Language::Groovy => {
+            language_spec_for_variant("Groovy")
+        }
+        betlang::Language::Haskell => language_spec_for_variant("Haskell"),
+        betlang::Language::Html => language_spec_for_variant("Html"),
+        betlang::Language::Ini => language_spec_for_variant("Ini"),
+        betlang::Language::Java => language_spec_for_variant("Java"),
+        betlang::Language::JavaScript => language_spec_for_variant("JavaScript"),
+        betlang::Language::Json => language_spec_for_variant("Json"),
+        betlang::Language::Julia => language_spec_for_variant("Julia"),
+        betlang::Language::Kotlin => language_spec_for_variant("Kotlin"),
+        betlang::Language::Lisp => language_spec_for_variant("CommonLisp"),
+        betlang::Language::Lua => language_spec_for_variant("Lua"),
+        betlang::Language::Markdown => language_spec_for_variant("Markdown"),
+        betlang::Language::ObjectiveC => language_spec_for_variant("ObjectiveC"),
+        betlang::Language::Ocaml => language_spec_for_variant("OCaml"),
+        betlang::Language::Perl => language_spec_for_variant("Perl"),
+        betlang::Language::Php => language_spec_for_variant("Php"),
+        betlang::Language::Powershell => language_spec_for_variant("PowerShell"),
+        betlang::Language::Python => language_spec_for_variant("Python"),
+        betlang::Language::R => language_spec_for_variant("R"),
+        betlang::Language::Rust => language_spec_for_variant("Rust"),
+        betlang::Language::Scala => language_spec_for_variant("Scala"),
+        betlang::Language::Shell => language_spec_for_variant("Bash"),
+        betlang::Language::Sql => language_spec_for_variant("Sql"),
+        betlang::Language::Swift => language_spec_for_variant("Swift"),
+        betlang::Language::Toml => language_spec_for_variant("Toml"),
+        betlang::Language::TypeScript => language_spec_for_variant("TypeScript"),
+        betlang::Language::Vba => language_spec_for_variant("VisualBasic"),
+        betlang::Language::Verilog => language_spec_for_variant("Verilog"),
+        betlang::Language::Xml => language_spec_for_variant("Xml"),
+        betlang::Language::Yaml => language_spec_for_variant("Yaml"),
+        _ => None,
+    }
 }
 
 fn expand_code(input: CodeInput) -> syn::Result<TokenStream2> {
@@ -300,18 +385,23 @@ fn expand_shared(
     let crate_path = dioxus_code_crate_path()?;
     let options_check = options_check_tokens(&crate_path, options.as_ref());
 
-    let Some(language) = options.as_ref().and_then(try_extract_language).or_else(|| {
-        origin_path
-            .as_ref()
-            .and_then(|path| arborium::detect_language(&path.to_string_lossy()).map(str::to_string))
-    }) else {
+    let Some(language) = options
+        .as_ref()
+        .and_then(try_extract_language)
+        .or_else(|| {
+            origin_path.as_ref().and_then(|path| {
+                arborium::detect_language(&path.to_string_lossy()).and_then(language_spec_for_slug)
+            })
+        })
+        .or_else(|| detect_source_language(&source))
+    else {
         let message = match origin_path.as_ref() {
             Some(path) => format!(
-                "could not detect language for `{}`; pass `CodeOptions::builder().with_language(Language::Rust)`",
+                "could not detect language for `{}`; pass `CodeOptions::builder().with_language(Language::Rust)` or enable `detection` with the matching `lang-*` feature or `all-languages`",
                 path.display()
             ),
             None => String::from(
-                "could not determine language for `code_str!`; pass `CodeOptions::builder().with_language(Language::Rust)`",
+                "could not determine language for `code_str!`; pass `CodeOptions::builder().with_language(Language::Rust)` or enable `detection` with the matching `lang-*` feature or `all-languages`",
             ),
         };
         return Ok(quote! {{
@@ -322,17 +412,10 @@ fn expand_shared(
 
     let mut highlighter = arborium::Highlighter::new();
     let spans = highlighter
-        .highlight_spans(&language, &source)
+        .highlight_spans(language.slug, &source)
         .map_err(|error| syn::Error::new(Span::call_site(), error.to_string()))?;
 
-    let Some(variant) = language_variant_for_slug(&language) else {
-        let message = format!("language `{language}` has no `Language` variant");
-        return Ok(quote! {{
-            #options_check
-            compile_error!(#message);
-        }});
-    };
-    let variant_ident = Ident::new(variant, Span::call_site());
+    let variant_ident = Ident::new(language.variant, Span::call_site());
 
     let source_expr = match origin_path {
         Some(path) => {
@@ -472,37 +555,59 @@ fn resolve_manifest_path(manifest_dir: &Path, path: &str) -> PathBuf {
 mod tests {
     use super::*;
 
-    fn language(expr: &str) -> Option<String> {
+    fn language(expr: &str) -> Option<LanguageSpec> {
         let expr = syn::parse_str::<Expr>(expr).unwrap();
         try_extract_language(&expr)
     }
 
+    fn slug(expr: &str) -> Option<&'static str> {
+        language(expr).map(|language| language.slug)
+    }
+
     #[test]
     fn extracts_language_variant_options() {
         assert_eq!(
-            language("CodeOptions::builder().with_language(Language::Rust)").as_deref(),
+            slug("CodeOptions::builder().with_language(Language::Rust)"),
             Some("rust"),
         );
         assert_eq!(
-            language("CodeOptions::builder().with_language(Some(Language::Rust))").as_deref(),
+            slug("CodeOptions::builder().with_language(Some(Language::Rust))"),
             Some("rust"),
         );
     }
 
     #[test]
     fn extracts_none_language_option() {
-        assert_eq!(
-            language("CodeOptions::builder().with_language(None)").as_deref(),
-            None,
-        );
+        assert_eq!(slug("CodeOptions::builder().with_language(None)"), None,);
     }
 
     #[test]
     fn unknown_method_chains_fall_back_silently() {
-        assert_eq!(language("CodeOptions::builder()").as_deref(), None);
+        assert_eq!(slug("CodeOptions::builder()"), None);
         assert_eq!(
-            language("CodeOptions::builder().with_themes(Language::Rust)").as_deref(),
+            slug("CodeOptions::builder().with_themes(Language::Rust)"),
             None,
         );
     }
+
+    #[cfg(feature = "detection")]
+    #[test]
+    fn maps_betlang_languages_directly() {
+        macro_rules! assert_betlang_mapping {
+            ($betlang:expr, $variant:literal, $slug:literal) => {
+                assert_eq!(
+                    language_spec_for_betlang($betlang),
+                    Some(LanguageSpec {
+                        variant: $variant,
+                        slug: $slug,
+                    })
+                );
+            };
+        }
+
+        assert_betlang_mapping!(betlang::Language::Cs, "CSharp", "c-sharp");
+        assert_betlang_mapping!(betlang::Language::Lisp, "CommonLisp", "commonlisp");
+        assert_betlang_mapping!(betlang::Language::Shell, "Bash", "bash");
+        assert_betlang_mapping!(betlang::Language::Vba, "VisualBasic", "vb");
+    }
 }

dioxus-code-macro/src/lib.rs

@@ -864,7 +864,7 @@ fn collect_spans(
 fn resolve_language(language: Language, _source: &str) -> Result<Language, HighlightError> {
     #[cfg(feature = "detection")]
     if language == Language::Auto {
-        return Language::detect(_source).ok_or(HighlightError::LanguageDetectionFailed);
+        return Language::detect_source(_source).ok_or(HighlightError::LanguageDetectionFailed);
     }
 
     Ok(language)
@@ -1534,6 +1534,61 @@ mod buffer_tests {
         );
     }
 
+    #[cfg(all(feature = "detection", feature = "lang-markdown"))]
+    #[test]
+    fn auto_detection_uses_source_content_not_path_suffix() {
+        let source = r#"# Betlang Fixture
+
+This Markdown file gives the detector a document-shaped source example.
+
+## Languages
+
+- Rust
+- Python
+- JavaScript
+
+```rust
+fn main() {
+    println!("hello");
+}
+```
+
+The surrounding prose and headings should make this look like Markdown rather
+than the fenced source language.
+
+main.rs"#;
+        let buffer = Buffer::new(Language::Auto, source).unwrap();
+
+        assert_eq!(buffer.language(), Language::Markdown);
+    }
+
+    #[cfg(all(feature = "detection", feature = "lang-c-sharp"))]
+    #[test]
+    fn auto_detection_maps_betlang_csharp_variant() {
+        let source = r#"using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace Betlang.Fixtures
+{
+    public sealed class Program
+    {
+        public static void Main(string[] args)
+        {
+            var names = new List<string> { "Ada", "Grace", "Linus" };
+            foreach (var name in names.Where(value => value.Length > 0))
+            {
+                Console.WriteLine($"Hello, {name}");
+            }
+        }
+    }
+}
+"#;
+        let buffer = Buffer::new(Language::Auto, source).unwrap();
+
+        assert_eq!(buffer.language(), Language::CSharp);
+    }
+
     #[test]
     fn edit_with_explicit_source_edit() {
         let mut buffer = Buffer::new(Language::Rust, "fn main() { let x = 1; }").unwrap();