From 6551e35f52088090b333527eea3307ac67c045f2 Mon Sep 17 00:00:00 2001 From: Anthony DePasquale Date: Thu, 28 May 2026 22:58:34 +0200 Subject: [PATCH 1/2] Register raise_exception in Jinja environment --- rust/src/core/template.rs | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/rust/src/core/template.rs b/rust/src/core/template.rs index bec2c67..f0759c8 100644 --- a/rust/src/core/template.rs +++ b/rust/src/core/template.rs @@ -27,6 +27,18 @@ pub(crate) fn make_environment() -> Environment<'static> { environment.add_function("strftime_now", |format: String| -> String { Local::now().format(&format).to_string() }); + // `raise_exception(message)` is a transformers Jinja extension chat templates + // call to reject invalid input. minijinja has no equivalent, so register it to + // surface the template's message instead of "unknown function raise_exception". + environment.add_function( + "raise_exception", + |message: String| -> Result { + Err(minijinja::Error::new( + minijinja::ErrorKind::InvalidOperation, + message, + )) + }, + ); environment.set_keep_trailing_newline(false); environment.set_trim_blocks(true); environment.set_lstrip_blocks(true); @@ -68,3 +80,24 @@ static GENERATION_TAG: LazyLock = fn normalize_template_source(template: &str) -> String { GENERATION_TAG.replace_all(template, "").into_owned() } + +#[cfg(test)] +mod tests { + use super::*; + + // `raise_exception` is a transformers Jinja extension chat templates use to + // reject invalid input; the rendered error must carry the template's message. + #[test] + fn raise_exception_surfaces_template_message() { + let environment = make_environment(); + let template = r#"{{ raise_exception("ids must be alphanumeric with length 9") }}"#; + let error = render(&environment, template, &JsonValue::Null).unwrap_err(); + let CoreError::ChatTemplate(message) = error else { + panic!("expected ChatTemplate error, got {error:?}"); + }; + assert!( + message.contains("alphanumeric") && message.contains("length 9"), + "message was: {message}" + ); + } +} From 6671880303aa441afe6b3fbec2f168e2b9151268 Mon Sep 17 00:00:00 2001 From: Anthony DePasquale Date: Thu, 28 May 2026 23:16:23 +0200 Subject: [PATCH 2/2] Pin Rust artifactbundle to tokenizers-rust-0.7.0 --- Package.swift | 4 ++-- rust/Pin.json | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Package.swift b/Package.swift index b2b812d..2904c7f 100644 --- a/Package.swift +++ b/Package.swift @@ -8,9 +8,9 @@ import PackageDescription // dependency consumers (both `Context.packageDirectory` and `#filePath` return // synthetic paths during dep evaluation). let tokenizersRustArtifactBundleURL = - "https://github.com/DePasqualeOrg/swift-tokenizers/releases/download/tokenizers-rust-0.6.3/TokenizersRust-0.6.3.artifactbundle.zip" + "https://github.com/DePasqualeOrg/swift-tokenizers/releases/download/tokenizers-rust-0.7.0/TokenizersRust-0.7.0.artifactbundle.zip" let tokenizersRustArtifactBundleChecksum = - "95e42afc2d1edbf2bd0209cd1e2cb202c1059900c1180942377cc3e800b52dc9" + "efb678601d645fc97cecacc8c65d9a968f0eda1664d84e2132fb6e1e8643d60c" let docsEnabled = Context.environment["TOKENIZERS_ENABLE_DOCS"] == "1" let localRustArtifactPath = Context.environment["TOKENIZERS_RUST_LOCAL_ARTIFACTBUNDLE_PATH"] diff --git a/rust/Pin.json b/rust/Pin.json index b4c437e..5604864 100644 --- a/rust/Pin.json +++ b/rust/Pin.json @@ -1,9 +1,9 @@ { "hash_schema_version": 3, - "version": "0.6.3", - "git_commit": "72a73df1a045541b6ce5bd93b65a6657507890dc", + "version": "0.7.0", + "git_commit": "6551e35f52088090b333527eea3307ac67c045f2", "rust_toolchain": "1.94.1", - "source_hash_sha256": "0118bc1b0c98e63cb61475793b124549bfa2793d34e2202ecd8649264d28c554", - "artifactbundle_url": "https://github.com/DePasqualeOrg/swift-tokenizers/releases/download/tokenizers-rust-0.6.3/TokenizersRust-0.6.3.artifactbundle.zip", - "checksum": "95e42afc2d1edbf2bd0209cd1e2cb202c1059900c1180942377cc3e800b52dc9" + "source_hash_sha256": "85cc73a419b4419bbe930d54ca04246f28d6a3d2ecd3dc7c8b7282e0845bfb7c", + "artifactbundle_url": "https://github.com/DePasqualeOrg/swift-tokenizers/releases/download/tokenizers-rust-0.7.0/TokenizersRust-0.7.0.artifactbundle.zip", + "checksum": "efb678601d645fc97cecacc8c65d9a968f0eda1664d84e2132fb6e1e8643d60c" }