Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion crates/terraphim_automata/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -55,5 +55,4 @@ medical = ["daachorse", "zstd", "anyhow"]
[dev-dependencies]
criterion = "0.8"
tempfile = "3.27"
lazy_static = "1.4.0"
tokio = { version = "1", features = ["io-util", "time","macros","rt","rt-multi-thread"] }
8 changes: 4 additions & 4 deletions crates/terraphim_automata/benches/autocomplete_bench.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,12 @@ use terraphim_automata::{AutomataPath, load_thesaurus};
use tokio::runtime::Runtime;

#[cfg(feature = "tokio-runtime")]
lazy_static::lazy_static! {
static ref TOKIO_RUNTIME: Runtime = tokio::runtime::Builder::new_current_thread()
static TOKIO_RUNTIME: std::sync::LazyLock<Runtime> = std::sync::LazyLock::new(|| {
tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap();
}
.unwrap()
});

// We can use this `block_on` function to run async code in the benchmarks when tokio is available
#[cfg(feature = "tokio-runtime")]
Expand Down
1 change: 0 additions & 1 deletion crates/terraphim_multi_agent/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@ rand = "0.9"
# tiktoken-rs = "0.6" # Now handled by rust-genai
tracing = "0.1"
regex = "1.12"
lazy_static = "1.4"

# Terraphim dependencies
terraphim_types = { path = "../terraphim_types" }
Expand Down
31 changes: 19 additions & 12 deletions crates/terraphim_multi_agent/src/prompt_sanitizer.rs
Original file line number Diff line number Diff line change
@@ -1,28 +1,32 @@
use lazy_static::lazy_static;
use regex::Regex;
use std::collections::HashSet;
use std::sync::LazyLock;
use tracing::warn;

const MAX_PROMPT_LENGTH: usize = 10_000;

lazy_static! {
static ref SUSPICIOUS_PATTERNS: Vec<Regex> = vec![
static SUSPICIOUS_PATTERNS: LazyLock<Vec<Regex>> = LazyLock::new(|| {
vec![
Regex::new(r"(?i)ignore\s+\s*(previous|above|prior)\s+\s*(instructions|prompts?)").unwrap(),
Regex::new(r"(?i)disregard\s+\s*(previous|above|all)\s+\s*(instructions|prompts?)").unwrap(),
Regex::new(r"(?i)disregard\s+\s*(previous|above|all)\s+\s*(instructions|prompts?)")
.unwrap(),
Regex::new(r"(?i)system\s*:\s*you\s+\s*are\s+\s*now").unwrap(),
Regex::new(r"(?i)<\|?im_start\|?>").unwrap(),
Regex::new(r"(?i)<\|?im_end\|?>").unwrap(),
Regex::new(r"(?i)###\s*instruction").unwrap(),
Regex::new(r"(?i)forget\s+\s*(everything|all|previous)").unwrap(),
Regex::new(r"\x00").unwrap(),
Regex::new(r"[\x01-\x08\x0B-\x0C\x0E-\x1F\x7F]").unwrap(),
];
static ref CONTROL_CHAR_PATTERN: Regex =
Regex::new(r"[\x00-\x08\x0B-\x0C\x0E-\x1F\x7F]").unwrap();
]
});

// Unicode special characters that can be used for obfuscation or attacks.
// HashSet for O(1) per-character lookup.
static ref UNICODE_SPECIAL_CHARS: HashSet<char> = [
static CONTROL_CHAR_PATTERN: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"[\x00-\x08\x0B-\x0C\x0E-\x1F\x7F]").unwrap());

// Unicode special characters that can be used for obfuscation or attacks.
// HashSet for O(1) per-character lookup.
static UNICODE_SPECIAL_CHARS: LazyLock<HashSet<char>> = LazyLock::new(|| {
[
'\u{202E}', // RIGHT-TO-LEFT OVERRIDE
'\u{202D}', // LEFT-TO-RIGHT OVERRIDE
'\u{202C}', // POP DIRECTIONAL FORMATTING
Expand All @@ -43,8 +47,11 @@ lazy_static! {
'\u{206D}', // ACTIVATE ARABIC FORM SHAPING
'\u{206E}', // NATIONAL DIGIT SHAPES
'\u{206F}', // NOMINAL DIGIT SHAPES
].iter().copied().collect();
}
]
.iter()
.copied()
.collect()
});

#[derive(Debug, Clone)]
pub struct SanitizedPrompt {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -129,8 +129,8 @@ async fn test_sanitizer_thread_safety() {
}

#[test]
fn test_lazy_static_thread_safety() {
// Verify lazy_static patterns are initialized safely
fn test_lazy_lock_thread_safety() {
// Verify LazyLock patterns are initialized safely
// This tests the regex compilation in SUSPICIOUS_PATTERNS
use std::thread;

Expand All @@ -150,7 +150,7 @@ fn test_lazy_static_thread_safety() {

#[tokio::test]
async fn test_unicode_chars_vec_concurrent_access() {
// Test concurrent access to UNICODE_SPECIAL_CHARS lazy_static
// Test concurrent access to UNICODE_SPECIAL_CHARS LazyLock
let mut handles = vec![];

for _ in 0..50 {
Expand Down
1 change: 0 additions & 1 deletion crates/terraphim_rolegraph/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ ahash = { version = "0.8.3", features = ["serde"] }
aho-corasick = "1.0.2"
anyhow = { version = "1.0", optional = true }
itertools = "0.14.0"
lazy_static = "1.4.0"
log = "0.4.29"
memoize = "0.5.1"
regex = "1.12.3"
Expand Down
5 changes: 2 additions & 3 deletions crates/terraphim_rolegraph/benches/throughput.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,8 @@ use terraphim_rolegraph::input::TEST_CORPUS;
use terraphim_rolegraph::split_paragraphs;
use terraphim_types::{Document, DocumentType, Thesaurus};

lazy_static::lazy_static! {
static ref TOKIO_RUNTIME: Runtime = Runtime::new().unwrap();
}
static TOKIO_RUNTIME: std::sync::LazyLock<Runtime> =
std::sync::LazyLock::new(|| Runtime::new().unwrap());

// We can use this `block_on` function to run async code in the benchmarks
// without having to use `async fn`, which is not supported by the `criterion` library.
Expand Down
7 changes: 2 additions & 5 deletions crates/terraphim_rolegraph/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1249,11 +1249,8 @@ impl From<RoleGraph> for RoleGraphSync {
}
}

#[macro_use]
extern crate lazy_static;
lazy_static! {
static ref RE: Regex = Regex::new(r"[?!|]\s+").unwrap();
}
static RE: std::sync::LazyLock<Regex> =
std::sync::LazyLock::new(|| Regex::new(r"[?!|]\s+").unwrap());

pub fn split_paragraphs(paragraphs: &str) -> Vec<&str> {
let sentences = UnicodeSegmentation::split_sentence_bounds(paragraphs);
Expand Down
Loading