Initial commit: DataFog 2.0 Rust monorepo

Rust-powered PII detection and anonymization with Python (PyO3) and WASM bindings.

- datafog-core: regex engine (7 patterns), anonymizer (redact/replace/hash),
  cascade orchestrator, chunking, optional NER via gline-rs + ONNX Runtime
- datafog-python: PyO3 bindings with maturin build, feature-gated NER support,
  model auto-download, runtime feature detection
- datafog-wasm: wasm-bindgen bindings (regex-only) with demo page
- 106 Rust tests, 40 Python tests, criterion benchmarks, GitHub Actions CI

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

Author: sidmohan0

.github/workflows/ci.yml

@@ -0,0 +1,82 @@
+name: CI
+
+on:
+  push:
+    branches: [main, dev]
+  pull_request:
+    branches: [main, dev]
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  rust-test:
+    name: Rust Tests
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          components: clippy, rustfmt
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Check formatting
+        run: cargo fmt --all -- --check
+
+      - name: Clippy
+        run: cargo clippy --workspace -- -D warnings
+
+      - name: Test (default features)
+        run: cargo test --workspace
+
+  python-build:
+    name: Python Build & Test
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: pip install maturin pytest
+
+      - name: Build and install (editable)
+        run: maturin develop
+
+      - name: Run Python tests
+        run: pytest -v
+
+  wasm-build:
+    name: WASM Build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          targets: wasm32-unknown-unknown
+
+      - name: Install wasm-pack
+        run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
+
+      - name: Build WASM package
+        run: wasm-pack build crates/datafog-wasm --target web
+
+  benchmarks:
+    name: Benchmarks
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Run benchmarks
+        run: cargo bench --package datafog-core -- --output-format bencher

.gitignore

@@ -0,0 +1,34 @@
+# Rust
+target/
+Cargo.lock
+
+# Python
+.venv/
+*.pyc
+__pycache__/
+*.egg-info/
+.eggs/
+dist/
+build/
+*.so
+*.dylib
+*.pyd
+*.whl
+
+# WASM
+pkg/
+
+# OS
+.DS_Store
+Thumbs.db
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+
+# Testing / benchmarks
+.pytest_cache/
+htmlcov/
+*.prof

Cargo.toml

@@ -0,0 +1,21 @@
+[workspace]
+resolver = "2"
+members = [
+    "crates/datafog-core",
+    "crates/datafog-python",
+    "crates/datafog-wasm",
+]
+
+[workspace.package]
+version = "0.1.0"
+edition = "2021"
+license = "Apache-2.0"
+authors = ["DataFog <hello@datafog.ai>"]
+repository = "https://github.com/DataFog/datafog"
+
+[workspace.dependencies]
+datafog-core = { path = "crates/datafog-core" }
+regex = "1"
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+thiserror = "2"

README.md

@@ -0,0 +1,173 @@
+# DataFog
+
+Fast PII detection and anonymization, built in Rust with Python and WASM bindings.
+
+DataFog detects structured PII (emails, phone numbers, SSNs, credit cards, IPs, dates of birth, ZIP codes) using compiled regex patterns, and optionally detects soft PII (names, organizations, addresses) using a GLiNER ONNX model via NER. The regex engine runs in microseconds per kilobyte with zero external dependencies.
+
+## Prerequisites
+
+- **Rust** (stable) — install via [rustup](https://rustup.rs/)
+- **Python 3.9+** — for the Python bindings
+- **maturin** — for building the Rust-to-Python bridge
+
+## Quick start (Python)
+
+```bash
+git clone https://github.com/DataFog/datafog.git
+cd datafog
+
+python3 -m venv .venv
+source .venv/bin/activate       # Windows: .venv\Scripts\activate
+pip install maturin pytest
+
+maturin develop                 # builds Rust, installs editable Python package
+pytest -v                       # runs the test suite
+```
+
+Then in Python:
+
+```python
+from datafog import DataFog, detect, anonymize_text
+
+# Detect PII
+entities = detect("Contact john@example.com or call 555-123-4567")
+# [{"type": "EMAIL", "value": "john@example.com", "start": 8, "end": 24, "score": 1.0},
+#  {"type": "PHONE", "value": "555-123-4567", "start": 33, "end": 45, "score": 1.0}]
+
+# Anonymize
+clean = anonymize_text("SSN is 123-45-6789", method="redact")
+# "SSN is [REDACTED]"
+
+# Class API with batch support
+fog = DataFog()
+results = fog.detect_batch(["john@test.com", "555-123-4567", "no pii here"])
+```
+
+## Quick start (Rust)
+
+Add to your `Cargo.toml`:
+
+```toml
+[dependencies]
+datafog-core = "0.1"
+```
+
+```rust
+use datafog_core::DataFog;
+use datafog_core::anonymizer::AnonymizeMethod;
+
+let fog = DataFog::new();
+let result = fog.detect("Contact john@example.com");
+println!("{:?}", result.spans);
+
+let anon = fog.anonymize("SSN is 123-45-6789", AnonymizeMethod::Redact);
+assert_eq!(anon.text, "SSN is [REDACTED]");
+```
+
+## NER engine (optional)
+
+The NER engine uses a [GLiNER](https://github.com/urchade/GLiNER) ONNX model to detect soft PII that regex can't catch: person names, organizations, locations, addresses, and more. It runs both regex and NER, then merges the results.
+
+Building with NER pulls in `gline-rs` + ONNX Runtime (~50 MB binary size increase).
+
+```bash
+# Build the Python package with NER + model auto-download
+maturin develop --features full
+```
+
+```python
+from datafog import DataFog, has_ner_support
+
+if has_ner_support():
+    fog = DataFog(engine="ner")  # downloads ~50 MB model on first use
+    entities = fog.detect("John Smith works at Acme Corp in Paris")
+    # detects PERSON, ORGANIZATION, LOCATION in addition to regex entities
+```
+
+You can also point to a local model directory:
+
+```python
+fog = DataFog(engine="ner", model="/path/to/model/dir")
+```
+
+The model directory must contain `tokenizer.json` and `model.onnx`.
+
+### Entity types
+
+| Engine | Entity types |
+|--------|-------------|
+| Regex | `EMAIL`, `PHONE`, `SSN`, `CREDIT_CARD`, `IP_ADDRESS`, `DOB`, `ZIP` |
+| NER | `PERSON`, `ORGANIZATION`, `LOCATION`, `ADDRESS`, `MEDICAL_RECORD_NUMBER`, `ACCOUNT_NUMBER`, `LICENSE_NUMBER`, `PASSPORT_NUMBER`, `URL` |
+
+When both engines are active, all entity types are available.
+
+## Anonymization methods
+
+```python
+from datafog import anonymize_text
+
+anonymize_text("Email: john@test.com", method="redact")    # "Email: [REDACTED]"
+anonymize_text("Email: john@test.com", method="replace")   # "Email: [EMAIL_1]"
+anonymize_text("Email: john@test.com", method="hash")      # "Email: a1b2c3d4..."
+```
+
+Available methods: `redact`, `replace`, `hash` (SHA-256), `hash_md5`, `hash_sha3`.
+
+## WASM
+
+The WASM target provides regex-only detection for browser and edge environments.
+
+```bash
+# Requires wasm-pack: https://rustwasm.github.io/wasm-pack/installer/
+wasm-pack build crates/datafog-wasm --target web
+```
+
+A demo page is included at `crates/datafog-wasm/demo/index.html`.
+
+## Project structure
+
+```
+datafog/
+├── crates/
+│   ├── datafog-core/        # Pure Rust library (regex, NER, anonymizer, cascade)
+│   ├── datafog-python/      # PyO3 bindings
+│   └── datafog-wasm/        # wasm-bindgen bindings (regex-only)
+├── python/datafog/          # Python package source + type stubs
+├── tests/                   # Python integration tests
+├── pyproject.toml           # maturin build config
+└── Cargo.toml               # Workspace root
+```
+
+## Feature flags (Rust)
+
+| Flag | What it adds | Dependencies |
+|------|-------------|-------------|
+| `default` | Regex-only detection | None beyond `regex`, `serde` |
+| `ner` | GLiNER NER engine | `gline-rs`, `ort` |
+| `model-download` | Auto-download models from HuggingFace | `reqwest`, `dirs` |
+| `parallel` | Rayon-based batch parallelism | `rayon` |
+| `ner-cuda` | CUDA GPU acceleration for NER | (implies `ner`) |
+| `ner-coreml` | Apple CoreML acceleration for NER | (implies `ner`) |
+
+## Development
+
+```bash
+# Run Rust tests
+cargo test --workspace
+
+# Run clippy and check formatting
+cargo clippy --workspace -- -D warnings
+cargo fmt --all -- --check
+
+# Run benchmarks
+cargo bench --package datafog-core
+
+# Build and test Python
+source .venv/bin/activate
+maturin develop
+pytest -v
+```
+
+## License
+
+Apache 2.0

crates/datafog-core/Cargo.toml

@@ -0,0 +1,51 @@
+[package]
+name = "datafog-core"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+authors.workspace = true
+repository.workspace = true
+description = "Fast PII detection and anonymization engine"
+keywords = ["pii", "privacy", "anonymization", "ner", "regex"]
+categories = ["text-processing"]
+
+[features]
+default = []
+ner = ["dep:gline-rs", "dep:ort"]
+ner-cuda = ["ner"]
+ner-coreml = ["ner"]
+model-download = ["dep:reqwest", "dep:tokio", "dep:dirs"]
+parallel = ["dep:rayon"]
+
+[dependencies]
+regex = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+thiserror = { workspace = true }
+md-5 = "0.10"
+sha2 = "0.10"
+sha3 = "0.10"
+rand = "0.8"
+
+# Optional: NER
+gline-rs = { version = "1", optional = true }
+ort = { version = "2.0.0-rc.9", optional = true }
+
+# Optional: Model download
+reqwest = { version = "0.12", features = ["blocking"], optional = true }
+tokio = { version = "1", features = ["rt-multi-thread"], optional = true }
+dirs = { version = "5", optional = true }
+
+# Optional: Parallel processing
+rayon = { version = "1", optional = true }
+
+[dev-dependencies]
+criterion = { version = "0.5", features = ["html_reports"] }
+
+[[bench]]
+name = "regex_bench"
+harness = false
+
+[[bench]]
+name = "anonymizer_bench"
+harness = false

crates/datafog-core/benches/anonymizer_bench.rs

@@ -0,0 +1,32 @@
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use datafog_core::anonymizer::{AnonymizeMethod, HashType};
+use datafog_core::cascade::DataFog;
+
+fn bench_anonymizer(c: &mut Criterion) {
+    let fog = DataFog::new();
+
+    let sample_text = "Contact John Doe at john.doe@example.com or call (555) 123-4567. \
+                       His SSN is 123-45-6789 and credit card 4111111111111111. \
+                       He lives at 123 Main St, New York, NY 10001. \
+                       His IP address is 192.168.1.1 and his birthday is 01/01/1980.";
+
+    c.bench_function("anonymize_redact", |b| {
+        b.iter(|| fog.anonymize(black_box(sample_text), AnonymizeMethod::Redact))
+    });
+
+    c.bench_function("anonymize_replace", |b| {
+        b.iter(|| fog.anonymize(black_box(sample_text), AnonymizeMethod::Replace))
+    });
+
+    c.bench_function("anonymize_hash_sha256", |b| {
+        b.iter(|| {
+            fog.anonymize(
+                black_box(sample_text),
+                AnonymizeMethod::Hash(HashType::Sha256),
+            )
+        })
+    });
+}
+
+criterion_group!(benches, bench_anonymizer);
+criterion_main!(benches);