feat: expose EstimateTokensFast/WithEncoding/ForModel at the top level

Add three top-level tok wrappers around the existing internal/core
estimators:

  - tok.EstimateTokensFast(text)         — fast heuristic, no BPE
  - tok.EstimateTokensWithEncoding(text, encoding)
                                          — BPE with explicit encoding
                                            (e.g. cl100k_base, o200k_base)
  - tok.EstimateTokensForModel(text, model)
                                          — BPE for a model family
                                            (e.g. gpt-4o, claude-3, gemini-2.5)

These were available internally (core.EstimateTokensFast,
core.EstimateTokensWithEncoding, core.EstimateTokensForModel) but not
exposed at the public tok API. The rtk-style API surface expects
these to be available for callers that want to track token counts
before compression (e.g. to decide whether compression is worth
doing at all).

Source: rtk-ai/rtk estimate_tokens module. The functions are direct
wrappers; no new logic.

Tests: 2 cases covering all variants and empty-input behavior.

Author: Patel230

estimate_test.go

@@ -0,0 +1,43 @@
+package tok_test
+
+import (
+	"testing"
+
+	"github.com/GrayCodeAI/tok"
+)
+
+func TestEstimateTokens_AllVariants(t *testing.T) {
+	in := "The quick brown fox jumps over the lazy dog"
+	tests := []struct {
+		name string
+		fn   func() int
+	}{
+		{"EstimateTokens", func() int { return tok.EstimateTokens(in) }},
+		{"EstimateTokensFast", func() int { return tok.EstimateTokensFast(in) }},
+		{"EstimateTokensPrecise", func() int { return tok.EstimateTokensPrecise(in) }},
+		{"EstimateTokensWithEncoding_cl100k", func() int { return tok.EstimateTokensWithEncoding(in, "cl100k_base") }},
+		{"EstimateTokensWithEncoding_o200k", func() int { return tok.EstimateTokensWithEncoding(in, "o200k_base") }},
+		{"EstimateTokensForModel_gpt4o", func() int { return tok.EstimateTokensForModel(in, "gpt-4o") }},
+		{"EstimateTokensForModel_claude", func() int { return tok.EstimateTokensForModel(in, "claude-3") }},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			n := tc.fn()
+			if n <= 0 {
+				t.Errorf("expected positive token count, got %d", n)
+			}
+		})
+	}
+}
+
+func TestEstimateTokens_Empty(t *testing.T) {
+	if n := tok.EstimateTokens(""); n != 0 {
+		t.Errorf("expected 0 for empty string, got %d", n)
+	}
+	if n := tok.EstimateTokensFast(""); n != 0 {
+		t.Errorf("expected 0 for empty string, got %d", n)
+	}
+	if n := tok.EstimateTokensPrecise(""); n != 0 {
+		t.Errorf("expected 0 for empty string, got %d", n)
+	}
+}

tok.go

@@ -39,11 +39,32 @@ func EstimateTokens(text string) int {
 	return core.EstimateTokens(text)
 }
 
+// EstimateTokensFast provides a fast estimate without BPE.
+// Use this when exact count isn't critical (e.g. internal budget
+// checks before doing precise BPE work).
+func EstimateTokensFast(text string) int {
+	return core.EstimateTokensFast(text)
+}
+
 // EstimateTokensPrecise uses BPE tokenization (slower, more accurate).
 func EstimateTokensPrecise(text string) int {
 	return core.EstimateTokensPrecise(text)
 }
 
+// EstimateTokensWithEncoding uses BPE for a specific encoding
+// (e.g. "cl100k_base", "o200k_base", "p50k_base"). The encoding
+// must be one of the names accepted by the BPE tokenizer.
+func EstimateTokensWithEncoding(text string, encoding string) int {
+	return core.EstimateTokensWithEncoding(text, encoding)
+}
+
+// EstimateTokensForModel uses BPE for the encoding associated with
+// a specific model (e.g. "gpt-4o", "claude-3", "gemini-2.5"). See
+// WithModel for the supported model prefixes.
+func EstimateTokensForModel(text string, model string) int {
+	return core.EstimateTokensForModel(text, model)
+}
+
 // WarmupTokenizer pre-initializes the BPE tokenizer in the background.
 // Call at application startup to avoid latency on the first Compress call.
 func WarmupTokenizer() {