diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c7420d8..99977c92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,13 +16,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Dollar-based cost savings tracking in Stats - Model-aware BPE encoding selection -### Added — Round 2 of rtk + caveman porting (2026-06-01) -- **`tok.CompressCaveman(text, intensity)`** — public Go API for the - caveman prompt-compression algorithm. Three intensity levels - (`CavemanLite`, `CavemanFull`, `CavemanUltra`) with drop-lists for +### Added — Round 2 porting (2026-06-01) +- **`tok.PromptCompress(text, intensity)`** — public Go API for the + prompt-compression prompt-compression algorithm. Three intensity levels + (`IntensityLite`, `IntensityFull`, `IntensityUltra`) with drop-lists for articles / filler / pleasantries and ~150 phrase substitutions. Auto-clarity: security / destructive segments pass through verbatim. - Returns a `CavemanStats` struct (OriginalBytes, CompressedBytes, + Returns a `PromptStats` struct (OriginalBytes, CompressedBytes, BytesSaved, PercentOff, PassThroughSegments, etc.). - **`tok.IsSensitiveFilename(path)`** — 3-layer path-based sensitive detection (exact basename, sensitive directory, name token). @@ -83,7 +83,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * add agent/model/provider attribution for token savings ([c8213f1](https://github.com/GrayCodeAI/tok/commit/c8213f1d84f8c8e0378852ae4a701588f4fb294b)) * add ALL 152 official git commands ([b48bc18](https://github.com/GrayCodeAI/tok/commit/b48bc189e54976a5280d389cc8365fc866055744)) * add all competitor features - tee recovery, cost tracking, live monitor, prompt debugger, discover, session tracking, dup detection ([dc23fda](https://github.com/GrayCodeAI/tok/commit/dc23fda6f237ebb2b529386bad7ceb3d64da3a0c)) -* add all rtk + caveman features to tok ([87b26bf](https://github.com/GrayCodeAI/tok/commit/87b26bf8b9dfcb4419e29cda1edd3758212b95d6)) +* add all rtk + prompt-compression features to tok ([87b26bf](https://github.com/GrayCodeAI/tok/commit/87b26bf8b9dfcb4419e29cda1edd3758212b95d6)) * add all RTK features and performance optimizations ([98295cd](https://github.com/GrayCodeAI/tok/commit/98295cd93d2f341b4c34b27e6e3b5919b21d9d4f)) * add beautiful real-time TUI dashboard ([01bd0d8](https://github.com/GrayCodeAI/tok/commit/01bd0d84b9de5786a486e9404f5d3d869e7844b5)) * add benchmark CI reporting and adaptive profile validation ([bb1ca2f](https://github.com/GrayCodeAI/tok/commit/bb1ca2f850343b5f591491e0f36642fca8056198)) @@ -158,9 +158,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Claude Code-style interface with open input box ([e46fde2](https://github.com/GrayCodeAI/tok/commit/e46fde2cdcacbba9d842e05899755d8e7b94dd98)) * clean chat-style CLI like Mistral Vibe ([fc2228e](https://github.com/GrayCodeAI/tok/commit/fc2228e32dff4a2b4c4cc4bea49ff5b489d3c8bb)) * **cli:** add compaction flags for Layer 11 ([f9c868c](https://github.com/GrayCodeAI/tok/commit/f9c868c987fe478109628ef9ba6baeb689f576c2)) -* close 9 world-class gaps vs rtk and caveman ([f0059eb](https://github.com/GrayCodeAI/tok/commit/f0059ebc2201cbe3c0ad5a8776b5f656e1bc8c87)) -* close last two caveman gaps ([3028726](https://github.com/GrayCodeAI/tok/commit/302872695f5482bb8a8e5c340e1613755a31dd1c)) -* close tok gaps vs rtk and caveman ([760f326](https://github.com/GrayCodeAI/tok/commit/760f3268c5de816c00e6f92657f8f19834457942)) +* close 9 world-class gaps vs rtk and prompt-compression ([f0059eb](https://github.com/GrayCodeAI/tok/commit/f0059ebc2201cbe3c0ad5a8776b5f656e1bc8c87)) +* close last two prompt-compression gaps ([3028726](https://github.com/GrayCodeAI/tok/commit/302872695f5482bb8a8e5c340e1613755a31dd1c)) +* close tok gaps vs rtk and prompt-compression ([760f326](https://github.com/GrayCodeAI/tok/commit/760f3268c5de816c00e6f92657f8f19834457942)) * complete honest competitor analysis - cloned and analyzed 15 OSS tools ([3000897](https://github.com/GrayCodeAI/tok/commit/3000897d43294cebc80fe4ad37f8c22142e2e642)) * complete integration test suite with archive fixes and MCP tools ([669293c](https://github.com/GrayCodeAI/tok/commit/669293c6fe9589696197e2dcbe9f22c5d5b71535)) * complete SIMD optimization with Go 1.26 ([a3dbc85](https://github.com/GrayCodeAI/tok/commit/a3dbc857d662ab98690cc06e90d888491bf0f0fa)) @@ -474,7 +474,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * add agent/model/provider attribution for token savings ([c8213f1](https://github.com/GrayCodeAI/tok/commit/c8213f1d84f8c8e0378852ae4a701588f4fb294b)) * add ALL 152 official git commands ([b48bc18](https://github.com/GrayCodeAI/tok/commit/b48bc189e54976a5280d389cc8365fc866055744)) * add all competitor features - tee recovery, cost tracking, live monitor, prompt debugger, discover, session tracking, dup detection ([dc23fda](https://github.com/GrayCodeAI/tok/commit/dc23fda6f237ebb2b529386bad7ceb3d64da3a0c)) -* add all rtk + caveman features to tok ([87b26bf](https://github.com/GrayCodeAI/tok/commit/87b26bf8b9dfcb4419e29cda1edd3758212b95d6)) +* add all rtk + prompt-compression features to tok ([87b26bf](https://github.com/GrayCodeAI/tok/commit/87b26bf8b9dfcb4419e29cda1edd3758212b95d6)) * add all RTK features and performance optimizations ([98295cd](https://github.com/GrayCodeAI/tok/commit/98295cd93d2f341b4c34b27e6e3b5919b21d9d4f)) * add beautiful real-time TUI dashboard ([01bd0d8](https://github.com/GrayCodeAI/tok/commit/01bd0d84b9de5786a486e9404f5d3d869e7844b5)) * add benchmark CI reporting and adaptive profile validation ([bb1ca2f](https://github.com/GrayCodeAI/tok/commit/bb1ca2f850343b5f591491e0f36642fca8056198)) @@ -548,9 +548,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Claude Code-style interface with open input box ([e46fde2](https://github.com/GrayCodeAI/tok/commit/e46fde2cdcacbba9d842e05899755d8e7b94dd98)) * clean chat-style CLI like Mistral Vibe ([fc2228e](https://github.com/GrayCodeAI/tok/commit/fc2228e32dff4a2b4c4cc4bea49ff5b489d3c8bb)) * **cli:** add compaction flags for Layer 11 ([f9c868c](https://github.com/GrayCodeAI/tok/commit/f9c868c987fe478109628ef9ba6baeb689f576c2)) -* close 9 world-class gaps vs rtk and caveman ([f0059eb](https://github.com/GrayCodeAI/tok/commit/f0059ebc2201cbe3c0ad5a8776b5f656e1bc8c87)) -* close last two caveman gaps ([3028726](https://github.com/GrayCodeAI/tok/commit/302872695f5482bb8a8e5c340e1613755a31dd1c)) -* close tok gaps vs rtk and caveman ([760f326](https://github.com/GrayCodeAI/tok/commit/760f3268c5de816c00e6f92657f8f19834457942)) +* close 9 world-class gaps vs rtk and prompt-compression ([f0059eb](https://github.com/GrayCodeAI/tok/commit/f0059ebc2201cbe3c0ad5a8776b5f656e1bc8c87)) +* close last two prompt-compression gaps ([3028726](https://github.com/GrayCodeAI/tok/commit/302872695f5482bb8a8e5c340e1613755a31dd1c)) +* close tok gaps vs rtk and prompt-compression ([760f326](https://github.com/GrayCodeAI/tok/commit/760f3268c5de816c00e6f92657f8f19834457942)) * complete honest competitor analysis - cloned and analyzed 15 OSS tools ([3000897](https://github.com/GrayCodeAI/tok/commit/3000897d43294cebc80fe4ad37f8c22142e2e642)) * complete integration test suite with archive fixes and MCP tools ([669293c](https://github.com/GrayCodeAI/tok/commit/669293c6fe9589696197e2dcbe9f22c5d5b71535)) * complete SIMD optimization with Go 1.26 ([a3dbc85](https://github.com/GrayCodeAI/tok/commit/a3dbc857d662ab98690cc06e90d888491bf0f0fa)) diff --git a/caveman.go b/compress.go similarity index 64% rename from caveman.go rename to compress.go index ee28afbf..1c9db33a 100644 --- a/caveman.go +++ b/compress.go @@ -1,14 +1,14 @@ -// Caveman prompt compression — public API. +// Prompt compression — public API. // -// This file exposes the caveman algorithm (port from github.com/JuliusBrussee/caveman) +// This file exposes the algorithm (port from prompt-compression algorithm) // as a top-level tok API. It is independent of the main compression pipeline // and does not use tiers/modes/budgets; it is a self-contained, deterministic // prose compression function with three intensity levels. // // Usage: // -// out, stats := tok.CompressCaveman("Sure, I can help you with that.", tok.CavemanLite) -// out, stats := tok.CompressCaveman(prompt, tok.CavemanUltra) +// out, stats := tok.PromptCompress("Sure, I can help you with that.", tok.IntensityLite) +// out, stats := tok.PromptCompress(prompt, tok.IntensityUltra) // // The function NEVER modifies sensitive segments (security warnings, destructive // commands, credential mentions) — those are passed through verbatim regardless @@ -18,7 +18,7 @@ package tok import "github.com/GrayCodeAI/tok/internal/compress" -// Intensity selects how aggressive caveman compression should be. +// Intensity selects how aggressive prompt compression should be. // // - IntensityLite: drops pleasantries only ("Sure", "Of course", "Please"). // - IntensityFull: Lite + articles ("a", "an", "the") + filler + hedging. @@ -28,17 +28,17 @@ import "github.com/GrayCodeAI/tok/internal/compress" // a longer output than a lower intensity on the same input. type Intensity = compress.Intensity -// Intensity preset constants. Use as a direct argument to CompressCaveman. +// Intensity preset constants. Use as a direct argument to PromptCompress. const ( - CavemanLite Intensity = compress.Lite - CavemanFull Intensity = compress.Full - CavemanUltra Intensity = compress.Ultra + IntensityLite Intensity = compress.Lite + IntensityFull Intensity = compress.Full + IntensityUltra Intensity = compress.Ultra ) -// CavemanStats is the statistics struct returned from CompressCaveman. -type CavemanStats = compress.Stats +// PromptStats is the statistics struct returned from PromptCompress. +type PromptStats = compress.Stats -// CompressCaveman applies the caveman prompt-compression algorithm to text. +// PromptCompress applies the prompt-compression algorithm to text. // // Behavior: // - Empty input returns ("", zero stats). @@ -48,6 +48,6 @@ type CavemanStats = compress.Stats // - Dictionary substitutions preserve the case of the first character. // // This is independent of tok.Compress() and does not consume options/presets. -func CompressCaveman(text string, intensity Intensity) (string, CavemanStats) { +func PromptCompress(text string, intensity Intensity) (string, PromptStats) { return compress.Compress(text, intensity) } diff --git a/caveman_test.go b/compress_test.go similarity index 65% rename from caveman_test.go rename to compress_test.go index 6d694b15..51650e34 100644 --- a/caveman_test.go +++ b/compress_test.go @@ -7,32 +7,32 @@ import ( "github.com/GrayCodeAI/tok" ) -func TestCompressCaveman_Lite(t *testing.T) { +func TestPromptCompress_Lite(t *testing.T) { in := "Sure, I can help you with that. Of course, the answer is yes." - out, stats := tok.CompressCaveman(in, tok.CavemanLite) + out, stats := tok.PromptCompress(in, tok.IntensityLite) if strings.Contains(out, "Sure,") { t.Errorf("expected 'Sure,' to be dropped at Lite, got %q", out) } if !strings.Contains(out, "help you with that") { t.Errorf("expected help text preserved, got %q", out) } - if stats.Intensity != tok.CavemanLite { + if stats.Intensity != tok.IntensityLite { t.Errorf("expected intensity=Lite, got %v", stats.Intensity) } } -func TestCompressCaveman_Full(t *testing.T) { +func TestPromptCompress_Full(t *testing.T) { in := "The quick brown fox jumps over the lazy dog." - out, _ := tok.CompressCaveman(in, tok.CavemanFull) + out, _ := tok.PromptCompress(in, tok.IntensityFull) // "the" should be dropped if strings.Contains(out, " the ") && !strings.Contains(out, "over") { t.Errorf("expected 'the' to be dropped at Full, got %q", out) } } -func TestCompressCaveman_Ultra(t *testing.T) { +func TestPromptCompress_Ultra(t *testing.T) { in := "However, the system is basically quite slow. Therefore, we need to optimize it." - out, stats := tok.CompressCaveman(in, tok.CavemanUltra) + out, stats := tok.PromptCompress(in, tok.IntensityUltra) if strings.Contains(out, "However,") { t.Errorf("expected 'However,' to be dropped at Ultra, got %q", out) } @@ -44,9 +44,9 @@ func TestCompressCaveman_Ultra(t *testing.T) { } } -func TestCompressCaveman_SensitivePassThrough(t *testing.T) { +func TestPromptCompress_SensitivePassThrough(t *testing.T) { in := "Be careful with rm -rf /tmp. The file is large." - out, stats := tok.CompressCaveman(in, tok.CavemanFull) + out, stats := tok.PromptCompress(in, tok.IntensityFull) // The segment containing "rm -rf" must be preserved verbatim. if !strings.Contains(out, "rm -rf /tmp.") { t.Errorf("expected 'rm -rf /tmp.' to be preserved, got %q", out) @@ -56,8 +56,8 @@ func TestCompressCaveman_SensitivePassThrough(t *testing.T) { } } -func TestCompressCaveman_Empty(t *testing.T) { - out, stats := tok.CompressCaveman("", tok.CavemanFull) +func TestPromptCompress_Empty(t *testing.T) { + out, stats := tok.PromptCompress("", tok.IntensityFull) if out != "" { t.Errorf("expected empty output, got %q", out) } @@ -66,9 +66,9 @@ func TestCompressCaveman_Empty(t *testing.T) { } } -func TestCompressCaveman_Dictionary(t *testing.T) { +func TestPromptCompress_Dictionary(t *testing.T) { in := "In order to install, you need to make use of the installer." - out, _ := tok.CompressCaveman(in, tok.CavemanFull) + out, _ := tok.PromptCompress(in, tok.IntensityFull) if strings.Contains(out, "In order to") { t.Errorf("expected 'In order to' to be replaced, got %q", out) } @@ -77,10 +77,10 @@ func TestCompressCaveman_Dictionary(t *testing.T) { } } -func TestCompressCaveman_DoesNotAffectTopLevelCompress(t *testing.T) { - // Regression: adding caveman API must not change the main Compress() output +func TestPromptCompress_DoesNotAffectTopLevelCompress(t *testing.T) { + // Regression: adding prompt-compression API must not change the main Compress() output // for a typical input. We test that both APIs can be called and return - // different shapes (caveman returns CavemanStats, main returns Stats). + // different shapes (PromptCompress returns PromptStats, main returns Stats). in := "The quick brown fox jumps over the lazy dog." mainOut, mainStats := tok.Compress(in, tok.Aggressive) if mainOut == "" { @@ -88,10 +88,10 @@ func TestCompressCaveman_DoesNotAffectTopLevelCompress(t *testing.T) { } _ = mainStats.OriginalTokens - cavemanOut, _ := tok.CompressCaveman(in, tok.CavemanFull) - // Caveman output may equal main output (if both drop "The") or differ — + promptCompressOut, _ := tok.PromptCompress(in, tok.IntensityFull) + // PromptCompress output may equal main output (if both drop "The") or differ — // the point is they don't crash each other. - if cavemanOut == "" { - t.Error("expected non-empty caveman output") + if promptCompressOut == "" { + t.Error("expected non-empty prompt-compression output") } } diff --git a/internal/compress/caveman.go b/internal/compress/algorithm.go similarity index 94% rename from internal/compress/caveman.go rename to internal/compress/algorithm.go index 7dac85c2..68570f92 100644 --- a/internal/compress/caveman.go +++ b/internal/compress/algorithm.go @@ -1,5 +1,5 @@ -// Package compress implements the caveman prompt-compression algorithm -// ported natively to Go. Source: github.com/JuliusBrussee/caveman (MIT). +// Package compress implements the prompt-compression algorithm +// ported natively to Go. Source: JuliusBrussee/prompt-compression (MIT). // // Algorithm overview: // @@ -17,7 +17,7 @@ import ( "strings" ) -// CompressResult reports what caveman did to the input. +// CompressResult reports what the compression did to the input. type CompressResult struct { // Original is the input verbatim. Original string @@ -50,11 +50,11 @@ type Stats struct { SensitiveKeywordsHit []string } -// Compress applies the caveman algorithm to s at the given intensity. +// Compress applies the algorithm to s at the given intensity. // // Behavior: // - Empty input returns (input, empty result) untouched. -// - CJK-heavy text is passed through (caveman rules assume Latin grammar). +// - CJK-heavy text is passed through (compression rules assume Latin grammar). // - Sensitive segments (security/destructive keywords) are preserved // verbatim regardless of intensity. // - Multi-word drop-list entries ("of course", "feel free") are matched diff --git a/internal/compress/caveman_test.go b/internal/compress/algorithm_test.go similarity index 100% rename from internal/compress/caveman_test.go rename to internal/compress/algorithm_test.go diff --git a/internal/compress/caveman_dict.go b/internal/compress/dict.go similarity index 98% rename from internal/compress/caveman_dict.go rename to internal/compress/dict.go index 4addf328..9f2e7dc7 100644 --- a/internal/compress/caveman_dict.go +++ b/internal/compress/dict.go @@ -1,7 +1,7 @@ package compress -// engV1Dictionary is the caveman "eng/v1" phrase-substitution dictionary. -// Source: caveman/skills/caveman/SKILL.md, "Dictionary" section. +// engV1Dictionary is the "eng/v1" phrase-substitution dictionary. +// Source: skills/prompt-compression/SKILL.md, "Dictionary" section. // // Keys are case-insensitive verbose phrases. Values are terse equivalents. // Multi-word keys are matched as whole phrases (whitespace-normalized). diff --git a/internal/compress/caveman_rules.go b/internal/compress/rules.go similarity index 92% rename from internal/compress/caveman_rules.go rename to internal/compress/rules.go index 7ce4c518..96f63379 100644 --- a/internal/compress/caveman_rules.go +++ b/internal/compress/rules.go @@ -1,5 +1,5 @@ -// Package compress implements the caveman prompt-compression algorithm -// ported natively to Go. Source: github.com/JuliusBrussee/caveman (MIT). +// Package compress implements the prompt-compression algorithm +// ported natively to Go. Source: JuliusBrussee/prompt-compression (MIT). // // The algorithm is a pure text-rewrite engine. Three intensity levels // (Lite, Full, Ultra) progressively drop articles, filler words, @@ -10,7 +10,7 @@ // warnings, destructive operations, and code-bearing segments. package compress -// Intensity controls the aggressiveness of caveman compression. +// Intensity controls the aggressiveness of prompt compression. // // The intensity ordering is Lite < Full < Ultra. Higher intensity drops // more word classes and applies more aggressive dictionary substitutions. @@ -20,7 +20,7 @@ const ( // Lite drops only pleasantries and obvious filler; preserves articles. Lite Intensity = iota // Full drops articles, filler, pleasantries, and hedging. - // This is the caveman default. + // This is the default. Full // Ultra additionally drops conjunctions and forces sentence fragments. Ultra @@ -41,7 +41,7 @@ func (i Intensity) String() string { } // dropLists maps each intensity to the set of words/phrases to drop. -// Source: caveman/skills/caveman/SKILL.md, "Intensity" section. +// Source: skills/prompt-compression/SKILL.md, "Intensity" section. // // Multi-word phrases ("of course", "feel free") are matched case-insensitive // as whole phrases, not as individual words. diff --git a/internal/compress/caveman_safety.go b/internal/compress/safety.go similarity index 95% rename from internal/compress/caveman_safety.go rename to internal/compress/safety.go index 05674cbc..eedaedfe 100644 --- a/internal/compress/caveman_safety.go +++ b/internal/compress/safety.go @@ -8,7 +8,7 @@ import ( // safetyKeywords are substrings that, if present, force the compressor // to fall back to "normal" prose (no compression) for that segment. // -// Source: caveman/skills/caveman/SKILL.md, "Auto-clarity rules": +// Source: skills/prompt-compression/SKILL.md, "Auto-clarity rules": // // "Drop to normal prose for security warnings, destructive ops, // ambiguous sequences." @@ -116,7 +116,7 @@ func SplitSafeSegments(s string) []segment { if s == "" { return nil } - // Split on sentence boundaries; the caveman rule operates per sentence. + // Split on sentence boundaries; the rule operates per sentence. segs := splitSentences(s) out := make([]segment, 0, len(segs)) for _, seg := range segs { @@ -222,7 +222,7 @@ func isUpperLetter(b byte) bool { } // isCJK reports whether s is primarily CJK characters (Chinese/Japanese/Korean). -// Used to decide if the caveman rules even apply — CJK text doesn't have +// Used to decide if the compression rules even apply — CJK text doesn't have // articles/filler to drop. func isCJK(s string) bool { cjk := 0