ci(boundary): enforce Hawk ecosystem import boundaries (#24)

* ci(boundary): add ecosystem boundary guard

yaad is a Hawk support engine with no cross-repo shared types of its own, so it
stays contract-free. Add the one-way ecosystem boundary guard to keep it that
way:

- add scripts/check-ecosystem-boundaries.sh (forbids hawk/internal and
  hawk/shared/types imports)
- wire the guard into the Makefile and CI
- document the boundary rule in the README

build and the boundary guard pass.

* refactor: decouple yaad from tok helpers

* docs: remove legacy shared types references

* chore: strip Co-authored-by trailers in lefthook hooks

* fix(boundary): fall back to grep when rg is unavailable

* build(deps): go mod tidy after dropping tok dependency

Author: Patel230

.github/workflows/ci.yml

@@ -52,6 +52,8 @@ jobs:
           cache: true
       - name: Clone tok (workspace dep)
         run: git clone --depth=1 https://github.com/GrayCodeAI/tok.git ../tok
+      - name: Boundary guard
+        run: bash ./scripts/check-ecosystem-boundaries.sh
       - name: gofumpt diff
         run: |
           go install mvdan.cc/gofumpt@v0.10.0
@@ -78,6 +80,8 @@ jobs:
           cache: true
       - name: Clone tok (workspace dep)
         run: git clone --depth=1 https://github.com/GrayCodeAI/tok.git ../tok
+      - name: Boundary guard
+        run: bash ./scripts/check-ecosystem-boundaries.sh
       - uses: golangci/golangci-lint-action@4afd733a84b1f43292c63897423277bb7f4313a9 # v7.0.0
         with:
           version: v2.1.0
@@ -99,6 +103,8 @@ jobs:
           cache: true
       - name: Clone tok (workspace dep)
         run: git clone --depth=1 https://github.com/GrayCodeAI/tok.git ../tok
+      - name: Boundary guard
+        run: bash ./scripts/check-ecosystem-boundaries.sh
       - name: Tidy check
         run: |
           go mod tidy

Makefile

@@ -31,9 +31,12 @@ GOVULNCHECK  := $(GOBIN_DIR)/govulncheck
 # ---------------------------------------------------------------------------
 # Phony declarations (alphabetical).
 # ---------------------------------------------------------------------------
-.PHONY: all bench build ci clean cover fmt help lint lint-fix \
+.PHONY: all bench boundaries build ci clean cover fmt help lint lint-fix \
         security sync-version test test-10x test-race tidy version vet
 
+boundaries: ## Enforce support-repo import boundaries.
+	bash ./scripts/check-ecosystem-boundaries.sh
+
 # ---------------------------------------------------------------------------
 # Default target.
 # ---------------------------------------------------------------------------
@@ -101,7 +104,7 @@ sync-version: ## Copy root VERSION into internal/version/VERSION (kept in sync f
 # ---------------------------------------------------------------------------
 # Composite gate used by CI and pre-push.
 # ---------------------------------------------------------------------------
-ci: tidy fmt vet lint test-race security ## Run everything CI runs.
+ci: tidy fmt vet lint boundaries test-race security ## Run everything CI runs.
 	@echo "All CI checks passed."
 
 # ---------------------------------------------------------------------------

README.md

@@ -42,6 +42,14 @@ sessions, across models, across projects — with no separate install or daemon
 > )
 > ```
 
+## Ecosystem Boundaries
+
+Yaad is a Hawk support engine. Keep the dependency edge one-way:
+
+- depend on `hawk-core-contracts` when a stable cross-repo contract is needed
+- do not import `hawk/internal/*`
+- do not import removed legacy path `hawk/shared/types`; use `hawk-core-contracts/types`
+
 ---
 
 ## What Happens Next

engine/context_pack.go

@@ -5,7 +5,6 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/GrayCodeAI/tok"
 	"github.com/GrayCodeAI/yaad/storage"
 )
 
@@ -84,7 +83,7 @@ func (cp *ContextPacker) Pack(nodes []*storage.Node) *PackedContext {
 
 		// Format and check budget
 		line := cp.formatNode(n)
-		lineTokens := tok.EstimateTokens(line)
+		lineTokens := estimateTokens(line)
 
 		if tokensUsed+lineTokens > cp.budget {
 			break

engine/token_utils.go

@@ -0,0 +1,57 @@
+package engine
+
+import (
+	"sync"
+
+	tiktoken "github.com/tiktoken-go/tokenizer"
+)
+
+var (
+	tokenizerOnce sync.Once
+	tokenizerBPE  tiktoken.Codec
+	tokenizerErr  error
+)
+
+func estimateTokens(content string) int {
+	if content == "" {
+		return 0
+	}
+
+	tokenizerOnce.Do(func() {
+		tokenizerBPE, tokenizerErr = tiktoken.Get(tiktoken.Cl100kBase)
+	})
+	if tokenizerErr == nil && tokenizerBPE != nil {
+		if count, err := tokenizerBPE.Count(content); err == nil {
+			return count
+		}
+	}
+
+	return fallbackTokenCount(content)
+}
+
+func fallbackTokenCount(content string) int {
+	length := len(content)
+	if length < 30 {
+		return (length + 2) / 3
+	}
+	if length < 100 {
+		return (length + 3) / 4
+	}
+
+	spaces := 0
+	sample := length
+	if sample > 200 {
+		sample = 200
+	}
+	for i := 0; i < sample; i++ {
+		switch content[i] {
+		case ' ', '\n', '\t', '\r':
+			spaces++
+		}
+	}
+
+	spaceRatio := float64(spaces) / float64(sample)
+	nonSpaceChars := float64(length) * (1 - spaceRatio)
+	spaceTokens := float64(length) * spaceRatio
+	return int(nonSpaceChars/3.5 + spaceTokens)
+}

go.mod

@@ -1,5 +1,3 @@
 go 1.26.4
 
 use .
-
-replace github.com/GrayCodeAI/tok => ../tok

go.sum

@@ -14,8 +14,6 @@ import (
 	"path/filepath"
 	"regexp"
 	"strings"
-
-	"github.com/GrayCodeAI/tok"
 )
 
 // Chunk represents a semantically meaningful unit of code.
@@ -806,9 +804,10 @@ func (c *Chunker) AddOverlap(chunks []Chunk, overlapLines int) []Chunk {
 }
 
 // EstimateTokens returns the estimated token count for the given text.
-// Delegates to tok.EstimateTokens for accurate heuristic-based estimation.
+// Uses a local BPE-first estimator to avoid engine-to-engine coupling on tok
+// while preserving the old chunking behavior closely.
 func EstimateTokens(content string) int {
-	return tok.EstimateTokens(content)
+	return estimateTokens(content)
 }
 
 // splitLargeChunk splits a chunk that exceeds MaxChunkTokens into smaller pieces.

go.work

@@ -0,0 +1,57 @@
+package ingest
+
+import (
+	"sync"
+
+	tiktoken "github.com/tiktoken-go/tokenizer"
+)
+
+var (
+	tokenizerOnce sync.Once
+	tokenizerBPE  tiktoken.Codec
+	tokenizerErr  error
+)
+
+func estimateTokens(content string) int {
+	if content == "" {
+		return 0
+	}
+
+	tokenizerOnce.Do(func() {
+		tokenizerBPE, tokenizerErr = tiktoken.Get(tiktoken.Cl100kBase)
+	})
+	if tokenizerErr == nil && tokenizerBPE != nil {
+		if count, err := tokenizerBPE.Count(content); err == nil {
+			return count
+		}
+	}
+
+	return fallbackTokenCount(content)
+}
+
+func fallbackTokenCount(content string) int {
+	length := len(content)
+	if length < 30 {
+		return (length + 2) / 3
+	}
+	if length < 100 {
+		return (length + 3) / 4
+	}
+
+	spaces := 0
+	sample := length
+	if sample > 200 {
+		sample = 200
+	}
+	for i := 0; i < sample; i++ {
+		switch content[i] {
+		case ' ', '\n', '\t', '\r':
+			spaces++
+		}
+	}
+
+	spaceRatio := float64(spaces) / float64(sample)
+	nonSpaceChars := float64(length) * (1 - spaceRatio)
+	spaceTokens := float64(length) * spaceRatio
+	return int(nonSpaceChars/3.5 + spaceTokens)
+}