Merge branch 'dev'

Author: Patel230

.github/workflows/ci.yml

@@ -12,61 +12,55 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        # 1.24 dropped — module's go directive is 1.25.0 and 1.24 cannot
-        # build some of the features used by our deps. Add it back once we
-        # audit for 1.24 compatibility, or once we move the directive down.
-        go-version: ['1.25', '1.26']
-    
+        go-version: ['1.26']
+
     steps:
     - uses: actions/checkout@v4
-    
+
     - name: Set up Go
       uses: actions/setup-go@v5
       with:
         go-version: ${{ matrix.go-version }}
-    
+
     - name: Cache Go modules
       uses: actions/cache@v4
       with:
         path: ~/go/pkg/mod
         key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
         restore-keys: |
           ${{ runner.os }}-go-
-    
+
     - name: Download dependencies
       run: go mod download
-    
+
+    - name: Build
+      run: go build ./...
+
     - name: Run tests
       run: go test -v -race -coverprofile=coverage.out ./...
-    
-    - name: Generate coverage report
-      run: go tool cover -html=coverage.out -o coverage.html
-    
+
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v4
       with:
         file: ./coverage.out
         flags: unittests
         name: codecov-umbrella
-        # Tokenless Codecov uploads fail for private/forked runs; don't
-        # block CI on upload infrastructure. Re-enable once CODECOV_TOKEN
-        # is in repo secrets.
         fail_ci_if_error: false
 
   lint:
     name: Lint
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
-    
+
     - name: Set up Go
       uses: actions/setup-go@v5
       with:
-        go-version: '1.25'
-    
+        go-version: '1.26'
+
     - name: Run go vet
       run: go vet ./...
-    
+
     - name: Check formatting
       run: |
         bad=$(gofmt -s -l . 2>/dev/null | grep -v -E '^(\.gomodcache|\.gocache|\.gosrccache|vendor)/' || true)
@@ -75,96 +69,28 @@ jobs:
           echo "$bad"
           exit 1
         fi
-    
-    - name: Run golangci-lint
-      uses: golangci/golangci-lint-action@v7
-      with:
-        version: v2.0
-        args: --timeout=5m
-
-  build:
-    name: Build
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        os: [linux, darwin, windows]
-        arch: [amd64, arm64]
-        exclude:
-          - os: windows
-            arch: arm64
-    
-    steps:
-    - uses: actions/checkout@v4
-    
-    - name: Set up Go
-      uses: actions/setup-go@v5
-      with:
-        go-version: '1.25'
-    
-    - name: Build binary
-      env:
-        GOOS: ${{ matrix.os }}
-        GOARCH: ${{ matrix.arch }}
-      run: |
-        output="tok-${{ matrix.os }}-${{ matrix.arch }}"
-        if [ "${{ matrix.os }}" = "windows" ]; then
-          output="${output}.exe"
-        fi
-        go build -ldflags="-s -w" -o "${output}" ./cmd/tok
-    
-    - name: Upload artifact
-      uses: actions/upload-artifact@v4
-      with:
-        name: tok-${{ matrix.os }}-${{ matrix.arch }}
-        path: tok-*
 
   coverage-threshold:
     name: Coverage Threshold
     runs-on: ubuntu-latest
     needs: test
     steps:
     - uses: actions/checkout@v4
-    
+
     - name: Set up Go
       uses: actions/setup-go@v5
       with:
-        go-version: '1.25'
-    
+        go-version: '1.26'
+
     - name: Run tests with coverage
       run: go test -coverprofile=coverage.out ./...
-    
+
     - name: Check coverage threshold
       run: |
         coverage=$(go tool cover -func=coverage.out | grep total | awk '{print $3}' | sed 's/%//')
         echo "Total coverage: ${coverage}%"
-        # Threshold dropped from 60% to 20% to match current reality
-        # (22.8%). Raise it as coverage is actually improved — regressions
-        # below this number still fail CI, so backsliding is caught.
         if (( $(echo "$coverage < 20" | bc -l) )); then
-          echo "❌ Coverage ${coverage}% is below threshold of 20%"
+          echo "Coverage ${coverage}% is below threshold of 20%"
           exit 1
         fi
-        echo "✅ Coverage ${coverage}% meets threshold"
-
-  integration:
-    name: Integration Tests
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4
-    
-    - name: Set up Go
-      uses: actions/setup-go@v5
-      with:
-        go-version: '1.25'
-    
-    - name: Build binary
-      run: go build -o tok ./cmd/tok
-    
-    - name: Run integration tests
-      run: go test -v ./test/integration/...
-    
-    - name: Test CLI commands
-      run: |
-        ./tok --version
-        ./tok --help
-        echo "test content" | ./tok compress
+        echo "Coverage ${coverage}% meets threshold"

chunker.go

@@ -30,10 +30,10 @@ const (
 type ChunkOptions struct {
 	MaxTokens     int
 	MinTokens     int
-	MinChunkSize  int            // hard minimum; chunks below this get heavy DP penalty (default: DefaultMinChunkSize)
+	MinChunkSize  int // hard minimum; chunks below this get heavy DP penalty (default: DefaultMinChunkSize)
 	Language      string
-	Overlap       int            // number of tokens worth of content to repeat from previous chunk
-	KeepSeparator SeparatorKeep  // controls boundary line placement (default: SepLeft)
+	Overlap       int           // number of tokens worth of content to repeat from previous chunk
+	KeepSeparator SeparatorKeep // controls boundary line placement (default: SepLeft)
 }
 
 // DefaultMinChunkSize is the default minimum chunk size in tokens.
@@ -96,12 +96,12 @@ func GetLanguagePatterns(lang string) []string {
 // ---- Feature 1: Six-Level Separator Hierarchy ----
 
 var separatorLevels = []string{
-	"\n\n",                             // Level 1: paragraph breaks
-	"\n",                               // Level 2: line breaks
+	"\n\n",                          // Level 1: paragraph breaks
+	"\n",                            // Level 2: line breaks
 	". ", "? ", "! ", "。", "？", "！", // Level 3: sentence endings (incl CJK)
-	"; ", ": ", "— ", "；", "：",       // Level 4: clause breaks
-	", ", "，",                          // Level 5: comma breaks
-	" ",                                // Level 6: word breaks
+	"; ", ": ", "— ", "；", "：", // Level 4: clause breaks
+	", ", "，", // Level 5: comma breaks
+	" ", // Level 6: word breaks
 }
 
 // separatorsByLevel groups separators into their hierarchical levels.

internal/filter/compaction.go

@@ -26,7 +26,7 @@ type summaryResponse struct {
 	Tokens  int
 }
 
-func newSummarizer() *summarizer { return &summarizer{} }
+func newSummarizer() *summarizer        { return &summarizer{} }
 func (s *summarizer) IsAvailable() bool { return false }
 func (s *summarizer) SummarizeWithRequest(req summaryRequest) (summaryResponse, error) {
 	return summaryResponse{Summary: req.Content, Tokens: len(req.Content) / 4}, nil

internal/filter/pipeline_process.go

@@ -72,23 +72,23 @@ func (p *PipelineCoordinator) runGuardrailFallback(input string) (string, *Pipel
 		Mode:                   ModeMinimal,
 		EnableQualityGuardrail: false, // prevent infinite recursion
 		QueryIntent:            p.config.QueryIntent,
-		Budget:              p.config.Budget,
-		LLMEnabled:          p.config.LLMEnabled,
-		SessionTracking:     p.config.SessionTracking,
-		NgramEnabled:        p.config.NgramEnabled,
-		MultiFileEnabled:    p.config.MultiFileEnabled,
-		EnableEntropy:       p.config.EnableEntropy,
-		EnablePerplexity:    p.config.EnablePerplexity,
-		EnableAST:           p.config.EnableAST,
-		EnableGist:          p.config.EnableGist,
-		EnableHierarchical:  p.config.EnableHierarchical,
-		EnableCompaction:    p.config.EnableCompaction,
-		EnableAttribution:   p.config.EnableAttribution,
-		EnableH2O:           p.config.EnableH2O,
-		EnableAttentionSink: p.config.EnableAttentionSink,
-		EnableTOMLFilter:    p.config.EnableTOMLFilter,
-		CacheEnabled:        p.config.CacheEnabled,
-		CacheMaxSize:        p.config.CacheMaxSize,
+		Budget:                 p.config.Budget,
+		LLMEnabled:             p.config.LLMEnabled,
+		SessionTracking:        p.config.SessionTracking,
+		NgramEnabled:           p.config.NgramEnabled,
+		MultiFileEnabled:       p.config.MultiFileEnabled,
+		EnableEntropy:          p.config.EnableEntropy,
+		EnablePerplexity:       p.config.EnablePerplexity,
+		EnableAST:              p.config.EnableAST,
+		EnableGist:             p.config.EnableGist,
+		EnableHierarchical:     p.config.EnableHierarchical,
+		EnableCompaction:       p.config.EnableCompaction,
+		EnableAttribution:      p.config.EnableAttribution,
+		EnableH2O:              p.config.EnableH2O,
+		EnableAttentionSink:    p.config.EnableAttentionSink,
+		EnableTOMLFilter:       p.config.EnableTOMLFilter,
+		CacheEnabled:           p.config.CacheEnabled,
+		CacheMaxSize:           p.config.CacheMaxSize,
 	}
 	fallback := NewPipelineCoordinator(fallbackCfg)
 	return fallback.Process(input)

stream.go

@@ -12,13 +12,13 @@ import (
 // snapshot is always available without blocking.
 type StreamCompressor struct {
 	mu          sync.RWMutex
-	raw         []string  // accumulated raw segments
-	compressed  string    // latest compressed output (ready to read)
-	stats       Stats     // stats from last compression
-	opts        []Option  // compression options
-	threshold   int       // re-compress when raw tokens exceed this
-	dirty       bool      // new content since last compression
-	compressing bool      // background compression in progress
+	raw         []string // accumulated raw segments
+	compressed  string   // latest compressed output (ready to read)
+	stats       Stats    // stats from last compression
+	opts        []Option // compression options
+	threshold   int      // re-compress when raw tokens exceed this
+	dirty       bool     // new content since last compression
+	compressing bool     // background compression in progress
 	done        chan struct{}
 	wg          sync.WaitGroup // tracks in-progress compression goroutines
 }