integration_test.go

package tok_test

import (
	"strings"
	"testing"

	"github.com/GrayCodeAI/tok"
)

// ---------------------------------------------------------------------------
// 1. Full Pipeline - End-to-end compression of text through all stages
// ---------------------------------------------------------------------------

func TestIntegration_FullPipeline_Minimal(t *testing.T) {
	input := `This is a fairly long piece of text that contains multiple sentences.
It talks about how compression works in the context of LLMs.
The goal is to reduce token usage while preserving meaning.
There are several layers involved in the pipeline.
Each layer contributes to the overall compression ratio.`

	output, stats := tok.Compress(input, tok.Minimal)

	if output == "" {
		t.Fatal("full pipeline (minimal) returned empty output")
	}
	if stats.OriginalTokens == 0 {
		t.Fatal("OriginalTokens should be non-zero")
	}
	if stats.FinalTokens == 0 {
		t.Fatal("FinalTokens should be non-zero")
	}
	// Output must be shorter than or equal to the input
	if len(output) > len(input) {
		t.Errorf("output length (%d) exceeds input length (%d)", len(output), len(input))
	}
}

func TestIntegration_FullPipeline_Aggressive(t *testing.T) {
	input := `This is a fairly long piece of text that contains multiple sentences.
It talks about how compression works in the context of LLMs.
The goal is to reduce token usage while preserving meaning.
There are several layers involved in the pipeline.
Each layer contributes to the overall compression ratio.
Additional filler content follows to give the compressor something to work with.
The quick brown fox jumps over the lazy dog.
Pack my box with five dozen liquor jugs.
How vexingly quick daft zebras jump.`

	output, stats := tok.Compress(input, tok.Aggressive)

	if output == "" {
		t.Fatal("full pipeline (aggressive) returned empty output")
	}
	if stats.OriginalTokens == 0 {
		t.Fatal("OriginalTokens should be non-zero")
	}
	// Aggressive mode should achieve some reduction on a long enough input
	if stats.ReductionPercent < 0 {
		t.Errorf("ReductionPercent should not be negative, got %.2f", stats.ReductionPercent)
	}
}

func TestIntegration_FullPipeline_WithQueryIntent(t *testing.T) {
	input := `[INFO] Starting application server on port 8080
[INFO] Connected to database successfully
[DEBUG] Loading configuration from /etc/app/config.yaml
[WARN] Cache miss for key user_session_12345
[INFO] Request received: GET /api/users/42
[ERROR] Connection refused to upstream service
[WARN] Retrying connection attempt 1 of 3
[INFO] Connection restored after retry
[DEBUG] Response sent: 200 OK in 45ms
[INFO] Health check passed`

	output, stats := tok.Compress(input, tok.WithQuery("find errors"))

	if output == "" {
		t.Fatal("query-aware compression returned empty output")
	}
	if stats.OriginalTokens == 0 {
		t.Fatal("OriginalTokens should be non-zero")
	}
	// The error line should still be present (query intent targets errors)
	if !strings.Contains(output, "ERROR") && !strings.Contains(output, "error") {
		t.Log("warning: query intent 'find errors' did not preserve the ERROR line")
	}
}

func TestIntegration_FullPipeline_AllTiers(t *testing.T) {
	input := strings.Repeat("line of content with enough words to trigger compression\n", 50)

	tiers := []struct {
		name string
		tier tok.Tier
	}{
		{"surface", tok.TierSurface},
		{"code", tok.TierCode},
		{"log", tok.TierLog},
		{"adaptive", tok.TierAdaptive},
	}

	for _, tt := range tiers {
		t.Run(tt.name, func(t *testing.T) {
			output, stats := tok.Compress(input, tok.WithTier(tt.tier))
			if output == "" {
				t.Errorf("tier %s returned empty output", tt.name)
			}
			if stats.OriginalTokens == 0 {
				t.Errorf("tier %s: OriginalTokens is zero", tt.name)
			}
		})
	}
}

// ---------------------------------------------------------------------------
// 2. Reversibility - Test that compression is deterministic and that
//    CompactionSchema round-trips correctly through JSON serialization.
// ---------------------------------------------------------------------------

func TestIntegration_Reversibility_DeterministicCompression(t *testing.T) {
	input := "Deterministic compression test: same input should produce same output every time."

	// Run compression twice with the same config
	out1, stats1 := tok.Compress(input, tok.Minimal)
	out2, stats2 := tok.Compress(input, tok.Minimal)

	if out1 != out2 {
		t.Errorf("compression is not deterministic:\n  run1: %q\n  run2: %q", out1, out2)
	}
	if stats1.OriginalTokens != stats2.OriginalTokens {
		t.Errorf("OriginalTokens differ: %d vs %d", stats1.OriginalTokens, stats2.OriginalTokens)
	}
	if stats1.FinalTokens != stats2.FinalTokens {
		t.Errorf("FinalTokens differ: %d vs %d", stats1.FinalTokens, stats2.FinalTokens)
	}
}

func TestIntegration_Reversibility_DeterministicAcrossModes(t *testing.T) {
	input := "Test content that will be compressed in both minimal and aggressive modes."

	outMin, statsMin := tok.Compress(input, tok.Minimal)
	outAgg, statsAgg := tok.Compress(input, tok.Aggressive)

	// Both should produce valid output
	if outMin == "" || outAgg == "" {
		t.Fatal("both modes should produce non-empty output")
	}
	// Different modes may produce different results
	if statsMin.OriginalTokens != statsAgg.OriginalTokens {
		t.Errorf("OriginalTokens should be identical across modes: %d vs %d",
			statsMin.OriginalTokens, statsAgg.OriginalTokens)
	}
}

func TestIntegration_Reversibility_CompactionSchemaRoundTrip(t *testing.T) {
	original := &tok.CompactionSchema{
		TaskOverview:         "Build test suite",
		CurrentState:         "Writing integration tests",
		ImportantDiscoveries: []string{"Pipeline has 20 layers", "Supports multiple tiers"},
		NextSteps:            []string{"Run all tests", "Fix any failures"},
		ContextToPreserve:    []string{"File: integration_test.go", "Module: github.com/GrayCodeAI/tok"},
	}

	// Serialize to prompt and re-parse (simulating LLM round-trip)
	prompt := original.ToPrompt()
	if prompt == "" {
		t.Fatal("ToPrompt returned empty string")
	}

	// Simulate an LLM JSON response based on the schema
	jsonResponse := `{
		"task_overview": "Build test suite",
		"current_state": "Writing integration tests",
		"important_discoveries": ["Pipeline has 20 layers", "Supports multiple tiers"],
		"next_steps": ["Run all tests", "Fix any failures"],
		"context_to_preserve": ["File: integration_test.go", "Module: github.com/GrayCodeAI/tok"]
	}`

	parsed, err := tok.ParseCompactionResponse(jsonResponse)
	if err != nil {
		t.Fatalf("ParseCompactionResponse failed: %v", err)
	}

	// Verify round-trip fidelity
	if parsed.TaskOverview != original.TaskOverview {
		t.Errorf("TaskOverview mismatch: %q vs %q", parsed.TaskOverview, original.TaskOverview)
	}
	if parsed.CurrentState != original.CurrentState {
		t.Errorf("CurrentState mismatch: %q vs %q", parsed.CurrentState, original.CurrentState)
	}
	if len(parsed.ImportantDiscoveries) != len(original.ImportantDiscoveries) {
		t.Errorf("ImportantDiscoveries length mismatch: %d vs %d",
			len(parsed.ImportantDiscoveries), len(original.ImportantDiscoveries))
	}
	if len(parsed.NextSteps) != len(original.NextSteps) {
		t.Errorf("NextSteps length mismatch: %d vs %d",
			len(parsed.NextSteps), len(original.NextSteps))
	}
	if len(parsed.ContextToPreserve) != len(original.ContextToPreserve) {
		t.Errorf("ContextToPreserve length mismatch: %d vs %d",
			len(parsed.ContextToPreserve), len(original.ContextToPreserve))
	}
}

func TestIntegration_Reversibility_SecretDetectionAndRedaction(t *testing.T) {
	detector := tok.DefaultSecretDetector()

	textWithSecrets := "My AWS key is AKIAIOSFODNN7EXAMPLE and my GitHub token is ghp_ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghij"

	matches := detector.DetectSecrets(textWithSecrets)
	if len(matches) == 0 {
		t.Fatal("should detect at least one secret")
	}

	redacted := detector.RedactSecrets(textWithSecrets)
	if strings.Contains(redacted, "AKIAIOSFODNN7EXAMPLE") {
		t.Error("redacted text should not contain the AWS key")
	}
	if strings.Contains(redacted, "ghp_ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghij") {
		t.Error("redacted text should not contain the GitHub token")
	}
	if !strings.Contains(redacted, "REDACTED") {
		t.Error("redacted text should contain REDACTED markers")
	}
}

// ---------------------------------------------------------------------------
// 3. Token Estimation - Test that token counts are accurate after compression
// ---------------------------------------------------------------------------

func TestIntegration_TokenEstimation_Accurate(t *testing.T) {
	// Short text: token count should be roughly len/4 (BPE heuristic)
	short := "Hello world"
	tokens := tok.EstimateTokens(short)
	if tokens <= 0 {
		t.Error("token estimate for short text should be positive")
	}
	if tokens > 10 {
		t.Errorf("token estimate for %q seems too high: %d", short, tokens)
	}
}

func TestIntegration_TokenEstimation_ScalesWithLength(t *testing.T) {
	small := tok.EstimateTokens("hello")
	large := tok.EstimateTokens(strings.Repeat("hello world test sentence. ", 100))

	if large <= small {
		t.Errorf("large text tokens (%d) should exceed small text tokens (%d)", large, small)
	}
}

func TestIntegration_TokenEstimation_AfterCompression(t *testing.T) {
	input := strings.Repeat("The quick brown fox jumps over the lazy dog. ", 50)
	output, stats := tok.Compress(input, tok.Aggressive)

	// The stats should reflect accurate token counting
	if stats.OriginalTokens <= 0 {
		t.Fatal("OriginalTokens should be positive")
	}
	if stats.FinalTokens <= 0 {
		t.Fatal("FinalTokens should be positive")
	}

	// FinalTokens should approximately match the actual token count of the output
	actualFinal := tok.EstimateTokens(output)
	diff := stats.FinalTokens - actualFinal
	if diff < -5 || diff > 5 {
		t.Logf("warning: stats.FinalTokens=%d but EstimateTokens(output)=%d (diff=%d)", stats.FinalTokens, actualFinal, diff)
	}

	// TokensSaved should equal OriginalTokens - FinalTokens
	expectedSaved := stats.OriginalTokens - stats.FinalTokens
	if expectedSaved < 0 {
		expectedSaved = 0
	}
	if stats.TokensSaved != expectedSaved {
		t.Errorf("TokensSaved=%d, but Original-Final=%d", stats.TokensSaved, expectedSaved)
	}
}

func TestIntegration_TokenEstimation_EmptyString(t *testing.T) {
	tokens := tok.EstimateTokens("")
	if tokens != 0 {
		t.Errorf("empty string should have 0 tokens, got %d", tokens)
	}
}

func TestIntegration_TokenEstimation_CodeSnippet(t *testing.T) {
	code := `func main() {
	fmt.Println("hello world")
	x := 42
	if x > 0 {
		return
	}
}`
	tokens := tok.EstimateTokens(code)
	if tokens <= 0 {
		t.Error("code snippet should have positive token count")
	}
	// Rough sanity: a ~100 byte snippet should be 20-60 tokens
	if tokens > 100 {
		t.Errorf("token count for small code snippet seems too high: %d", tokens)
	}
}

// ---------------------------------------------------------------------------
// 4. Language Detection - Test that language-specific compression works
// ---------------------------------------------------------------------------

func TestIntegration_LanguageDetection_Go(t *testing.T) {
	goCode := `package main

import "fmt"

func main() {
	fmt.Println("hello world")
}`

	output, _ := tok.Compress(goCode, tok.Code)
	if output == "" {
		t.Fatal("code-tier compression of Go code returned empty output")
	}
	// Go-specific structures should be preserved
	if !strings.Contains(output, "func") {
		t.Error("Go code compression should preserve 'func' keyword")
	}
}

func TestIntegration_LanguageDetection_Python(t *testing.T) {
	pythonCode := `def hello_world():
    print("Hello, World!")

class MyClass:
    def __init__(self):
        self.value = 42

    def get_value(self):
        return self.value`

	output, _ := tok.Compress(pythonCode, tok.Code)
	if output == "" {
		t.Fatal("code-tier compression of Python code returned empty output")
	}
}

func TestIntegration_LanguageDetection_TypeScript(t *testing.T) {
	tsCode := `interface User {
    name: string;
    age: number;
}

const greet = (user: User): void => {
    console.log("Hello " + user.name);
};

export default greet;`

	output, _ := tok.Compress(tsCode, tok.Code)
	if output == "" {
		t.Fatal("code-tier compression of TypeScript code returned empty output")
	}
}

func TestIntegration_LanguageDetection_Rust(t *testing.T) {
	rustCode := `fn main() {
    let greeting: &str = "Hello, world!";
    println!("{}", greeting);
}

pub fn add(a: i32, b: i32) -> i32 {
    a + b
}`

	output, _ := tok.Compress(rustCode, tok.Code)
	if output == "" {
		t.Fatal("code-tier compression of Rust code returned empty output")
	}
	if !strings.Contains(output, "fn") {
		t.Error("Rust code compression should preserve 'fn' keyword")
	}
}

func TestIntegration_LanguageDetection_SQL(t *testing.T) {
	sqlCode := `SELECT u.name, u.email, COUNT(o.id) as order_count
FROM users u
LEFT JOIN orders o ON u.id = o.user_id
WHERE u.active = true
GROUP BY u.name, u.email
ORDER BY order_count DESC;`

	output, _ := tok.Compress(sqlCode, tok.Code)
	if output == "" {
		t.Fatal("code-tier compression of SQL returned empty output")
	}
}

func TestIntegration_LanguageDetection_LogContent(t *testing.T) {
	logContent := `[2026-05-28 10:00:01] [INFO] Application started
[2026-05-28 10:00:02] [INFO] Connected to database
[2026-05-28 10:00:03] [WARN] Slow query detected (450ms)
[2026-05-28 10:00:04] [INFO] Request processed successfully
[2026-05-28 10:00:05] [ERROR] Connection timeout to service-x
[2026-05-28 10:00:06] [INFO] Retrying connection
[2026-05-28 10:00:07] [INFO] Connection restored
[2026-05-28 10:00:08] [DEBUG] Cache hit ratio: 87.3%
[2026-05-28 10:00:09] [INFO] Health check passed
[2026-05-28 10:00:10] [INFO] Metrics exported`

	output, stats := tok.Compress(logContent, tok.Log)
	if output == "" {
		t.Fatal("log-tier compression returned empty output")
	}
	if stats.OriginalTokens == 0 {
		t.Fatal("OriginalTokens should be non-zero for log content")
	}
}

func TestIntegration_LanguageDetection_ConversationContent(t *testing.T) {
	conversation := `User: Can you help me write a function to sort a list?
Assistant: Sure! Here's a quick sort implementation in Python:
def quicksort(arr):
    if len(arr) <= 1:
        return arr
    pivot = arr[len(arr) // 2]
    left = [x for x in arr if x < pivot]
    middle = [x for x in arr if x == pivot]
    right = [x for x in arr if x > pivot]
    return quicksort(left) + middle + quicksort(right)
User: Thanks! Can you also add error handling?
Assistant: Of course, I've added a try-except block around the main logic.`

	output, _ := tok.Compress(conversation, tok.Adaptive)
	if output == "" {
		t.Fatal("adaptive compression of conversation returned empty output")
	}
}

// ---------------------------------------------------------------------------
// 5. Edge Cases - Test empty input, single-word, very large input
// ---------------------------------------------------------------------------

func TestIntegration_EdgeCase_EmptyInput(t *testing.T) {
	output, stats := tok.Compress("")
	if output != "" {
		t.Errorf("empty input should return empty output, got %q", output)
	}
	if stats.OriginalTokens != 0 {
		t.Errorf("empty input should have 0 original tokens, got %d", stats.OriginalTokens)
	}
	if stats.FinalTokens != 0 {
		t.Errorf("empty input should have 0 final tokens, got %d", stats.FinalTokens)
	}
}

func TestIntegration_EdgeCase_SingleWord(t *testing.T) {
	output, stats := tok.Compress("hello")
	if output == "" {
		t.Fatal("single-word input should not return empty output")
	}
	if stats.OriginalTokens == 0 {
		t.Error("single-word input should have non-zero original tokens")
	}
}

func TestIntegration_EdgeCase_SingleCharacter(t *testing.T) {
	output, stats := tok.Compress("x")
	if output == "" {
		t.Fatal("single-character input should not return empty output")
	}
	if stats.OriginalTokens == 0 {
		t.Error("single-character input should have non-zero original tokens")
	}
}

func TestIntegration_EdgeCase_OnlyWhitespace(t *testing.T) {
	output, _ := tok.Compress("   \n\n\t  \n   ")
	// Whitespace-only input should produce some output (pipeline doesn't strip everything)
	// The important thing is it doesn't panic
	_ = output
}

func TestIntegration_EdgeCase_OnlyNewlines(t *testing.T) {
	input := strings.Repeat("\n", 100)
	output, stats := tok.Compress(input, tok.Aggressive)
	// Should not panic; output may be empty or short
	if stats.OriginalTokens < 0 {
		t.Error("OriginalTokens should not be negative")
	}
	_ = output
}

func TestIntegration_EdgeCase_VeryLargeInput(t *testing.T) {
	// 100 KB of repetitive text
	input := strings.Repeat("This is a line of text that repeats many times for testing purposes.\n", 5000)

	output, stats := tok.Compress(input, tok.Aggressive)

	if output == "" {
		t.Fatal("very large input returned empty output")
	}
	if stats.OriginalTokens == 0 {
		t.Fatal("very large input should have non-zero original tokens")
	}
	// Should achieve significant compression on repetitive input
	if len(output) >= len(input) {
		t.Error("compressor should reduce repetitive large input")
	}
}

func TestIntegration_EdgeCase_UnicodeContent(t *testing.T) {
	input := "Unicode test: éèê üöä 世界 Привет مرحبا 😀😁😂"
	output, stats := tok.Compress(input)
	if output == "" {
		t.Fatal("unicode input returned empty output")
	}
	if stats.OriginalTokens == 0 {
		t.Error("unicode input should have non-zero original tokens")
	}
}

func TestIntegration_EdgeCase_JSONContent(t *testing.T) {
	input := `{
  "name": "test-application",
  "version": "1.0.0",
  "description": "A test application for compression pipeline integration tests",
  "main": "index.js",
  "scripts": {
    "start": "node index.js",
    "test": "jest --coverage",
    "build": "webpack --mode production",
    "lint": "eslint src/**/*.js"
  },
  "dependencies": {
    "express": "^4.18.0",
    "lodash": "^4.17.21",
    "moment": "^2.29.4"
  },
  "devDependencies": {
    "jest": "^29.0.0",
    "eslint": "^8.0.0",
    "webpack": "^5.0.0"
  }
}`
	output, stats := tok.Compress(input)
	if output == "" {
		t.Fatal("JSON input returned empty output")
	}
	if stats.OriginalTokens == 0 {
		t.Error("JSON input should have non-zero original tokens")
	}
}

func TestIntegration_EdgeCase_DiffContent(t *testing.T) {
	input := `diff --git a/src/main.go b/src/main.go
index 1234567..abcdefg 100644
--- a/src/main.go
+++ b/src/main.go
@@ -10,6 +10,8 @@ import (
 	"fmt"
 	"os"
+	"log"
+	"net/http"
 )

-func main() {
+func main() {
+	http.HandleFunc("/", handler)
 	fmt.Println("hello")
+	log.Println("server starting")
 }`
	output, stats := tok.Compress(input)
	if output == "" {
		t.Fatal("diff input returned empty output")
	}
	if stats.OriginalTokens == 0 {
		t.Error("diff input should have non-zero original tokens")
	}
}

func TestIntegration_EdgeCase_RepetitiveContent(t *testing.T) {
	input := strings.Repeat("the same line over and over again\n", 200)
	output, stats := tok.Compress(input, tok.Aggressive)

	if output == "" {
		t.Fatal("repetitive input returned empty output")
	}
	// Repetitive content should compress well
	if stats.ReductionPercent < 0 {
		t.Errorf("ReductionPercent should not be negative for repetitive input: %.2f", stats.ReductionPercent)
	}
}

// Performance, Configuration, concurrency, CompactionSchema, and token
// estimation precision integration tests moved to integration_advanced_test.go.