Trie cleanup (#4)

* Remove ControlChar enum, just use ZeroWidth

* Remove CombiningMark enum, just use ZeroWidth

* Combine East Asian Full & Wide

* No need to cast to property in the trie

* Move benchmarks and compat into dedicated module

Author: clipperhouse

AGENTS.md

@@ -16,9 +16,13 @@ dependencies and are hard to cleanup.)
 If you make changes to the trie generation in internal/gen, it can be invoked
 by running `go generate` from the top package directory.
 
-## Pull Requests
+## Pull Requests and branches
 
-For PRs, you can use the gh CLI tool to retrieve or post comments.
+For PRs (pull requests), you can use the gh CLI tool to retrieve details,
+or post comments. Then, compare the current branch with main. Reviewing a PR
+and reviewing a branch are about the same, but the PR may add context.
+
+Look for bugs. Think like GitHub Copilot or Cursor BugBot.
 
 ## Comparisons to go-runewidth
 

benchmark_test.go comparison/benchmark_test.gobenchmark_test.go renamed to comparison/benchmark_test.go

@@ -1,10 +1,11 @@
-package displaywidth
+package comparison
 
 import (
 	"strings"
 	"testing"
 
-	"github.com/clipperhouse/displaywidth/internal/testdata"
+	"github.com/clipperhouse/displaywidth"
+	"github.com/clipperhouse/displaywidth/comparison/testdata"
 	"github.com/mattn/go-runewidth"
 )
 
@@ -98,7 +99,7 @@ func BenchmarkStringDefault(b *testing.B) {
 		for i := 0; i < b.N; i++ {
 			for _, tc := range testCases {
 				// Test with default settings (eastAsianWidth=false, strictEmojiNeutral=false)
-				_ = String(tc.Input)
+				_ = displaywidth.String(tc.Input)
 			}
 		}
 
@@ -134,7 +135,7 @@ func BenchmarkStringDefault(b *testing.B) {
 }
 
 func BenchmarkString_EAW(b *testing.B) {
-	options := Options{
+	options := displaywidth.Options{
 		EastAsianWidth:     true,
 		StrictEmojiNeutral: false,
 	}
@@ -192,7 +193,7 @@ func BenchmarkString_EAW(b *testing.B) {
 
 // BenchmarkString_StrictEmoji benchmarks our package with strict emoji neutral
 func BenchmarkString_StrictEmoji(b *testing.B) {
-	options := Options{
+	options := displaywidth.Options{
 		EastAsianWidth:     false,
 		StrictEmojiNeutral: true,
 	}
@@ -262,7 +263,7 @@ func BenchmarkString_ASCII(b *testing.B) {
 		b.ResetTimer()
 		for i := 0; i < b.N; i++ {
 			for _, s := range asciiStrings {
-				_ = String(s)
+				_ = displaywidth.String(s)
 			}
 		}
 
@@ -310,7 +311,7 @@ func BenchmarkString_Unicode(b *testing.B) {
 		b.ResetTimer()
 		for i := 0; i < b.N; i++ {
 			for _, s := range unicodeStrings {
-				_ = String(s)
+				_ = displaywidth.String(s)
 			}
 		}
 
@@ -354,7 +355,7 @@ func BenchmarkStringWidth_Emoji(b *testing.B) {
 		b.ResetTimer()
 		for i := 0; i < b.N; i++ {
 			for _, s := range emojiStrings {
-				_ = String(s)
+				_ = displaywidth.String(s)
 			}
 		}
 
@@ -396,7 +397,7 @@ func BenchmarkString_Mixed(b *testing.B) {
 		b.ResetTimer()
 		for i := 0; i < b.N; i++ {
 			for _, s := range mixedStrings {
-				_ = String(s)
+				_ = displaywidth.String(s)
 			}
 		}
 
@@ -441,7 +442,7 @@ func BenchmarkString_ControlChars(b *testing.B) {
 		b.ResetTimer()
 		for i := 0; i < b.N; i++ {
 			for _, s := range controlStrings {
-				_ = String(s)
+				_ = displaywidth.String(s)
 			}
 		}
 
@@ -499,7 +500,7 @@ func BenchmarkRuneDefault(b *testing.B) {
 		b.ResetTimer()
 		for i := 0; i < b.N; i++ {
 			for _, r := range testRunes {
-				_ = Rune(r)
+				_ = displaywidth.Rune(r)
 			}
 		}
 
@@ -529,7 +530,7 @@ func BenchmarkRuneDefault(b *testing.B) {
 
 // BenchmarkRuneWidth_EAW benchmarks rune width with East Asian Width option
 func BenchmarkRuneWidth_EAW(b *testing.B) {
-	options := Options{
+	options := displaywidth.Options{
 		EastAsianWidth:     true,
 		StrictEmojiNeutral: false,
 	}
@@ -591,7 +592,7 @@ func BenchmarkRuneWidth_ASCII(b *testing.B) {
 		b.ResetTimer()
 		for i := 0; i < b.N; i++ {
 			for _, r := range asciiRunes {
-				_ = Rune(r)
+				_ = displaywidth.Rune(r)
 			}
 		}
 

comparison/compatibility_test.go

@@ -0,0 +1,47 @@
+package comparison
+
+import (
+	"testing"
+	"unicode"
+
+	"github.com/clipperhouse/displaywidth"
+	"github.com/mattn/go-runewidth"
+)
+
+// Investigate differences between displaywidth and go-runewidth.
+// Not meant to be a real test, more of a tool, but good to
+// fail if something changes in the future.
+func TestCompatibility(t *testing.T) {
+	t.Skip("skipping compatibility test")
+	if unicode.Version < "15" {
+		// We only care about Unicode 15 and above,
+		// which I believe was Go version 1.21.
+		return
+	}
+
+	for r := rune(0); r <= unicode.MaxRune; r++ {
+		w1 := displaywidth.Rune(r)
+		w2 := runewidth.RuneWidth(r)
+		if w1 != w2 {
+			if unicode.Is(unicode.Mn, r) {
+				// these are in the trie, known
+				// we will return width 0,
+				// go-runewidth may return width 1
+				continue
+			}
+			if unicode.Is(unicode.Cf, r) {
+				// these are in the trie, known
+				// we will return width 0,
+				// go-runewidth may return width 1
+				continue
+			}
+			if unicode.Is(unicode.Mc, r) {
+				// these are deliberately excluded from the trie, known
+				// we will return width 1,
+				// go-runewidth may return width 0
+				continue
+			}
+			t.Errorf("%#x: runewidth is %d, displaywidth is %d, difference is %d", r, w2, w1, w2-w1)
+		}
+	}
+}

comparison/go.mod

@@ -0,0 +1,10 @@
+module github.com/clipperhouse/displaywidth/comparison
+
+go 1.18
+
+require (
+	github.com/clipperhouse/displaywidth v0.2.0
+	github.com/mattn/go-runewidth v0.0.19
+)
+
+require github.com/clipperhouse/uax29/v2 v2.2.0 // indirect

comparison/go.sum

@@ -0,0 +1,6 @@
+github.com/clipperhouse/displaywidth v0.2.0 h1:rgsptyf+vdcUj2E5DADgM6nMMlcj5dOmpJqWeO4t/0Q=
+github.com/clipperhouse/displaywidth v0.2.0/go.mod h1:0pW8rEIr4/Pjji52WJsYnMpoj/8Cse3d703ZVdU6vBI=
+github.com/clipperhouse/uax29/v2 v2.2.0 h1:ChwIKnQN3kcZteTXMgb1wztSgaU+ZemkgWdohwgs8tY=
+github.com/clipperhouse/uax29/v2 v2.2.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
+github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw=
+github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=

internal/testdata/UTF-8-test.txt comparison/testdata/UTF-8-test.txtinternal/testdata/UTF-8-test.txt renamed to comparison/testdata/UTF-8-test.txt

@@ -4,5 +4,4 @@ go 1.18
 
 require (
 	github.com/clipperhouse/uax29/v2 v2.2.0
-	github.com/mattn/go-runewidth v0.0.19
 )