perf: Speed up UTF-16 decoding (#138)

* speed up UTF-16 decoding

* fix UTF-16 slice length

Author: rabbitstack

pkg/kevent/kparams/readers.go

@@ -22,8 +22,7 @@
 package kparams
 
 import (
-	"syscall"
-	"unicode/utf16"
+	"github.com/rabbitstack/fibratus/pkg/syscall/utf16"
 	"unsafe"
 )
 
@@ -79,7 +78,7 @@ func ReadUTF16String(buf uintptr, offset, length uint16) (string, uint16) {
 		return "", 0
 	}
 	s := (*[1<<30 - 1]uint16)(unsafe.Pointer(buf + uintptr(offset)))[: length-offset : length-offset]
-	return syscall.UTF16ToString(s), uint16(len(s) + 2)
+	return utf16.Decode(s[:len(s)/2-1-2]), uint16(len(s) + 2)
 }
 
 // ConsumeUTF16String reads the byte slice with UTF16-encoded string
@@ -89,13 +88,13 @@ func ConsumeUTF16String(buf uintptr, offset, length uint16) string {
 		return ""
 	}
 	s := (*[1<<30 - 1]uint16)(unsafe.Pointer(buf + uintptr(offset)))[: length-offset : length-offset]
-	return string(utf16.Decode(s[:len(s)/2-1]))
+	return utf16.Decode(s[:len(s)/2-1])
 }
 
 // ReadSID reads the security identifier from the provided buffer.
 func ReadSID(buf uintptr, offset uint16) ([]byte, uint16) {
 	// this is a Security Token which can be null and takes 4 bytes.
-	// Otherwise it is an 8 byte structure (TOKEN_USER) followed by SID,
+	// Otherwise, it is an 8 byte structure (TOKEN_USER) followed by SID,
 	// which is variable size depending on the 2nd byte in the SID
 	sid := ReadUint32(buf, offset)
 	if sid == 0 {

pkg/syscall/utf16/string.go

@@ -25,6 +25,7 @@ import (
 	"reflect"
 	"syscall"
 	"unicode/utf16"
+	"unicode/utf8"
 	"unsafe"
 )
 
@@ -74,3 +75,37 @@ func PtrToString(p unsafe.Pointer) string {
 	// Remove trailing NUL and decode into a Go string.
 	return string(utf16.Decode(s[:len(s)-1]))
 }
+
+const (
+	// 0xd800-0xdc00 encodes the high 10 bits of a pair.
+	surr1 = 0xd800
+	// 0xdc00-0xe000 encodes the low 10 bits of a pair.
+	surr2 = 0xdc00
+)
+
+func isHighSurrogate(r rune) bool { return r >= surr1 && r <= 0xdbff }
+func isLowSurrogate(r rune) bool  { return r >= surr2 && r <= 0xdfff }
+
+// Decode decodes the UTF16-encoded string to UTF-8 string. This function
+// exhibits much better performance than the standard library counterpart.
+// All credits go to: https://gist.github.com/skeeto/09f1410183d246f9b18cba95c4e602f0
+func Decode(p []uint16) string {
+	s := make([]byte, 0, 2*len(p))
+	for i := 0; i < len(p); i++ {
+		r := rune(0xfffd)
+		r1 := rune(p[i])
+		if isHighSurrogate(r1) {
+			if i+1 < len(p) {
+				r2 := rune(p[i+1])
+				if isLowSurrogate(r2) {
+					i++
+					r = 0x10000 + (r1-surr1)<<10 + (r2 - surr2)
+				}
+			}
+		} else if !isLowSurrogate(r) {
+			r = r1
+		}
+		s = utf8.AppendRune(s, r)
+	}
+	return string(s)
+}

pkg/syscall/utf16/string_test.go

@@ -0,0 +1,70 @@
+/*
+ * Copyright 2021-2022 by Nedim Sabic Sabic
+ * https://www.fibratus.io
+ * All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package utf16
+
+import (
+	"github.com/stretchr/testify/require"
+	"math/rand"
+	"testing"
+	"unicode/utf16"
+)
+
+func TestDecode(t *testing.T) {
+	for i := 0; i < 24; i++ {
+		buf := genbuf(1 << i)
+		w := string(utf16.Decode(buf))
+		g := Decode(buf)
+		if w != g {
+			t.Errorf("mismatch on 1<<%d", i)
+		}
+	}
+	s := []rune("Do you want café?")
+	encoded := utf16.Encode(s)
+	require.Equal(t, "Do you want café?", Decode(encoded))
+}
+
+func BenchmarkStdlibDecode(b *testing.B) {
+	b.ReportAllocs()
+	b.StopTimer()
+	buf := genbuf(b.N)
+	b.StartTimer()
+	_ = string(utf16.Decode(buf))
+}
+
+func BenchmarkDecode(b *testing.B) {
+	b.ReportAllocs()
+	b.StopTimer()
+	buf := genbuf(b.N)
+	b.StartTimer()
+	_ = Decode(buf)
+}
+
+func genbuf(n int) []uint16 {
+	r := rand.New(rand.NewSource(int64(n)))
+	buf := make([]rune, n)
+	for i := 0; i < n; i++ {
+		// simulate mostly-ASCII
+		if r.Intn(100) == 0 {
+			buf[i] = rune(r.Intn(0x10ffff + 1))
+		} else {
+			buf[i] = rune(r.Intn(1 << 7))
+		}
+	}
+	return utf16.Encode(buf)
+}