Merge pull request #37 from Basekick-Labs/fix/float32-double-narrowing

Fix/float32 double narrowing

Author: xe-nvdk

.github/workflows/build.yml

@@ -2,26 +2,26 @@ name: Go
 
 on:
   push:
-    branches: [v5]
+    branches: [v6]
   pull_request:
-    branches: [v5]
+    branches: [v6]
 
 jobs:
   build:
-    name: build
+    name: test (Go ${{ matrix.go-version }})
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go-version: [1.21.x]
+        go-version: ['1.25.x', '1.26.x']
 
     steps:
-      - name: Set up ${{ matrix.go-version }}
-        uses: actions/setup-go@v2
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Go ${{ matrix.go-version }}
+        uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.go-version }}
 
-      - name: Checkout code
-        uses: actions/checkout@v2
-
       - name: Test
         run: make test

.github/workflows/commitlint.yml

@@ -5,7 +5,7 @@ jobs:
   commitlint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: wagoid/commitlint-github-action@v4
+      - uses: wagoid/commitlint-github-action@v6

.github/workflows/release.yml

@@ -9,9 +9,9 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - uses: ncipollo/release-action@v1
         with:
           body:
             Please refer to
-            [CHANGELOG.md](https://github.com/vmihailenco/msgpack/blob/v5/CHANGELOG.md) for details
+            [CHANGELOG.md](https://github.com/Basekick-Labs/msgpack/blob/v6/CHANGELOG.md) for details

CHANGELOG.md

@@ -1,3 +1,29 @@
+## v6 (Basekick-Labs fork)
+
+### Performance
+
+- **decode:** zero-allocation byte-slice reader for `Unmarshal()` — replaces `bytes.NewReader` + `bufio.NewReader` with a direct `byteSliceReader`, eliminating 2 allocations per call (~21% faster decode, ~50% less memory)
+- **decode:** `*interface{}` fast path in `Decode()` — skips `reflect.ValueOf` for the most common `Unmarshal(b, &interface{})` pattern (~14% faster)
+- **encode:** pooled byte buffer in `Marshal()` — replaces per-call `bytes.Buffer` with a reusable `[]byte` embedded in the pooled `Encoder` struct
+- **encode:** `byteSliceWriter` for `Marshal()` path — native `WriteByte` implementation eliminates per-byte heap allocation
+- **encode:** `byteWriter.WriteByte` scratch fix for streaming path — uses `[1]byte` scratch instead of allocating `[]byte{c}`
+- **encode:** `Encode()` fast paths for `map[string]interface{}` and `[]interface{}` — bypasses `reflect.ValueOf` + sync.Map encoder lookup
+
+### Bug Fixes
+
+- **decode:** cap `decodeSlice()` allocation at `sliceAllocLimit` (1M) to prevent OOM from malicious payloads ([#1](https://github.com/Basekick-Labs/msgpack/issues/1))
+- **decode:** cap `DecodeMap()` allocation at `maxMapSize` (1M) — same OOM vector for `map[string]interface{}` path
+- **decode:** fix `disableAllocLimitFlag` check in `decodeSliceValue` — `!= 1` was always true because the flag value is `1 << 3 = 8`, so the alloc limit in `growSliceValue()` was never applied
+- **decode:** fix error message in `DecodeFloat64` — said "decoding float32" instead of "decoding float64" ([#13](https://github.com/Basekick-Labs/msgpack/issues/13))
+
+### Chores
+
+- Modernize GitHub Actions (checkout@v4, setup-go@v5)
+- Go version matrix: 1.25.x, 1.26.x
+- Bump `go.mod` to Go 1.26
+
+---
+
 ## [5.4.1](https://github.com/vmihailenco/msgpack/compare/v5.4.0...v5.4.1) (2023-10-26)
 
 

README.md

@@ -1,19 +1,33 @@
 # MessagePack encoding for Golang
 
-[![Build Status](https://travis-ci.org/vmihailenco/msgpack.svg)](https://travis-ci.org/vmihailenco/msgpack)
+[![Build Status](https://github.com/Basekick-Labs/msgpack/actions/workflows/build.yml/badge.svg?branch=v6)](https://github.com/Basekick-Labs/msgpack/actions/workflows/build.yml)
 [![PkgGoDev](https://pkg.go.dev/badge/github.com/vmihailenco/msgpack/v5)](https://pkg.go.dev/github.com/vmihailenco/msgpack/v5)
-[![Documentation](https://img.shields.io/badge/msgpack-documentation-informational)](https://msgpack.uptrace.dev/)
-[![Chat](https://discordapp.com/api/guilds/752070105847955518/widget.png)](https://discord.gg/rWtp5Aj)
+[![Discord](https://img.shields.io/badge/discord-chat-5865F2?logo=discord&logoColor=white)](https://discord.gg/nxnWfUxsdm)
 
-> msgpack is brought to you by :star: [**uptrace/uptrace**](https://github.com/uptrace/uptrace).
-> Uptrace is an [open source APM](https://uptrace.dev/get/open-source-apm.html) and blazingly fast
-> [distributed tracing tool](https://get.uptrace.dev/compare/distributed-tracing-tools.html) powered
-> by OpenTelemetry and ClickHouse. Give it a star as well!
+> A performance-optimized fork of [vmihailenco/msgpack/v5](https://github.com/vmihailenco/msgpack),
+> maintained by [Basekick Labs](https://github.com/Basekick-Labs). Built for
+> [Arc](https://github.com/Basekick-Labs/arc), a high-performance time-series database.
+> The upstream module path is preserved for drop-in compatibility.
+
+## What's New in v6
+
+**Decode** — ~21% faster, ~50% less memory:
+- Zero-allocation byte-slice reader for `Unmarshal()`
+- `*interface{}` fast path bypasses reflect for the most common decode pattern
+
+**Encode** — ~12% faster, ~43% fewer allocations:
+- Pooled byte buffer in `Marshal()` eliminates per-call `bytes.Buffer`
+- Native `WriteByte` on the Marshal path removes per-byte heap allocations
+- Type-switch fast paths for `map[string]interface{}` and `[]interface{}`
+
+**Security:**
+- OOM protection: slice and map allocations from untrusted input are capped at 1M elements
+
+See [CHANGELOG.md](CHANGELOG.md) for full details.
 
 ## Resources
 
-- [Documentation](https://msgpack.uptrace.dev)
-- [Chat](https://discord.gg/rWtp5Aj)
+- [Discord](https://discord.gg/nxnWfUxsdm)
 - [Reference](https://pkg.go.dev/github.com/vmihailenco/msgpack/v5)
 - [Examples](https://pkg.go.dev/github.com/vmihailenco/msgpack/v5#pkg-examples)
 
@@ -53,7 +67,7 @@ msgpack supports 2 last Go versions and requires support for
 go mod init github.com/my/repo
 ```
 
-And then install msgpack/v5 (note _v5_ in the import; omitting it is a popular mistake):
+And then install msgpack (the module path is unchanged from upstream for drop-in compatibility):
 
 ```shell
 go get github.com/vmihailenco/msgpack/v5
@@ -84,17 +98,10 @@ func ExampleMarshal() {
 }
 ```
 
-## See also
-
-- [Golang ORM](https://github.com/uptrace/bun) for PostgreSQL, MySQL, MSSQL, and SQLite
-- [Golang PostgreSQL](https://bun.uptrace.dev/postgres/)
-- [Golang HTTP router](https://github.com/uptrace/bunrouter)
-- [Golang ClickHouse ORM](https://github.com/uptrace/go-clickhouse)
-
 ## Contributors
 
 Thanks to all the people who already contributed!
 
-<a href="https://github.com/vmihailenco/msgpack/graphs/contributors">
-  <img src="https://contributors-img.web.app/image?repo=vmihailenco/msgpack" />
+<a href="https://github.com/Basekick-Labs/msgpack/graphs/contributors">
+  <img src="https://contributors-img.web.app/image?repo=Basekick-Labs/msgpack" />
 </a>

byteslice_reader.go

@@ -0,0 +1,46 @@
+package msgpack
+
+import (
+	"errors"
+	"io"
+)
+
+// byteSliceReader implements bufReader (io.Reader + io.ByteScanner) for
+// zero-allocation decoding from a []byte. When Unmarshal is called with
+// a complete byte slice, this avoids the overhead of bytes.NewReader +
+// bufio.NewReader (2 allocations + 4KB buffer + interface dispatch per byte).
+type byteSliceReader struct {
+	data []byte
+	pos  int
+}
+
+func (r *byteSliceReader) reset(data []byte) {
+	r.data = data
+	r.pos = 0
+}
+
+func (r *byteSliceReader) ReadByte() (byte, error) {
+	if r.pos >= len(r.data) {
+		return 0, io.EOF
+	}
+	c := r.data[r.pos]
+	r.pos++
+	return c, nil
+}
+
+func (r *byteSliceReader) UnreadByte() error {
+	if r.pos <= 0 {
+		return errors.New("msgpack: at beginning of input")
+	}
+	r.pos--
+	return nil
+}
+
+func (r *byteSliceReader) Read(p []byte) (int, error) {
+	if r.pos >= len(r.data) {
+		return 0, io.EOF
+	}
+	n := copy(p, r.data[r.pos:])
+	r.pos += n
+	return n, nil
+}

byteslice_writer.go

@@ -0,0 +1,18 @@
+package msgpack
+
+// byteSliceWriter is a zero-allocation writer that appends directly to a
+// []byte buffer. It implements the writer interface (io.Writer + WriteByte).
+// Used by Marshal() to avoid allocating a bytes.Buffer on every call.
+type byteSliceWriter struct {
+	buf *[]byte
+}
+
+func (w *byteSliceWriter) Write(p []byte) (int, error) {
+	*w.buf = append(*w.buf, p...)
+	return len(p), nil
+}
+
+func (w *byteSliceWriter) WriteByte(c byte) error {
+	*w.buf = append(*w.buf, c)
+	return nil
+}

decode.go

@@ -2,7 +2,6 @@ package msgpack
 
 import (
 	"bufio"
-	"bytes"
 	"errors"
 	"fmt"
 	"io"
@@ -46,6 +45,10 @@ func GetDecoder() *Decoder {
 func PutDecoder(dec *Decoder) {
 	dec.r = nil
 	dec.s = nil
+	dec.bsr.data = nil
+	if dec.buf != nil {
+		dec.buf = dec.buf[:0]
+	}
 	decPool.Put(dec)
 }
 
@@ -56,7 +59,7 @@ func PutDecoder(dec *Decoder) {
 func Unmarshal(data []byte, v interface{}) error {
 	dec := GetDecoder()
 	dec.UsePreallocateValues(true)
-	dec.Reset(bytes.NewReader(data))
+	dec.ResetBytes(data)
 	err := dec.Decode(v)
 
 	PutDecoder(dec)
@@ -68,6 +71,7 @@ func Unmarshal(data []byte, v interface{}) error {
 type Decoder struct {
 	r          io.Reader
 	s          io.ByteScanner
+	bsr        byteSliceReader
 	mapDecoder func(*Decoder) (interface{}, error)
 	structTag  string
 	buf        []byte
@@ -126,6 +130,19 @@ func (d *Decoder) ResetReader(r io.Reader) {
 	}
 }
 
+// ResetBytes is like Reset but optimized for decoding from a byte slice.
+// It avoids allocating bytes.NewReader and bufio.NewReader by using an
+// embedded byte-slice reader with zero-copy sub-slicing.
+func (d *Decoder) ResetBytes(data []byte) {
+	d.bsr.reset(data)
+	d.r = &d.bsr
+	d.s = &d.bsr
+	d.mapDecoder = nil
+	d.flags = 0
+	d.structTag = ""
+	d.dict = nil
+}
+
 func (d *Decoder) SetMapDecoder(fn func(*Decoder) (interface{}, error)) {
 	d.mapDecoder = fn
 }
@@ -194,6 +211,11 @@ func (d *Decoder) Buffered() io.Reader {
 func (d *Decoder) Decode(v interface{}) error {
 	var err error
 	switch v := v.(type) {
+	case *interface{}:
+		if v != nil && *v == nil {
+			*v, err = d.decodeInterfaceCond()
+			return err
+		}
 	case *string:
 		if v != nil {
 			*v, err = d.DecodeString()
@@ -625,6 +647,19 @@ func (d *Decoder) readFull(b []byte) error {
 }
 
 func (d *Decoder) readN(n int) ([]byte, error) {
+	// Fast path: byte-slice reader — zero-copy sub-slice of input buffer.
+	if d.bsr.data != nil {
+		if d.bsr.pos+n > len(d.bsr.data) {
+			return nil, io.ErrUnexpectedEOF
+		}
+		b := d.bsr.data[d.bsr.pos : d.bsr.pos+n]
+		d.bsr.pos += n
+		if d.rec != nil {
+			d.rec = append(d.rec, b...)
+		}
+		return b, nil
+	}
+
 	var err error
 	if d.flags&disableAllocLimitFlag != 0 {
 		d.buf, err = readN(d.r, d.buf, n)
@@ -635,7 +670,6 @@ func (d *Decoder) readN(n int) ([]byte, error) {
 		return nil, err
 	}
 	if d.rec != nil {
-		// TODO: read directly into d.rec?
 		d.rec = append(d.rec, d.buf...)
 	}
 	return d.buf, nil

decode_map.go

@@ -52,7 +52,27 @@ func (d *Decoder) decodeMapDefault() (interface{}, error) {
 	if d.mapDecoder != nil {
 		return d.mapDecoder(d)
 	}
-	return d.DecodeMap()
+
+	n, err := d.DecodeMapLen()
+	if err != nil {
+		return nil, err
+	}
+	if n == -1 {
+		return nil, nil
+	}
+	if n == 0 {
+		return make(map[string]interface{}), nil
+	}
+
+	code, err := d.PeekCode()
+	if err != nil {
+		return nil, err
+	}
+
+	if msgpcode.IsString(code) {
+		return d.decodeMapStringInterfaceN(n)
+	}
+	return d.decodeTypedMapN(n)
 }
 
 // DecodeMapLen decodes map length. Length is -1 when map is nil.
@@ -152,12 +172,19 @@ func (d *Decoder) DecodeMap() (map[string]interface{}, error) {
 	if err != nil {
 		return nil, err
 	}
-
 	if n == -1 {
 		return nil, nil
 	}
+	return d.decodeMapStringInterfaceN(n)
+}
 
-	m := make(map[string]interface{}, n)
+func (d *Decoder) decodeMapStringInterfaceN(n int) (map[string]interface{}, error) {
+	ln := n
+	if d.flags&disableAllocLimitFlag == 0 && ln > maxMapSize {
+		ln = maxMapSize
+	}
+
+	m := make(map[string]interface{}, ln)
 
 	for i := 0; i < n; i++ {
 		mk, err := d.DecodeString()
@@ -213,7 +240,10 @@ func (d *Decoder) DecodeTypedMap() (interface{}, error) {
 	if n <= 0 {
 		return nil, nil
 	}
+	return d.decodeTypedMapN(n)
+}
 
+func (d *Decoder) decodeTypedMapN(n int) (interface{}, error) {
 	key, err := d.decodeInterfaceCond()
 	if err != nil {
 		return nil, err