contextjson: Fix crash

Author: nekohasekai

common/json/internal/contextjson/context_test.go

@@ -292,6 +292,102 @@ func TestTrailingCommaToken(t *testing.T) {
 	}
 }
 
+func TestTokenMoreAfterCommaAcrossRefill(t *testing.T) {
+	t.Parallel()
+	// Positions the comma at decoder buffer offset 500 and the next value
+	// beyond the initial 512-byte read.
+	value := strings.Repeat("a", 497)
+	decoder := json.NewDecoder(strings.NewReader(`["` + value + `",` + strings.Repeat(" ", 11) + `2]`))
+	token, err := decoder.Token()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if token != json.Delim('[') {
+		t.Fatalf("start token = %v", token)
+	}
+	if !decoder.More() {
+		t.Fatal("expected first array value")
+	}
+	token, err = decoder.Token()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if token != value {
+		t.Fatalf("first value = %#v", token)
+	}
+	if !decoder.More() {
+		t.Fatal("expected second array value")
+	}
+	token, err = decoder.Token()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if token != float64(2) {
+		t.Fatalf("second value = %#v", token)
+	}
+	if decoder.More() {
+		t.Fatal("expected end of array")
+	}
+	token, err = decoder.Token()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if token != json.Delim(']') {
+		t.Fatalf("end token = %v", token)
+	}
+}
+
+func TestTrailingCommaTokenAcrossRefill(t *testing.T) {
+	t.Parallel()
+	value := strings.Repeat("a", 497)
+	tests := []struct {
+		name   string
+		suffix string
+	}{
+		{
+			name:   "spaces",
+			suffix: strings.Repeat(" ", 11) + `]`,
+		},
+		{
+			name:   "line comment",
+			suffix: strings.Repeat(" ", 11) + "// trailing\n]",
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			t.Parallel()
+			decoder := json.NewDecoder(strings.NewReader(`["` + value + `",` + test.suffix))
+			token, err := decoder.Token()
+			if err != nil {
+				t.Fatal(err)
+			}
+			if token != json.Delim('[') {
+				t.Fatalf("start token = %v", token)
+			}
+			if !decoder.More() {
+				t.Fatal("expected array value")
+			}
+			token, err = decoder.Token()
+			if err != nil {
+				t.Fatal(err)
+			}
+			if token != value {
+				t.Fatalf("value = %#v", token)
+			}
+			if decoder.More() {
+				t.Fatal("expected trailing comma to end array")
+			}
+			token, err = decoder.Token()
+			if err != nil {
+				t.Fatal(err)
+			}
+			if token != json.Delim(']') {
+				t.Fatalf("end token = %v", token)
+			}
+		})
+	}
+}
+
 func TestTrailingComma(t *testing.T) {
 	t.Parallel()
 	var array []int

common/json/internal/contextjson/stream.go

@@ -504,10 +504,7 @@ func (dec *Decoder) More() bool {
 		return false
 	}
 	if c == ',' {
-		scanp := dec.scanp
-		dec.scanp++
-		c, err = dec.peekNoRefill()
-		dec.scanp = scanp
+		c, _, err = dec.peekFrom(dec.scanp + 1)
 		if err != nil {
 			return false
 		}
@@ -519,71 +516,62 @@ func (dec *Decoder) More() bool {
 }
 
 func (dec *Decoder) peek() (byte, error) {
+	c, scanp, err := dec.peekFrom(dec.scanp)
+	if err != nil {
+		return 0, err
+	}
+	dec.scanp = scanp
+	return c, nil
+}
+
+func (dec *Decoder) peekFrom(scanp int) (byte, int, error) {
 	var err error
 	for {
 	Buffer:
-		for i := dec.scanp; i < len(dec.buf); {
-			c := dec.buf[i]
+		for scanp < len(dec.buf) {
+			c := dec.buf[scanp]
 			if isSpace(c) {
-				i++
+				scanp++
 				continue
 			}
 			if c == '#' {
-				next, ok := dec.skipLineCommentInBuffer(i + 1)
+				next, ok := dec.skipLineCommentInBuffer(scanp + 1)
 				if !ok {
 					break
 				}
-				i = next
+				scanp = next
 				continue
 			}
 			if c == '/' {
-				if i+1 >= len(dec.buf) {
+				if scanp+1 >= len(dec.buf) {
 					break Buffer
 				}
-				switch dec.buf[i+1] {
+				switch dec.buf[scanp+1] {
 				case '/':
-					next, ok := dec.skipLineCommentInBuffer(i + 2)
+					next, ok := dec.skipLineCommentInBuffer(scanp + 2)
 					if !ok {
 						break Buffer
 					}
-					i = next
+					scanp = next
 					continue
 				case '*':
-					next, ok := dec.skipBlockCommentInBuffer(i + 2)
+					next, ok := dec.skipBlockCommentInBuffer(scanp + 2)
 					if !ok {
 						break Buffer
 					}
-					i = next
+					scanp = next
 					continue
 				}
 			}
-			dec.scanp = i
-			return c, nil
+			return c, scanp, nil
 		}
 		// buffer has been scanned, now report any error
 		if err != nil {
-			return 0, err
-		}
-		err = dec.refill()
-	}
-}
-
-func (dec *Decoder) peekNoRefill() (byte, error) {
-	var err error
-	for {
-		for i := dec.scanp; i < len(dec.buf); i++ {
-			c := dec.buf[i]
-			if isSpace(c) {
-				continue
-			}
-			dec.scanp = i
-			return c, nil
-		}
-		// buffer has been scanned, now report any error
-		if err != nil {
-			return 0, err
+			return 0, 0, err
 		}
+		oldScanp := dec.scanp
 		err = dec.refill()
+		scanp -= oldScanp
 	}
 }