Charset: Limit _wp_scan_utf8() ASCII scan to remaining code points.

The ASCII fast-path in `_wp_scan_utf8()` uses `strspn()` to skip past ASCII bytes. When a code point limit was provided without a byte limit, the scan would include the rest of the input even when there was a code point limit. Because ASCII characters are single-byte code points, the fast-path scan length can be bounded by the number of remaining code points. This improves performance when working with some large documents.

Developed in WordPress#12214.

Follow-up to [60768].

Props jonsurrell, dmsnell, zieladam.
Fixes #65483. See #63863.


git-svn-id: https://develop.svn.wordpress.org/trunk@62523 602fd350-edb4-49c9-b593-d223f7449a82

Author: sirreal

src/wp-includes/compat-utf8.php

@@ -65,7 +65,7 @@ function _wp_scan_utf8( string $bytes, int &$at, int &$invalid_length, ?int $max
 			"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" .
 			" !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f",
 			$i,
-			$end - $i
+			min( $end - $i, $max_count - $count )
 		);
 
 		if ( $count + $ascii_byte_count >= $max_count ) {

tests/phpunit/tests/compat/wpUtf8CodePointSpan.php

@@ -0,0 +1,103 @@
+<?php
+/**
+ * Unit tests covering fallback UTF-8 code-point span detection.
+ *
+ * @package    WordPress
+ * @subpackage Charset
+ *
+ * @group      compat
+ *
+ * @covers ::_wp_utf8_codepoint_span()
+ */
+class Tests_Compat_wpUtf8CodePointSpan extends WP_UnitTestCase {
+	/**
+	 * Ensures that the span accounts for the requested number of code points.
+	 *
+	 * @dataProvider data_codepoint_spans
+	 *
+	 * @ticket 65483
+	 * @ticket 63863
+	 *
+	 * @param string $text
+	 * @param int    $byte_offset
+	 * @param int    $max_code_points
+	 * @param int    $expected_span
+	 * @param int    $expected_found
+	 */
+	public function test_finds_codepoint_spans( string $text, int $byte_offset, int $max_code_points, int $expected_span, int $expected_found ) {
+		$found_code_points = null;
+
+		$this->assertSame(
+			$expected_span,
+			_wp_utf8_codepoint_span( $text, $byte_offset, $max_code_points, $found_code_points ),
+			'Should have found the expected byte span.'
+		);
+
+		$this->assertSame(
+			$expected_found,
+			$found_code_points,
+			'Should have reported the expected number of code points.'
+		);
+	}
+
+	/**
+	 * Data provider.
+	 *
+	 * @return array<string, array{0: string, 1: int, 2: int, 3: int, 4: int}>
+	 */
+	public static function data_codepoint_spans() {
+		$long_ascii_run = str_repeat( 'a', 1024 );
+
+		return array(
+			'zero code point budget'                  => array(
+				'abcdef',
+				0,
+				0,
+				0,
+				0,
+			),
+			'long ASCII run at start'                 => array(
+				$long_ascii_run,
+				0,
+				5,
+				5,
+				5,
+			),
+			'long ASCII run from non-zero offset'     => array(
+				"zz{$long_ascii_run}",
+				2,
+				5,
+				5,
+				5,
+			),
+			'multibyte character before the boundary' => array(
+				"ab\u{1F170}cd",
+				0,
+				2,
+				2,
+				2,
+			),
+			'multibyte character at the boundary'     => array(
+				"ab\u{1F170}cd",
+				0,
+				3,
+				strlen( "ab\u{1F170}" ),
+				3,
+			),
+			'invalid span after the boundary'         => array(
+				"ab\xF0\x9Fzz",
+				0,
+				2,
+				2,
+				2,
+			),
+			'invalid span at the boundary'            => array(
+				"ab\xF0\x9Fzz",
+				0,
+				3,
+				4,
+				3,
+			),
+		);
+	}
+}