Skip to content

Commit db48dcf

Browse files
sirrealsirreal
committed
HTML API: Replace locale-dependent ctype check in HTML decoder.
`ctype_alnum()` behaves differently depending on the host system and locale. Replace it with a direct ASCII byte comparison that behaves consistently across environments. Developed in WordPress#12286. Props jonsurrell, dmsnell. See #65372. git-svn-id: https://develop.svn.wordpress.org/trunk@62573 602fd350-edb4-49c9-b593-d223f7449a82
1 parent cb14d22 commit db48dcf

2 files changed

Lines changed: 207 additions & 23 deletions

File tree

‎src/wp-includes/html-api/class-wp-html-decoder.php‎

Lines changed: 49 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -367,34 +367,60 @@ public static function read_character_reference( $context, $text, $at = 0, &$mat
367367

368368
$after_name = $name_at + $name_length;
369369

370-
// If the match ended with a semicolon then it should always be decoded.
371-
if ( ';' === $text[ $name_at + $name_length - 1 ] ) {
372-
$match_byte_length = $after_name - $at;
373-
return $replacement;
374-
}
375-
376-
/*
377-
* At this point though there's a match for an entry in the named
378-
* character reference table but the match doesn't end in `;`.
379-
* It may be allowed if it's followed by something unambiguous.
370+
/**
371+
* For historical reasons, a matched named character reference is left as literal
372+
* text (its decoded replacement is not used) when all of the following hold:
373+
*
374+
* 1. It was matched in attribute context.
375+
* 2. The match does not end in U+003B SEMICOLON (;) — i.e. it is one of the
376+
* legacy forms recognized without a trailing semicolon.
377+
* 3. The next input character is U+003D EQUALS SIGN (=) or an ASCII alphanumeric.
378+
*
379+
* Some illustrative examples follow. Note that both `not` and `not;` appear in the
380+
* named character references list. References start with `&` and typically end with
381+
* `;`, but the legacy forms are recognized without one.
382+
*
383+
* - In _data context_, "&notme" is decoded to "¬me": condition 1 fails (not an
384+
* attribute), so the reference is decoded.
385+
* - In _attribute context_, "¬me" is decoded to "¬me": the longest match is
386+
* "not;", which ends in a semicolon, so condition 2 fails.
387+
* - In _attribute context_, "&not己" is decoded to "¬己": the following character
388+
* "己" is a letter but not an ASCII alphanumeric (nor "="), so condition 3 fails.
389+
* - In _attribute context_, "&not" is decoded to "¬": there is no next input
390+
* character, so condition 3 fails.
391+
* - In _attribute context_, "&not=me" is left as the literal text "&not=me": all
392+
* three conditions hold.
393+
* - In _attribute context_, "&notme" is left as the literal text "&notme": all
394+
* three conditions hold.
395+
*
396+
* Without these special rules, ordinary URL query strings could have surprising
397+
* replacements applied. Consider:
398+
*
399+
* <a href="/?random&degree&gt=0&lt=360&not=90">
400+
*
401+
* The literal attribute value `/?random&degree&gt=0&lt=360&not=90` is preserved
402+
* by the special handling. Otherwise, the value would decode to
403+
* `/?random°ree>=0<=360¬=90`, which is unlikely to be the author's intent.
404+
*
405+
* (Authors should not rely on this. Escaping the example as
406+
* `/?random&amp;degree&amp;gt=0&amp;lt=360&amp;not=90` produces the intended
407+
* value regardless of the following character.)
408+
*
409+
* @see https://html.spec.whatwg.org/multipage/parsing.html#named-character-reference-state
410+
* @see https://html.spec.whatwg.org/multipage/named-characters.html#named-character-references
380411
*/
381-
$ambiguous_follower = (
382-
$after_name < $length &&
383-
$name_at < $length &&
384-
(
385-
ctype_alnum( $text[ $after_name ] ) ||
386-
'=' === $text[ $after_name ]
387-
)
388-
);
389-
390-
// It's non-ambiguous, safe to leave it in.
391-
if ( ! $ambiguous_follower ) {
412+
if ( 'attribute' !== $context || ';' === $text[ $after_name - 1 ] || $after_name >= $length ) {
392413
$match_byte_length = $after_name - $at;
393414
return $replacement;
394415
}
395416

396-
// It's ambiguous, which isn't allowed inside attributes.
397-
if ( 'attribute' === $context ) {
417+
$follower_byte = ord( $text[ $after_name ] );
418+
if (
419+
0x3D === $follower_byte || // EQUALS SIGN
420+
( $follower_byte >= 0x30 && $follower_byte <= 0x39 ) || // ASCII digits 0-9
421+
( $follower_byte >= 0x41 && $follower_byte <= 0x5A ) || // ASCII upper alpha A-Z
422+
( $follower_byte >= 0x61 && $follower_byte <= 0x7A ) // ASCII lower alpha a-z
423+
) {
398424
return null;
399425
}
400426

‎tests/phpunit/tests/html-api/wpHtmlDecoder.php‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,55 @@
1212
* @coversDefaultClass WP_HTML_Decoder
1313
*/
1414
class Tests_HtmlApi_WpHtmlDecoder extends WP_UnitTestCase {
15+
/**
16+
* Original LC_CTYPE locale.
17+
*
18+
* @var string|bool
19+
*/
20+
private static $original_lc_ctype = false;
21+
22+
/**
23+
* Locale where ctype_alnum() classifies high-bit bytes as alphanumeric.
24+
*
25+
* @var string|null
26+
*/
27+
private static ?string $problematic_lc_ctype = null;
28+
29+
public static function set_up_before_class() {
30+
parent::set_up_before_class();
31+
32+
self::$original_lc_ctype = setlocale( LC_CTYPE, 0 );
33+
34+
// Find a locale where ctype_alnum() classifies high-bit bytes as alphanumeric.
35+
$locale_candidates = array(
36+
'C.UTF-8',
37+
'C.utf8',
38+
'en_US.UTF-8',
39+
'en_US.utf8',
40+
'en_GB.UTF-8',
41+
'en_GB.utf8',
42+
);
43+
foreach ( $locale_candidates as $locale ) {
44+
$candidate_locale = setlocale( LC_CTYPE, $locale );
45+
46+
if ( false !== $candidate_locale && ctype_alnum( "\xC2" ) ) {
47+
self::$problematic_lc_ctype = $candidate_locale;
48+
break;
49+
}
50+
}
51+
52+
if ( self::$original_lc_ctype ) {
53+
setlocale( LC_CTYPE, self::$original_lc_ctype );
54+
}
55+
}
56+
57+
public function tear_down() {
58+
if ( self::$original_lc_ctype ) {
59+
setlocale( LC_CTYPE, self::$original_lc_ctype );
60+
}
61+
parent::tear_down();
62+
}
63+
1564
/**
1665
* Ensures proper decoding of edge cases.
1766
*
@@ -61,6 +110,115 @@ static function ( int $errno, string $errstr ) use ( &$errors ) {
61110
$this->assertSame( "&\x00b", $decoded, 'Should have decoded the text without changing it.' );
62111
}
63112

113+
/**
114+
* Ensures semicolonless legacy references decode before non-ASCII UTF-8 bytes in attributes.
115+
*
116+
* @dataProvider data_semicolonless_attribute_behaviors
117+
*
118+
* @ticket 65372
119+
*/
120+
public function test_semicolonless_legacy_reference_before_multibyte_attribute_follower( string $encoded_attribute_value, string $expected, string $expected_decode, int $expected_byte_length ): void {
121+
if ( null !== self::$problematic_lc_ctype ) {
122+
setlocale( LC_CTYPE, self::$problematic_lc_ctype );
123+
}
124+
125+
$this->assertSame(
126+
$expected,
127+
WP_HTML_Decoder::decode_attribute( $encoded_attribute_value ),
128+
'Failed to decode the full attribute value as expected.'
129+
);
130+
131+
$match_byte_length = null;
132+
$this->assertSame(
133+
$expected_decode,
134+
WP_HTML_Decoder::read_character_reference( 'attribute', $encoded_attribute_value, 0, $match_byte_length ),
135+
'Failed to decode the character reference as expected.'
136+
);
137+
$this->assertSame( $expected_byte_length, $match_byte_length, 'Failed to produce expected byte length.' );
138+
}
139+
140+
/**
141+
* Data provider.
142+
*
143+
* Attribute values encoded with character references including followers that are
144+
* treated as alphanumerics by `ctype_alnum()` on some systems, but should never
145+
* be recognized as ASCII Alphanumerics according to the HTML standards.
146+
*
147+
* @see https://html.spec.whatwg.org/#named-character-reference-state
148+
*
149+
* @return array<array{
150+
* string, // Encoded attribute value.
151+
* string, // Expected full decode.
152+
* string, // Expected character decode.
153+
* int, // Replaced character reference byte length.
154+
* }> Test cases.
155+
*/
156+
public static function data_semicolonless_attribute_behaviors(): array {
157+
return array(
158+
array( '&copy¯\_(ツ)_/¯', '©¯\_(ツ)_/¯', '©', 5 ),
159+
array( '&notಠ_ಠ', '¬ಠ_ಠ', '¬', 4 ),
160+
array( '&nbsp£20', "\u{00A0}£20", "\u{00A0}", 5 ),
161+
array( '&nbsp🎉', "\u{00A0}🎉", "\u{00A0}", 5 ),
162+
array( '&reg™', '®™', '®', 4 ),
163+
);
164+
}
165+
166+
/**
167+
* Ensures ambiguous ampersand is recognized with trailing ASCII alphanumerics.
168+
*
169+
* @dataProvider data_semicolonless_attribute_character_reference_no_decode_followers
170+
*
171+
* @ticket 65372
172+
*
173+
* @param string $raw_attribute Raw attribute value with an ambiguous legacy reference follower.
174+
*/
175+
public function test_ascii_alphanumeric_attribute_follower_is_ambiguous( string $raw_attribute ): void {
176+
$this->assertSame(
177+
$raw_attribute,
178+
WP_HTML_Decoder::decode_attribute( $raw_attribute ),
179+
'Should not have decoded an ambiguous semicolonless legacy reference.'
180+
);
181+
182+
$match_byte_length = 'sentinel';
183+
$this->assertNull(
184+
WP_HTML_Decoder::read_character_reference( 'attribute', $raw_attribute, 0, $match_byte_length ),
185+
'Should not have matched an ambiguous semicolonless legacy reference.'
186+
);
187+
$this->assertSame( 'sentinel', $match_byte_length );
188+
}
189+
190+
/**
191+
* Data provider.
192+
*
193+
* HTML character references with followers that trigger the literal flush behavior
194+
* when parsing attribute values. HTML defines this as `=` or an ASCII alphanumeric character.
195+
*
196+
* > An ASCII alphanumeric is an ASCII digit or ASCII alpha.
197+
* > An ASCII alpha is an ASCII upper alpha or ASCII lower alpha.
198+
*
199+
* @see https://html.spec.whatwg.org/#named-character-reference-state
200+
*
201+
* @return Generator<string, array{ string }> Test cases.
202+
*/
203+
public static function data_semicolonless_attribute_character_reference_no_decode_followers(): Generator {
204+
yield "Equals sign follower '='" => array( '&Aacute=' );
205+
// > An ASCII digit is a code point in the range U+0030 (0) to U+0039 (9), inclusive.
206+
for ( $i = 0x30; $i <= 0x39; $i++ ) {
207+
$char = chr( $i );
208+
yield "ASCII digit follower '{$char}'" => array( "&Aacute{$char}" );
209+
}
210+
// > An ASCII upper alpha is a code point in the range U+0041 (A) to U+005A (Z), inclusive.
211+
for ( $i = 0x41; $i <= 0x5A; $i++ ) {
212+
$char = chr( $i );
213+
yield "ASCII upper alpha follower '{$char}'" => array( "&Aacute{$char}" );
214+
}
215+
// > An ASCII lower alpha is a code point in the range U+0061 (a) to U+007A (z), inclusive.
216+
for ( $i = 0x61; $i <= 0x7A; $i++ ) {
217+
$char = chr( $i );
218+
yield "ASCII lower alpha follower '{$char}'" => array( "&Aacute{$char}" );
219+
}
220+
}
221+
64222
/**
65223
* Ensures proper detection of attribute prefixes ignoring ASCII case.
66224
*

0 commit comments

Comments
 (0)