Skip to content

Commit ddc17f4

Browse files
dmsnelldmsnell
committed
WIP: HTML API: Extract previous text and HTML chunks while processing.
The HTML API should be able to provide the ability to generate excerpts from HTMl documents given a specific maximum length. In this patch we're exploring the addition of text and HTML chunks that can be extracted while processing in order to do just this. The text chunks are similar to `.textContent` on the DOM while the HTML chunks contain raw and unprocessed HTML. These functions should likely remain low-level in the Tag Processor and be exposed from the HTML Processor to ensure that proper semantics are heeded when extracting this information, such as how `PRE` tags ignore a leading newline inside their content or how `SCRIPT` and `STYLE` content isn't part of what we want with something like `strip_tags()`. In the process of this work it's evident again that the Tag Processor ought to expose the ability to visit every token and non-tag tokens should be classified. This has already been explored in #7.
1 parent 6f74f61 commit ddc17f4

2 files changed

Lines changed: 203 additions & 0 deletions

File tree

‎src/wp-includes/html-api/class-wp-html-tag-processor.php‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -378,6 +378,15 @@ class WP_HTML_Tag_Processor {
378378
*/
379379
private $is_closing_tag;
380380

381+
/**
382+
* Stores the position of the last-matched tag, or the start of the document if not matched yet.
383+
*
384+
* @var WP_HTML_Span
385+
*/
386+
private $last_position = null;
387+
388+
private $last_token_end = 0;
389+
381390
/**
382391
* Lazily-built index of attributes found within an HTML tag, keyed by the attribute name.
383392
*
@@ -507,6 +516,8 @@ class WP_HTML_Tag_Processor {
507516
*/
508517
public function __construct( $html ) {
509518
$this->html = $html;
519+
520+
$this->last_position = new WP_HTML_Span( 0, 0 );
510521
}
511522

512523
/**
@@ -530,6 +541,16 @@ public function next_tag( $query = null ) {
530541
$this->parse_query( $query );
531542
$already_found = 0;
532543

544+
if ( null !== $this->tag_name_starts_at ) {
545+
$rewind_amount = $this->is_closing_tag ? 2 : 1;
546+
$before_tag = $this->tag_name_starts_at - $rewind_amount;
547+
$end_of_tag = $this->tag_ends_at;
548+
549+
$this->last_position->start = $before_tag;
550+
$this->last_position->end = $end_of_tag;
551+
$this->last_token_end = $this->tag_ends_at + 1;
552+
}
553+
533554
do {
534555
if ( $this->bytes_already_parsed >= strlen( $this->html ) ) {
535556
return false;
@@ -1876,6 +1897,74 @@ public function is_tag_closer() {
18761897
return $this->is_closing_tag;
18771898
}
18781899

1900+
/**
1901+
* Returns the chunk of text from the end of the preceding tag or token to the
1902+
* start of the matched tag or token, with decoded character references.
1903+
*
1904+
* Example:
1905+
*
1906+
* $q = array( 'tag_closers' => 'visit' );
1907+
* $processor = new WP_HTML_Tag_Processor( 'Before<div>Inside</div>After' );
1908+
* $processor->next_tag( $q ); 'Before' === $processor->get_prev_text_chunk();
1909+
* $processor->next_tag( $q ); 'Inside' === $processor->get_prev_text_chunk();
1910+
* $processor->next_tag( $q ); 'After' === $processor->get_prev_text_chunk();
1911+
*
1912+
* @since 6.4.0
1913+
*
1914+
* @return string|null Chunk of text from end of last token to current token, or NULL if not yet matched.
1915+
*/
1916+
public function get_previous_text_chunk() {
1917+
if ( $this->bytes_already_parsed >= strlen( $this->html ) ) {
1918+
$chunk = substr( $this->html, $this->last_position->end === 0 ? 0 : $this->last_position->end + 1 );
1919+
$chunk = preg_replace( '/<[^a-z].*>/i', '', $chunk );
1920+
return html_entity_decode( $chunk, ENT_HTML5 | ENT_QUOTES | ENT_SUBSTITUTE );
1921+
}
1922+
1923+
if ( ! $this->tag_name_starts_at ) {
1924+
return null;
1925+
}
1926+
1927+
$chunk_start = $this->last_position->end === 0 ? 0 : $this->last_position->end + 1;
1928+
$chunk_end = $this->is_tag_closer() ? $this->tag_name_starts_at - 2 : $this->tag_name_starts_at - 1;
1929+
$chunk = substr( $this->html, $chunk_start, $chunk_end - $chunk_start );
1930+
$chunk = preg_replace( '/<[^a-z].*>/i', '', $chunk );
1931+
return html_entity_decode( $chunk, ENT_HTML5 | ENT_QUOTES | ENT_SUBSTITUTE );
1932+
}
1933+
1934+
/**
1935+
* Returns the chunk of html from the start of the preceding tag or token to the
1936+
* start of the matched tag or token, without decoded character references.
1937+
*
1938+
* Example:
1939+
*
1940+
* $q = array( 'tag_closers' => 'visit' );
1941+
* $processor = new WP_HTML_Tag_Processor( 'Before<div>Inside</div>After' );
1942+
* $processor->next_tag( $q ); 'Before' === $processor->get_prev_text_chunk();
1943+
* $processor->next_tag( $q ); '<div>Inside' === $processor->get_prev_text_chunk();
1944+
* $processor->next_tag( $q ); '</div>After' === $processor->get_prev_text_chunk();
1945+
*
1946+
* @since 6.4.0
1947+
*
1948+
* @return array|null Chunk of text from end of last token to current token, or NULL if not yet matched.
1949+
*/
1950+
public function get_previous_html_chunk() {
1951+
if ( $this->bytes_already_parsed >= strlen( $this->html ) ) {
1952+
$html = substr( $this->html, $this->last_position->start, $this->last_token_end - $this->last_position->start );
1953+
$text = substr( $this->html, $this->last_token_end );
1954+
1955+
return array( $html, $text );
1956+
}
1957+
1958+
if ( ! $this->tag_name_starts_at ) {
1959+
return null;
1960+
}
1961+
1962+
$html = substr( $this->html, $this->last_position->start, $this->last_token_end - $this->last_position->start );
1963+
$text = substr( $this->html, $this->last_token_end, ( $this->is_tag_closer() ? $this->tag_name_starts_at - 2 : $this->tag_name_starts_at - 1 ) - $this->last_token_end );
1964+
1965+
return array( $html, $text );
1966+
}
1967+
18791968
/**
18801969
* Updates or creates a new attribute on the currently matched tag with the passed value.
18811970
*

‎tests/phpunit/tests/html-api/wpHtmlProcessor-stringBuilder.php‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
<?php
2+
/**
3+
* Unit tests covering WP_HTML_Processor string building functionality.
4+
*
5+
* @package WordPress
6+
* @subpackage HTML-API
7+
*/
8+
9+
/**
10+
* @group html-api
11+
*
12+
* @coversDefaultClass WP_HTML_Processor
13+
*/
14+
class Tests_HtmlApi_WpHtmlProcessor_StringBuilder extends WP_UnitTestCase {
15+
/**
16+
* @ticket {TICKET_NUMBER}
17+
*
18+
* @dataProvider data_html_and_associated_text_content
19+
*
20+
* @param string $html HTML containing text that should be extracted.
21+
* @param string $text_content Plaintext content represented inside the given HTML.
22+
*/
23+
public function test_extracts_text_chunks_properly( $html, $text_content ) {
24+
$processor = new WP_HTML_Tag_Processor( $html );
25+
26+
$extracted_text_content = '';
27+
while ( $processor->next_tag( array( 'tag_closers' => 'visit' ) ) ) {
28+
$extracted_text_content .= $processor->get_previous_text_chunk();
29+
}
30+
$extracted_text_content .= $processor->get_previous_text_chunk();
31+
32+
$this->assertEquals( $text_content, $extracted_text_content, 'Extracted unexpected text content.' );
33+
}
34+
35+
/**
36+
* Data provider.
37+
*
38+
* @return array[].
39+
*/
40+
public function data_html_and_associated_text_content() {
41+
return array(
42+
'Basic text without HTML.' => array( 'This is plain text.', 'This is plain text.' ),
43+
'Basic text with a character reference.' => array( 'A &lt; B', 'A < B' ),
44+
'Text before tag.' => array( 'Before<img>', 'Before' ),
45+
'Text after tag.' => array( '<img>After', 'After' ),
46+
'Text inside tag.' => array( '<div>Inside</div>', 'Inside' ),
47+
'Text around tag.' => array( 'In <em>the</em> jungle.', 'In the jungle.' ),
48+
'Text interrupted by many tags.' => array( 'A <em>wild <a><img><span>adventure</span></a> awaits.', 'A wild adventure awaits.' ),
49+
'Text with comment inside it.' => array( 'Ignore <!-- everything inside this --> comment.', 'Ignore comment.' ),
50+
'Text with empty comment inside it.' => array( 'Ignore <!--> comment.', 'Ignore comment.' ),
51+
'Text with invalid comment inside it.' => array( 'Ignore </^$%> comment.', 'Ignore comment.' ),
52+
'Skipping SCRIPT content.' => array( '<div>This <script>does not exist</script> in the output.', 'This in the output.' ),
53+
);
54+
}
55+
56+
/**
57+
* @ticket {TICKET_NUMBER}
58+
*
59+
* @dataProvider data_html_and_associated_html_content
60+
*
61+
* @param string $html HTML containing text that should be extracted.
62+
* @param int $max_code_points Stop iterating after this many code points have been extracted.
63+
* @param string $html_content Full HTML containing text of max code point length from input.
64+
*/
65+
public function test_extracts_html_chunks_properly( $html, $max_code_points, $html_content ) {
66+
$processor = new WP_HTML_Tag_Processor( $html );
67+
68+
$code_points = 0;
69+
$extracted_html_content = '';
70+
while ( $processor->next_tag( array( 'tag_closers' => 'visit' ) ) ) {
71+
$text_chunk = $processor->get_previous_text_chunk();
72+
$chunk_cps = mb_strlen( $text_chunk );
73+
list( $html, $text ) = $processor->get_previous_html_chunk();
74+
$extracted_html_content .= $html;
75+
if ( 0 === $max_code_points || $code_points + $chunk_cps <= $max_code_points ) {
76+
$extracted_html_content .= $text;
77+
$code_points += $chunk_cps;
78+
} else {
79+
break;
80+
}
81+
}
82+
83+
$text_chunk = $processor->get_previous_text_chunk();
84+
$chunk_cps = mb_strlen( $text_chunk );
85+
list( $html, $text ) = $processor->get_previous_html_chunk();
86+
$extracted_html_content .= $html;
87+
if ( 0 === $max_code_points || $code_points + $chunk_cps <= $max_code_points ) {
88+
$extracted_html_content .= $text;
89+
}
90+
91+
$this->assertEquals( $html_content, $extracted_html_content, 'Extracted unexpected HTML content.' );
92+
}
93+
94+
/**
95+
* Data provider.
96+
*
97+
* @return array[].
98+
*/
99+
public function data_html_and_associated_html_content() {
100+
return array(
101+
'Basic text without HTML.' => array( 'This is plain text.', 0, 'This is plain text.' ),
102+
'Basic text without HTML (too long).' => array( 'This is plain text.', 8, '' ),
103+
'Basic text with a character reference.' => array( 'A &lt; B', 0, 'A &lt; B' ),
104+
'Character reference wider than text' => array( 'A &lt; B', 5, 'A &lt; B' ),
105+
'Text before tag.' => array( 'Before<img>', 0, 'Before<img>' ),
106+
'Text after tag.' => array( '<img>After', 0, '<img>After' ),
107+
'Text inside tag.' => array( '<div>Inside</div>', 0, '<div>Inside</div>' ),
108+
'Text around tag.' => array( 'In <em>the</em> jungle.', 0, 'In <em>the</em> jungle.' ),
109+
'Text interrupted by many tags.' => array( 'A <em>wild <a><img><span>adventure</span></a> awaits.', 0, 'A <em>wild <a><img><span>adventure</span></a> awaits.' ),
110+
'Text interrupted by many tags (long).' => array( 'A <em>wild <a><img><span>adventure</span></a> awaits.', 16, 'A <em>wild <a><img><span>adventure</span></a>' ),
111+
'Text with comment inside it.' => array( 'Ignore <!-- everything inside this --> comment.', 0, 'Ignore <!-- everything inside this --> comment.' ),
112+
);
113+
}
114+
}

0 commit comments

Comments
 (0)