General: Add support for unicode email addresses in is_email and sanitize_email

This adds support for the unicode address extensions in RFC 6530-3 and refactors the code so there are fewer long regexes and less duplication between sanitize_email and is_email. A new class, WP_Email_Address, provides the shared parts.

Opting out of unicode support is easy, default-filters.php adds unicode support by adding filters, which can be removed.

`sanitize_email` no longer does major changes like removing an entire subdomain from someone's address, it only cleans up things like soft hyphens and whitespace — changes that happen when coping an email address from text.

Developed in: WordPress#5237
Discussed in: https://core.trac.wordpress.org/ticket/31992

Props agulbra, akirk, benniledl, dmsnell, ironprogrammer, justlevine, mdawaffe, mukeshpanchal27, SirLouen, tusharbharti.
Fixes #31992.


git-svn-id: https://develop.svn.wordpress.org/trunk@62482 602fd350-edb4-49c9-b593-d223f7449a82

Author: dmsnell

src/wp-includes/class-wp-email-address.php

@@ -87,6 +87,17 @@
 	add_filter( $filter, 'wp_filter_kses' );
 }
 
+// Email addresses: Allow unicode if and only if as the database can
+// store them. This affects all addresses, including those entered
+// into contact forms.
+if ( 'utf8mb4' === $wpdb->charset ) {
+	add_filter( 'is_email', 'wp_is_unicode_email', 10, 3 );
+	add_filter( 'sanitize_email', 'wp_sanitize_unicode_email', 10, 3 );
+} else {
+	add_filter( 'is_email', 'wp_is_ascii_email', 10, 3 );
+	add_filter( 'sanitize_email', 'wp_sanitize_ascii_email', 10, 3 );
+}
+
 // Display URL.
 foreach ( array( 'user_url', 'link_url', 'link_image', 'link_rss', 'comment_url', 'post_guid' ) as $filter ) {
 	if ( is_admin() ) {

src/wp-includes/default-filters.php

@@ -2176,6 +2176,7 @@ function sanitize_user( $username, $strict = false ) {
 	return apply_filters( 'sanitize_user', $username, $raw_username, $strict );
 }
 
+
 /**
  * Sanitizes a string key.
  *
@@ -3589,7 +3590,14 @@ function convert_smilies( $text ) {
 /**
  * Verifies that an email is valid.
  *
- * Does not grok i18n domains. Not RFC compliant.
+ * This accepts the addresses that matches the WHATWG specifications,
+ * i.e. what browsers use for `<input type=email>`. It also accepts some
+ * additional addresses.
+ *
+ * By default this accepts addresses like info@grå.org (also accepted
+ * by Firefox) `<input type=email>`. You can disable Unicode support by
+ * using the wp_is_ascii_email filter instead of wp_is_unicode_email,
+ * which is the default.
  *
  * @since 0.71
  *
@@ -3602,84 +3610,65 @@ function is_email( $email, $deprecated = false ) {
 		_deprecated_argument( __FUNCTION__, '3.0.0' );
 	}
 
-	// Test for the minimum length the email can be.
-	if ( strlen( $email ) < 6 ) {
-		/**
-		 * Filters whether an email address is valid.
-		 *
-		 * This filter is evaluated under several different contexts, such as 'email_too_short',
-		 * 'email_no_at', 'local_invalid_chars', 'domain_period_sequence', 'domain_period_limits',
-		 * 'domain_no_periods', 'sub_hyphen_limits', 'sub_invalid_chars', or no specific context.
-		 *
-		 * @since 2.8.0
-		 *
-		 * @param string|false $is_email The email address if successfully passed the is_email() checks, false otherwise.
-		 * @param string       $email    The email address being checked.
-		 * @param string       $context  Context under which the email was tested.
-		 */
-		return apply_filters( 'is_email', false, $email, 'email_too_short' );
-	}
-
-	// Test for an @ character after the first position.
-	if ( false === strpos( $email, '@', 1 ) ) {
-		/** This filter is documented in wp-includes/formatting.php */
-		return apply_filters( 'is_email', false, $email, 'email_no_at' );
-	}
-
-	// Split out the local and domain parts.
-	list( $local, $domain ) = explode( '@', $email, 2 );
-
-	/*
-	 * LOCAL PART
-	 * Test for invalid characters.
-	 */
-	if ( ! preg_match( '/^[a-zA-Z0-9!#$%&\'*+\/=?^_`{|}~\.-]+$/', $local ) ) {
-		/** This filter is documented in wp-includes/formatting.php */
-		return apply_filters( 'is_email', false, $email, 'local_invalid_chars' );
-	}
-
-	/*
-	 * DOMAIN PART
-	 * Test for sequences of periods.
+	/**
+	 * Filters whether an email address is valid.
+	 *
+	 * This filter is evaluated under several different contexts, such as
+	 * 'local_invalid_chars', 'domain_no_periods', or no specific context.
+	 * Filters registered on this hook perform the actual validation; the
+	 * default filter is registered in default-filters.php.
+	 *
+	 * @since 2.8.0
+	 *
+	 * @param string|false $is_email The email address if successfully passed the is_email() checks, false otherwise.
+	 * @param string       $email    The email address being checked.
+	 * @param string|null  $context  Context under which the email was tested, or null for the initial call.
 	 */
-	if ( preg_match( '/\.{2,}/', $domain ) ) {
-		/** This filter is documented in wp-includes/formatting.php */
-		return apply_filters( 'is_email', false, $email, 'domain_period_sequence' );
-	}
-
-	// Test for leading and trailing periods and whitespace.
-	if ( trim( $domain, " \t\n\r\0\x0B." ) !== $domain ) {
-		/** This filter is documented in wp-includes/formatting.php */
-		return apply_filters( 'is_email', false, $email, 'domain_period_limits' );
-	}
-
-	// Split the domain into subs.
-	$subs = explode( '.', $domain );
+	return apply_filters( 'is_email', false, $email, null );
+}
 
-	// Assume the domain will have at least two subs.
-	if ( 2 > count( $subs ) ) {
-		/** This filter is documented in wp-includes/formatting.php */
-		return apply_filters( 'is_email', false, $email, 'domain_no_periods' );
+/**
+ * Default is_email filter for databases that support Unicode (db charset is utf8mb4).
+ *
+ * Validates the email address using {@see WP_Email_Address::from_string()} with Unicode enabled.
+ * Only acts when $context is null (which it is in the initial validation call); later rescue-context calls are passed through.
+ *
+ * @since 7.1.0
+ *
+ * @param string|false $value   The current filter value.
+ * @param string       $email   The email address being checked.
+ * @param string|null  $context Validation context, or null for the initial call.
+ * @return string|false The email address if valid, false otherwise.
+ */
+function wp_is_unicode_email( $value, $email, $context ) {
+	if ( null !== $context ) {
+		return $value;
 	}
 
-	// Loop through each sub.
-	foreach ( $subs as $sub ) {
-		// Test for leading and trailing hyphens and whitespace.
-		if ( trim( $sub, " \t\n\r\0\x0B-" ) !== $sub ) {
-			/** This filter is documented in wp-includes/formatting.php */
-			return apply_filters( 'is_email', false, $email, 'sub_hyphen_limits' );
-		}
+	$result = WP_Email_Address::from_string( $email, 'unicode' );
+	return $result ? $result->get_unicode_address() : false;
+}
 
-		// Test for invalid characters.
-		if ( ! preg_match( '/^[a-z0-9-]+$/i', $sub ) ) {
-			/** This filter is documented in wp-includes/formatting.php */
-			return apply_filters( 'is_email', false, $email, 'sub_invalid_chars' );
-		}
+/**
+ * Default is_email filter for databases that do not support Unicode (db charset is not utf8mb4).
+ *
+ * Validates the email address using {@see WP_Email_Address::from_string()} with Unicode disabled.
+ * Only acts when $context is null (which it is in the initial validation call); later rescue-context calls are passed through.
+ *
+ * @since 7.1.0
+ *
+ * @param string|false $value   The current filter value.
+ * @param string       $email   The email address being checked.
+ * @param string|null  $context Validation context, or null for the initial call.
+ * @return string|false The email address if valid, false otherwise.
+ */
+function wp_is_ascii_email( $value, $email, $context ) {
+	if ( null !== $context ) {
+		return $value;
 	}
 
-	// Congratulations, your email made it!
-	/** This filter is documented in wp-includes/formatting.php */
-	return apply_filters( 'is_email', $email, $email, null );
+	$result = WP_Email_Address::from_string( $email, 'ascii' );
+	return $result ? $result->get_unicode_address() : false;
 }
 
 /**
@@ -3808,109 +3797,96 @@ function iso8601_to_datetime( $date_string, $timezone = 'user' ) {
 }
 
 /**
- * Strips out all characters that are not allowable in an email.
+ * Sanitizes an email address.
+ *
+ * Strips stray whitespace from the input, then strips trailing dots from the domain.
+ * This is designed to recover from cut/paste mistakes without any risk of transforming
+ * the input into a different address than the user intended.
+ *
+ * Validation and final form are determined by the 'sanitize_email' filter; the default
+ * filter is registered in default-filters.php and delegates to {@see WP_Email_Address::from_string()}.
  *
  * @since 1.5.0
+ * @since 7.1.0 Accepts Unicode email addresses on supporting platforms.
  *
- * @param string $email Email address to filter.
- * @return string Filtered email address.
+ * @param string $email Email address to sanitize.
+ * @return string The sanitized email address, or an empty string if invalid.
  */
 function sanitize_email( $email ) {
-	// Test for the minimum length the email can be.
-	if ( strlen( $email ) < 6 ) {
-		/**
-		 * Filters a sanitized email address.
-		 *
-		 * This filter is evaluated under several contexts, including 'email_too_short',
-		 * 'email_no_at', 'local_invalid_chars', 'domain_period_sequence', 'domain_period_limits',
-		 * 'domain_no_periods', 'domain_no_valid_subs', or no context.
-		 *
-		 * @since 2.8.0
-		 *
-		 * @param string $sanitized_email The sanitized email address.
-		 * @param string $email           The email address, as provided to sanitize_email().
-		 * @param string|null $message    A message to pass to the user. null if email is sanitized.
-		 */
-		return apply_filters( 'sanitize_email', '', $email, 'email_too_short' );
-	}
-
-	// Test for an @ character after the first position.
-	if ( false === strpos( $email, '@', 1 ) ) {
-		/** This filter is documented in wp-includes/formatting.php */
-		return apply_filters( 'sanitize_email', '', $email, 'email_no_at' );
-	}
-
-	// Split out the local and domain parts.
-	list( $local, $domain ) = explode( '@', $email, 2 );
+	// Strip surrounding whitespace.
+	$email = trim( $email );
 
-	/*
-	 * LOCAL PART
-	 * Test for invalid characters.
-	 */
-	$local = preg_replace( '/[^a-zA-Z0-9!#$%&\'*+\/=?^_`{|}~\.-]/', '', $local );
-	if ( '' === $local ) {
-		/** This filter is documented in wp-includes/formatting.php */
-		return apply_filters( 'sanitize_email', '', $email, 'local_invalid_chars' );
+	// Extract the address from "Display Name <username@domain>" format.
+	if ( 1 === preg_match( '/<([^>]+)>$/', $email, $matches ) ) {
+		$email = $matches[1];
 	}
 
 	/*
-	 * DOMAIN PART
-	 * Test for sequences of periods.
+	 * Strip soft hyphens and whitespace adjacent to structural separators (dots and @),
+	 * e.g. copy-paste artifacts like "info@example\u{00AD}.com" or "info@example .com".
+	 *
+	 * In some cases, e.g. autocorrect, some older software has been seen to add the
+	 * space for unrecognized TLDs. This re-joins the parts for proper examination.
 	 */
-	$domain = preg_replace( '/\.{2,}/', '', $domain );
-	if ( '' === $domain ) {
-		/** This filter is documented in wp-includes/formatting.php */
-		return apply_filters( 'sanitize_email', '', $email, 'domain_period_sequence' );
-	}
-
-	// Test for leading and trailing periods and whitespace.
-	$domain = trim( $domain, " \t\n\r\0\x0B." );
-	if ( '' === $domain ) {
-		/** This filter is documented in wp-includes/formatting.php */
-		return apply_filters( 'sanitize_email', '', $email, 'domain_period_limits' );
-	}
-
-	// Split the domain into subs.
-	$subs = explode( '.', $domain );
-
-	// Assume the domain will have at least two subs.
-	if ( 2 > count( $subs ) ) {
-		/** This filter is documented in wp-includes/formatting.php */
-		return apply_filters( 'sanitize_email', '', $email, 'domain_no_periods' );
-	}
-
-	// Create an array that will contain valid subs.
-	$new_subs = array();
-
-	// Loop through each sub.
-	foreach ( $subs as $sub ) {
-		// Test for leading and trailing hyphens.
-		$sub = trim( $sub, " \t\n\r\0\x0B-" );
+	$email = preg_replace( '/[\x{00AD}\s]*([.@])[\x{00AD}\s]*/u', '$1', $email ) ?? $email;
 
-		// Test for invalid characters.
-		$sub = preg_replace( '/[^a-z0-9-]+/i', '', $sub );
-
-		// If there's anything left, add it to the valid subs.
-		if ( '' !== $sub ) {
-			$new_subs[] = $sub;
-		}
-	}
-
-	// If there aren't 2 or more valid subs.
-	if ( 2 > count( $new_subs ) ) {
-		/** This filter is documented in wp-includes/formatting.php */
-		return apply_filters( 'sanitize_email', '', $email, 'domain_no_valid_subs' );
+	// Strip a trailing dot from the domain (e.g. if pasted from the end of a sentence).
+	if ( str_contains( $email, '@' ) ) {
+		list( $local, $domain ) = explode( '@', $email, 2 );
+		$domain                 = rtrim( $domain, '.' );
+		$email                  = $local . '@' . $domain;
 	}
 
-	// Join valid subs into the new domain.
-	$domain = implode( '.', $new_subs );
+	/**
+	 * Filters a sanitized email address.
+	 *
+	 * Filters registered on this hook perform the actual validation and return
+	 * the canonical email string on success or an empty string on failure.
+	 * The default filter is registered in default-filters.php.
+	 *
+	 * @since 2.8.0
+	 *
+	 * @param string      $sanitized_email The sanitized email address, or empty string.
+	 * @param string      $email           The email address as provided to sanitize_email().
+	 * @param string|null $context         Validation context, or null for the initial call.
+	 */
+	return apply_filters( 'sanitize_email', '', $email, null );
+}
 
-	// Put the email back together.
-	$sanitized_email = $local . '@' . $domain;
+/**
+ * Default sanitize_email filter for databases that support Unicode (db charset is utf8mb4).
+ *
+ * Returns the canonical address from {@see WP_Email_Address::from_string()} with Unicode
+ * enabled, or an empty string if the address is invalid.
+ *
+ * @since 7.1.0
+ *
+ * @param string      $value   The current filter value.
+ * @param string      $email   The email address being sanitized.
+ * @param string|null $context Sanitization context, always null.
+ * @return string The canonical email address if valid, empty string otherwise.
+ */
+function wp_sanitize_unicode_email( $value, $email, $context ) {
+	$result = WP_Email_Address::from_string( $email, 'unicode' );
+	return $result ? $result->get_unicode_address() : '';
+}
 
-	// Congratulations, your email made it!
-	/** This filter is documented in wp-includes/formatting.php */
-	return apply_filters( 'sanitize_email', $sanitized_email, $email, null );
+/**
+ * Default sanitize_email filter for databases that do not support Unicode (db charset is not utf8mb4).
+ *
+ * Returns the canonical address from {@see WP_Email_Address::from_string()} with Unicode
+ * disabled, or an empty string if the address is invalid.
+ *
+ * @since 7.1.0
+ *
+ * @param string      $value   The current filter value.
+ * @param string      $email   The email address being sanitized.
+ * @param string|null $context Sanitization context, always null.
+ * @return string The canonical email address if valid, empty string otherwise.
+ */
+function wp_sanitize_ascii_email( $value, $email, $context ) {
+	$result = WP_Email_Address::from_string( $email, 'ascii' );
+	return $result ? $result->get_unicode_address() : '';
 }
 
 /**

src/wp-includes/formatting.php

@@ -112,6 +112,7 @@
 require ABSPATH . WPINC . '/class-wp-list-util.php';
 require ABSPATH . WPINC . '/class-wp-token-map.php';
 require ABSPATH . WPINC . '/utf8.php';
+require ABSPATH . WPINC . '/class-wp-email-address.php';
 require ABSPATH . WPINC . '/formatting.php';
 require ABSPATH . WPINC . '/meta.php';
 require ABSPATH . WPINC . '/functions.php';

src/wp-settings.php

@@ -1520,7 +1520,7 @@ public function test_wp_authenticate_cookie_with_invalid_cookie() {
 	 */
 	public function test_wp_signon_using_email_with_an_apostrophe() {
 		$user_args = array(
-			'user_email' => "mail\'@example.com",
+			'user_email' => "mail'@example.com",
 			'user_pass'  => 'password',
 		);
 		self::factory()->user->create( $user_args );
@@ -1833,7 +1833,7 @@ static function ( $available, WP_User $user ) {
 	 */
 	public function test_reset_password_with_apostrophe_in_email() {
 		$user_args = array(
-			'user_email' => "jo'hn@example.com",
+			'user_email' => "jo\'hn@example.com",
 			'user_pass'  => 'password',
 		);
 

tests/phpunit/tests/auth.php

@@ -34,6 +34,9 @@ public function data_returns_valid_utf8() {
 			'plain with ip'        => array( 'ace@204.32.222.14' ),
 			'deep subdomain'       => array( 'kevin@many.subdomains.make.a.happy.man.edu' ),
 			'short address'        => array( 'a@b.co' ),
+			'ascii@nonascii'       => array( 'info@grå.org' ),
+			'nonascii@nonascii'    => array( 'grå@grå.org' ),
+			'decomposed unicode'   => array( "gr\u{0061}\u{030a}blå@grå.org" ),
 			'weird but legal dots' => array( '..@example.com' ),
 			'umlauts'              => array( 'bücher@gmx.de' ),
 			'three-byte UTF-8'     => array( "\u{FFFD}@who.knows.com" ),