Consider all Default_Ignorable_Code_Points unprintable

These characters may be hidden/invisible otherwise.

Author: Jules-Bertholet

library/core/src/char/methods.rs

@@ -493,6 +493,7 @@ impl char {
                 || self.is_private_use()
                 || self.is_whitespace()
                 || args.escape_grapheme_extender && self.is_grapheme_extender()
+                || self.is_default_ignorable()
                 || self.is_format_control()
                 || self.is_unassigned() =>
             {
@@ -1225,6 +1226,37 @@ impl char {
         }
     }
 
+    /// Returns `true` if this `char` has the `Default_Ignorable_Code_Point` property.
+    /// These characters [should be displayed as invisible in fallback rendering](https://www.unicode.org/faq/unsup_char#3).
+    ///
+    /// `Default_Ignorable_Code_Point` is [described] in Chapter 5 (Implementation Guidelines) of the Unicode Standard,
+    /// and [specified] in the Unicode Character Database [`DerivedCoreProperties.txt`].
+    ///
+    /// [described]: https://www.unicode.org/versions/latest/core-spec/chapter-5/#G40120
+    /// [specified]: https://www.unicode.org/reports/tr44/#Default_Ignorable_Code_Point
+    /// [`DerivedCoreProperties.txt`]: https://www.unicode.org/Public/UCD/latest/ucd/DerivedCoreProperties.txt
+    ///
+    /// # Examples
+    ///
+    /// Basic usage:
+    ///
+    /// ```ignore(private)
+    /// assert!('\u{AD}'.is_default_ignorable()); // SOFT HYPHEN
+    /// assert!('\u{115F}'.is_default_ignorable()); // HANGUL CHOSEONG FILLER
+    /// assert!('\u{200B}'.is_default_ignorable()); // ZERO WIDTH SPACE
+    /// assert!('\u{E0041}'.is_default_ignorable()); // TAG LATIN CAPITAL LETTER A
+    /// assert!(!'۝'.is_default_ignorable()); // ARABIC END OF AYAH
+    /// assert!(!'𓐲'.is_default_ignorable()); // EGYPTIAN HIEROGLYPH INSERT AT TOP START
+    /// assert!(!' '.is_default_ignorable());
+    /// assert!(!'\n'.is_default_ignorable());
+    /// assert!(!'\0'.is_default_ignorable());
+    /// assert!(!'q'.is_default_ignorable());
+    #[must_use]
+    #[inline]
+    fn is_default_ignorable(self) -> bool {
+        self > '\u{AC}' && unicode::Default_Ignorable_Code_Point(self)
+    }
+
     /// Returns `true` if this `char` has the `Grapheme_Extend` property.
     ///
     /// `Grapheme_Extend` is [described] in Chapter 3 (Conformance) of the Unicode Standard,

library/core/src/unicode/mod.rs

@@ -11,6 +11,7 @@ pub(crate) use unicode_data::alphabetic::lookup as Alphabetic;
 pub(crate) use unicode_data::case_ignorable::lookup as Case_Ignorable;
 pub(crate) use unicode_data::cf::lookup as Cf;
 pub(crate) use unicode_data::cn_planes_0_3::lookup as Cn_planes_0_3;
+pub(crate) use unicode_data::default_ignorable_code_point::lookup as Default_Ignorable_Code_Point;
 pub(crate) use unicode_data::grapheme_extend::lookup as Grapheme_Extend;
 pub(crate) use unicode_data::lowercase::lookup as Lowercase;
 pub(crate) use unicode_data::lt::lookup as Lt;

library/core/src/unicode/unicode_data.rs

@@ -3,6 +3,7 @@
 // Case_Ignorable              :  1063 bytes,   2789 codepoints in 459 ranges (U+0000A8 - U+0E01F0) using skiplist
 // Cf                          :    87 bytes,    170 codepoints in  21 ranges (U+0000AD - U+0E0080) using skiplist
 // Cn_Planes_0_3               :  1677 bytes,  94165 codepoints in 730 ranges (U+000378 - U+03FFFE) using skiplist
+// Default_Ignorable_Code_Point:    83 bytes,   4174 codepoints in  17 ranges (U+0000AD - U+0E1000) using skiplist
 // Grapheme_Extend             :   899 bytes,   2232 codepoints in 383 ranges (U+000300 - U+0E01F0) using skiplist
 // Lowercase                   :   943 bytes,   2569 codepoints in 676 ranges (U+0000AA - U+01E944) using bitset
 // Lt                          :    33 bytes,     31 codepoints in  10 ranges (U+0001C5 - U+001FFD) using skiplist
@@ -12,7 +13,7 @@
 // to_lower                    :  1112 bytes,   1462 codepoints in 185 ranges (U+0000C0 - U+01E921) using 2-level LUT
 // to_upper                    :  1998 bytes,   1554 codepoints in 299 ranges (U+0000B5 - U+01E943) using 2-level LUT
 // to_title                    :   340 bytes,    135 codepoints in  49 ranges (U+0000DF - U+00FB17) using 2-level LUT
-// Total                       : 11393 bytes
+// Total                       : 11476 bytes
 
 #[inline(always)]
 const fn bitset_search<
@@ -488,6 +489,44 @@ pub mod cn_planes_0_3 {
     }
 }
 
+#[rustfmt::skip]
+pub mod default_ignorable_code_point {
+    use super::ShortOffsetRunHeader;
+
+    static SHORT_OFFSET_RUNS: [ShortOffsetRunHeader; 12] = [
+        ShortOffsetRunHeader::new(0, 847), ShortOffsetRunHeader::new(3, 1564),
+        ShortOffsetRunHeader::new(5, 4447), ShortOffsetRunHeader::new(7, 6068),
+        ShortOffsetRunHeader::new(9, 8203), ShortOffsetRunHeader::new(13, 12644),
+        ShortOffsetRunHeader::new(19, 65024), ShortOffsetRunHeader::new(21, 113824),
+        ShortOffsetRunHeader::new(29, 119155), ShortOffsetRunHeader::new(31, 917504),
+        ShortOffsetRunHeader::new(33, 921600), ShortOffsetRunHeader::new(34, 2035712),
+    ];
+    static OFFSETS: [u8; 35] = [
+        173, 1, 0, 1, 0, 1, 0, 2, 0, 2, 85, 5, 0, 5, 26, 5, 49, 16, 0, 1, 0, 16, 239, 1, 160, 1,
+        79, 9, 0, 4, 0, 8, 0, 0, 0,
+    ];
+    #[inline]
+    pub fn lookup(c: char) -> bool {
+        debug_assert!(!c.is_ascii());
+        (c as u32) >= 0xad && lookup_slow(c)
+    }
+
+    #[inline(never)]
+    fn lookup_slow(c: char) -> bool {
+        const {
+            assert!(SHORT_OFFSET_RUNS.last().unwrap().0 > char::MAX as u32);
+            let mut i = 0;
+            while i < SHORT_OFFSET_RUNS.len() {
+                assert!(SHORT_OFFSET_RUNS[i].start_index() < OFFSETS.len());
+                i += 1;
+            }
+        }
+        // SAFETY: We just ensured the last element of `SHORT_OFFSET_RUNS` is greater than `std::char::MAX`
+        // and the start indices of all elements in `SHORT_OFFSET_RUNS` are smaller than `OFFSETS.len()`.
+        unsafe { super::skip_search(c, &SHORT_OFFSET_RUNS, &OFFSETS) }
+    }
+}
+
 #[rustfmt::skip]
 pub mod grapheme_extend {
     use super::ShortOffsetRunHeader;

library/coretests/tests/unicode.rs

@@ -71,6 +71,15 @@ fn cn_planes_0_3() {
     test_boolean_property(test_data::CN_PLANES_0_3, unicode_data::cn_planes_0_3::lookup);
 }
 
+#[test]
+#[cfg_attr(miri, ignore)] // Miri is too slow
+fn default_ignorable_code_point() {
+    test_boolean_property(
+        test_data::DEFAULT_IGNORABLE_CODE_POINT,
+        unicode_data::default_ignorable_code_point::lookup,
+    );
+}
+
 #[test]
 #[cfg_attr(miri, ignore)] // Miri is too slow
 fn grapheme_extend() {

library/coretests/tests/unicode/test_data.rs

@@ -638,6 +638,16 @@ pub(super) static CN_PLANES_0_3: &[RangeInclusive<char>; 730] = &[
     '\u{3347a}'..='\u{3fffd}',
 ];
 
+#[rustfmt::skip]
+pub(super) static DEFAULT_IGNORABLE_CODE_POINT: &[RangeInclusive<char>; 17] = &[
+    '\u{ad}'..='\u{ad}', '\u{34f}'..='\u{34f}', '\u{61c}'..='\u{61c}', '\u{115f}'..='\u{1160}',
+    '\u{17b4}'..='\u{17b5}', '\u{180b}'..='\u{180f}', '\u{200b}'..='\u{200f}',
+    '\u{202a}'..='\u{202e}', '\u{2060}'..='\u{206f}', '\u{3164}'..='\u{3164}',
+    '\u{fe00}'..='\u{fe0f}', '\u{feff}'..='\u{feff}', '\u{ffa0}'..='\u{ffa0}',
+    '\u{fff0}'..='\u{fff8}', '\u{1bca0}'..='\u{1bca3}', '\u{1d173}'..='\u{1d17a}',
+    '\u{e0000}'..='\u{e0fff}',
+];
+
 #[rustfmt::skip]
 pub(super) static GRAPHEME_EXTEND: &[RangeInclusive<char>; 383] = &[
     '\u{300}'..='\u{36f}', '\u{483}'..='\u{489}', '\u{591}'..='\u{5bd}', '\u{5bf}'..='\u{5bf}',

src/tools/unicode-table-generator/src/main.rs

@@ -93,6 +93,7 @@ static PROPERTIES: &[&str] = &[
     "Case_Ignorable",
     "Cf",
     "Cn_Planes_0_3",
+    "Default_Ignorable_Code_Point",
     "Grapheme_Extend",
     "Lowercase",
     "Lt",