Change enum FilterType to SelectorFilterType

axunonb · axunonb · commit ab623c159f4c · 2025-11-11T10:56:39.000+01:00
Implement proposals from review:

* SelectorFilterType.Alphanumeric: alphanumeric characters (upper and lower case), plus '_' and '-'
* SelectorFilterType.VisualUnicodeChars: All Unicode characters are allowed in a selector, except 68 non-visual characters: Control Characters (U+0000–U+001F, U+007F), Format Characters (Category: Cf), Directional Formatting (Category: Cf), Invisible Separator, Common Combining Marks (Category: Mn), Whitespace Characters (non-glyph spacing).
diff --git a/src/SmartFormat.Tests/Core/ParserTests.cs b/src/SmartFormat.Tests/Core/ParserTests.cs
@@ -211,7 +211,7 @@ public void Parser_Error_Action_OutputErrorInResult()
         {
             Parser = new ParserSettings
                 {
-                    SelectorCharFilter = FilterType.Allowlist, // default
+                    SelectorCharFilter = SelectorFilterType.Alphanumeric, // default
                     ErrorAction = ParseErrorAction.OutputErrorInResult
                 }
         });
@@ -471,7 +471,7 @@ public void Escaping_TheEscapingCharacter_ShouldWork()
     [Test]
     public void Parsing_Selector_With_CharFromBlocklist_ShouldThrow()
     {
-        var settings = new SmartSettings { Parser = new ParserSettings { SelectorCharFilter = FilterType.Blocklist } };
+        var settings = new SmartSettings { Parser = new ParserSettings { SelectorCharFilter = SelectorFilterType.VisualUnicodeChars } };
         var parser = GetRegularParser(settings);
 
         // The newline character is in the default blocklist of disallowed characters
@@ -609,7 +609,7 @@ public void Selector_WorksWithAllUnicodeChars(string selector)
         // See https://github.com/axuno/SmartFormat/issues/454
 
         // settings must be set before parser instantiation
-        var settings = new SmartSettings { Parser = { SelectorCharFilter = FilterType.Blocklist } };
+        var settings = new SmartSettings { Parser = { SelectorCharFilter = SelectorFilterType.VisualUnicodeChars } };
         const string expected = "The Value";
         // The default formatter with default settings should be able to handle any
         // Unicode characters in selectors except the "magic" disallowed ones
@@ -751,7 +751,7 @@ public void ParseHtmlInput_Without_ParserSetting_IsHtml(string input, string sel
             StringFormatCompatibility = false,
             Parser = new ParserSettings
             {
-                SelectorCharFilter = FilterType.Blocklist,
+                SelectorCharFilter = SelectorFilterType.VisualUnicodeChars,
                 ErrorAction = ParseErrorAction.ThrowError,
                 ParseInputAsHtml = false
             }
@@ -780,7 +780,7 @@ public void ParseHtmlInput_Without_ParserSetting_IsHtml(string input, bool shoul
             StringFormatCompatibility = false,
             Parser = new ParserSettings
             {
-                SelectorCharFilter = FilterType.Allowlist,
+                SelectorCharFilter = SelectorFilterType.Alphanumeric,
                 ErrorAction = ParseErrorAction.ThrowError,
                 ParseInputAsHtml = false
             }
diff --git a/src/SmartFormat.Tests/Core/SettingsTests.cs b/src/SmartFormat.Tests/Core/SettingsTests.cs
@@ -27,21 +27,21 @@ public void ExistingSelectorCharacter_Should_Not_Be_Added()
         });
     }
 
-    [TestCase(FilterType.Allowlist)]
-    [TestCase(FilterType.Blocklist)]
-    public void ControlCharacters_Should_Be_Added_As_SelectorChars(FilterType filterType)
+    [TestCase(SelectorFilterType.Alphanumeric)]
+    [TestCase(SelectorFilterType.VisualUnicodeChars)]
+    public void NonVisualCharacters_Should_Be_AddedBack_As_SelectorChars(SelectorFilterType filterType)
     {
         var settings = new SmartSettings { Parser = { SelectorCharFilter = filterType } };
-        var controlChars = ParserSettings.ControlChars().ToList();
-        settings.Parser.AddCustomSelectorChars(controlChars);
+        var nonVisualChars = ParserSettings.NonVisualUnicodeCharacters;
+        settings.Parser.AddCustomSelectorChars(nonVisualChars);
         
         Assert.Multiple(() =>
         {
-            Assert.That(settings.Parser.CustomSelectorChars, Has.Count.EqualTo(controlChars.Count));
+            Assert.That(settings.Parser.CustomSelectorChars, Has.Count.EqualTo(nonVisualChars.Length));
             foreach (var c in settings.Parser.CustomSelectorChars)
             {
-                Assert.That(settings.Parser.GetSelectorChars(), filterType == FilterType.Allowlist ? Does.Contain(c) : Does.Not.Contain(c),
-                    $"Control char U+{(int) c:X4} should be allowed as selector char.");
+                Assert.That(settings.Parser.GetSelectorChars(), filterType == SelectorFilterType.Alphanumeric ? Does.Contain(c) : Does.Not.Contain(c),
+                    $"Character U+{(int) c:X4} should be allowed as selector char.");
             }
         });
     }
diff --git a/src/SmartFormat/Core/Settings/FilterType.cs b/src/SmartFormat/Core/Settings/FilterType.cs
diff --git a/src/SmartFormat/Core/Settings/ParserSettings.cs b/src/SmartFormat/Core/Settings/ParserSettings.cs
@@ -18,7 +18,7 @@ public class ParserSettings
 {
     private readonly List<char> _customSelectorChars = [];
     private readonly List<char> _customOperatorChars = [];
-    private FilterType _selectorCharFilter = FilterType.Allowlist;
+    private SelectorFilterType _selectorCharFilter = SelectorFilterType.Alphanumeric;
 
     private const string StandardAllowlist = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_-";
 
@@ -40,17 +40,21 @@ public class ParserSettings
     internal List<char> CustomOperatorChars => _customOperatorChars;
 
     /// <summary>
-    /// When <see cref="FilterType.Allowlist"/> (default) is set, an allowlist of selector characters is used.
+    /// When <see cref="SelectorFilterType.Alphanumeric"/> (default) is set, an allowlist of selector characters is used.
     /// The allowlist contains alphanumeric characters (upper and lower case), plus '_' and '-'.
     /// On top, any custom selector characters added with <see cref="AddCustomSelectorChars"/> are included.
     /// <para/>
-    /// When <see cref="FilterType.Blocklist"/>, all Unicode characters are allowed in a selector,
-    /// except control characters (ASCII 0-31 and 127). Excluded control characters can be added back
-    /// using <see cref="AddCustomSelectorChars"/>.
+    /// When <see cref="SelectorFilterType.VisualUnicodeChars"/> is set, all Unicode characters are allowed in a selector,
+    /// except 68 non-visual characters: Control Characters (U+0000–U+001F, U+007F), Format Characters (Category: Cf),
+    /// Directional Formatting (Category: Cf), Invisible Separator, Common Combining Marks (Category: Mn),
+    /// Whitespace Characters (non-glyph spacing).<br/>
+    /// Excluded characters can be added back using <see cref="AddCustomSelectorChars"/>.
+    /// <para/>
+    /// {}[]()\.? are characters with special functions that are never allowed.
     /// <para/>
     /// Changing this setting clears any custom operator characters added with <see cref="AddCustomOperatorChars"/>.
     /// </summary>
-    public FilterType SelectorCharFilter
+    public SelectorFilterType SelectorCharFilter
     {
         get
         {
@@ -68,7 +72,7 @@ public FilterType SelectorCharFilter
     /// This can be an allowlist, which contains explicitly allowed characters,
     /// or a blocklist, when all Unicode characters are allowed, except those from the blocklist.
     /// </summary>
-    internal CharSet GetSelectorChars() => SelectorCharFilter == FilterType.Allowlist ? CreateAllowlist() : CreateBlocklist();
+    internal CharSet GetSelectorChars() => SelectorCharFilter == SelectorFilterType.Alphanumeric ? CreateAllowlist() : CreateBlocklist();
 
     private CharSet CreateBlocklist()
     {
@@ -79,8 +83,7 @@ private CharSet CreateBlocklist()
         chars.AddRange(SelectorDelimitingChars.AsSpan());
         chars.AddRange(OperatorChars.AsSpan()); // no overlaps
         chars.AddRange(_customOperatorChars); // no overlaps
-        // Hard to visualize and debug, disallow by default - can be added back as custom selector chars
-        chars.AddRange(ControlChars());
+        chars.AddRange(NonVisualUnicodeCharacters.AsSpan());
 
         // Remove characters used as custom selector chars from the blocklist
         foreach (var c in _customSelectorChars) chars.Remove(c);
@@ -209,13 +212,90 @@ private CharSet CreateAllowlist()
     ];
 
     /// <summary>
-    /// Gets the set of control characters (ASCII 0-31 and 127).
+    /// All 68 non-visual Unicode characters that are typically not used in selectors.
     /// </summary>
-    internal static IEnumerable<char> ControlChars()
-    {
-        for (var i = 0; i <= 31; i++) yield return (char) i;
-        yield return (char) 127; // delete character
-    }
+    internal static char[] NonVisualUnicodeCharacters =
+    [
+        // Control Characters (U+0000–U+001F, U+007F)
+        '\u0000', // NULL – string terminator
+        '\u0001', // START OF HEADING – protocol control
+        '\u0002', // START OF TEXT – protocol control
+        '\u0003', // END OF TEXT – protocol control
+        '\u0004', // END OF TRANSMISSION – protocol control
+        '\u0005', // ENQUIRY – request for response
+        '\u0006', // ACKNOWLEDGE – positive response
+        '\u0007', // BELL – triggers alert
+        '\u0008', // BACKSPACE – moves cursor back
+        '\u0009', // CHARACTER TABULATION – horizontal tab
+        '\u000A', // LINE FEED – line break
+        '\u000B', // LINE TABULATION – vertical tab
+        '\u000C', // FORM FEED – page break
+        '\u000D', // CARRIAGE RETURN – return to line start
+        '\u000E', // SHIFT OUT – alternate character set
+        '\u000F', // SHIFT IN – return to standard set
+        '\u0010', // DATA LINK ESCAPE – protocol framing
+        '\u0011', // DEVICE CONTROL 1 – device-specific
+        '\u0012', // DEVICE CONTROL 2 – device-specific
+        '\u0013', // DEVICE CONTROL 3 – device-specific
+        '\u0014', // DEVICE CONTROL 4 – device-specific
+        '\u0015', // NEGATIVE ACKNOWLEDGE – error signal
+        '\u0016', // SYNCHRONOUS IDLE – timing control
+        '\u0017', // END OF TRANSMISSION BLOCK – block end
+        '\u0018', // CANCEL – cancel transmission
+        '\u0019', // END OF MEDIUM – physical medium end
+        '\u001A', // SUBSTITUTE – invalid character
+        '\u001B', // ESCAPE – escape sequence initiator
+        '\u001C', // FILE SEPARATOR – data structuring
+        '\u001D', // GROUP SEPARATOR – data structuring
+        '\u001E', // RECORD SEPARATOR – data structuring
+        '\u001F', // UNIT SEPARATOR – data structuring
+        '\u007F', // DELETE – erase character
+
+        // Format Characters (Category: Cf)
+        '\u200B', // ZERO WIDTH SPACE – invisible space
+        '\u200C', // ZERO WIDTH NON-JOINER – prevents ligature
+        '\u200D', // ZERO WIDTH JOINER – forces ligature
+        '\u2060', // WORD JOINER – prevents line break
+        '\uFEFF', // ZERO WIDTH NO-BREAK SPACE – BOM or NBSP
+
+        // Directional Formatting (Category: Cf)
+        '\u202A', // LEFT-TO-RIGHT EMBEDDING – sets LTR context
+        '\u202B', // RIGHT-TO-LEFT EMBEDDING – sets RTL context
+        '\u202C', // POP DIRECTIONAL FORMATTING – ends override
+        '\u202D', // LEFT-TO-RIGHT OVERRIDE – forces LTR rendering
+        '\u202E', // RIGHT-TO-LEFT OVERRIDE – forces RTL rendering
+        '\u2066', // LEFT-TO-RIGHT ISOLATE – isolates LTR segment
+        '\u2067', // RIGHT-TO-LEFT ISOLATE – isolates RTL segment
+        '\u2068', // FIRST STRONG ISOLATE – isolates with inferred direction
+        '\u2069', // POP DIRECTIONAL ISOLATE – ends isolate
+
+        // Invisible Separator
+        '\u2063', // INVISIBLE SEPARATOR – semantic boundary marker
+
+        // Common Combining Marks (Category: Mn)
+        '\u0300', // COMBINING GRAVE ACCENT – diacritic (invisible alone)
+        '\u0301', // COMBINING ACUTE ACCENT – diacritic (invisible alone)
+        '\u0302', // COMBINING CIRCUMFLEX ACCENT – diacritic (invisible alone)
+        '\u0308', // COMBINING DIAERESIS – diacritic (invisible alone)
+
+        // Whitespace Characters (non-glyph spacing)
+        '\u00A0', // NO-BREAK SPACE – non-breaking space
+        '\u1680', // OGHAM SPACE MARK – special spacing
+        '\u2000', // EN QUAD – fixed-width space
+        '\u2001', // EM QUAD – fixed-width space
+        '\u2002', // EN SPACE – fixed-width space
+        '\u2003', // EM SPACE – fixed-width space
+        '\u2004', // THREE-PER-EM SPACE – narrow space
+        '\u2005', // FOUR-PER-EM SPACE – narrow space
+        '\u2006', // SIX-PER-EM SPACE – narrow space
+        '\u2007', // FIGURE SPACE – aligns digits
+        '\u2008', // PUNCTUATION SPACE – aligns punctuation
+        '\u2009', // THIN SPACE – narrow space
+        '\u200A', // HAIR SPACE – ultra-thin space
+        '\u202F', // NARROW NO-BREAK SPACE – narrow NBSP
+        '\u205F', // MEDIUM MATHEMATICAL SPACE – math spacing
+        '\u3000' // IDEOGRAPHIC SPACE – full-width CJK space
+    ];
 
     /// <summary>
     /// Add a list of allowable selector characters on top of the default selector characters.
@@ -225,26 +305,23 @@ internal static IEnumerable<char> ControlChars()
     /// On top, any custom selector characters added with <see cref="AddCustomSelectorChars"/> are included.
     /// <para/>
     /// When <see cref="SelectorCharFilter"/> is <see langword="false"/>, all Unicode characters are allowed in a selector,
-    /// except control characters (ASCII 0-31 and 127). Excluded control characters can be added back
-    /// using <see cref="AddCustomSelectorChars"/>.
+    /// except 68 non-visual characters. Excluded characters can be added back using <see cref="AddCustomSelectorChars"/>.
     /// <para/>
     /// Operator chars and selector chars must be different.
     /// </summary>
     public void AddCustomSelectorChars(IList<char> characters)
     {
-        var controlChars = ControlChars().ToList();
-
         foreach (var c in characters)
         {
             // Explicitly disallow certain characters
             if (SelectorDelimitingChars.Contains(c) || c == CharLiteralEscapeChar
                 || OperatorChars.Contains(c) || CustomOperatorChars.Contains(c))
                 throw new ArgumentException($"Cannot add '{c}' as a custom selector character. It is disallowed or in use as an operator character.");
 
-            if (controlChars.Contains(c))
+            if (NonVisualUnicodeCharacters.Contains(c))
                 _customSelectorChars.Add(c);
 
-            if (SelectorCharFilter == FilterType.Allowlist && !(StandardAllowlist.Contains(c) || _customSelectorChars.Contains(c))) _customSelectorChars.Add(c);
+            if (SelectorCharFilter == SelectorFilterType.Alphanumeric && !(StandardAllowlist.Contains(c) || _customSelectorChars.Contains(c))) _customSelectorChars.Add(c);
         }
     }
 
diff --git a/src/SmartFormat/Core/Settings/SelectorFilterType.cs b/src/SmartFormat/Core/Settings/SelectorFilterType.cs
@@ -0,0 +1,27 @@
+﻿// 
+// Copyright SmartFormat Project maintainers and contributors.
+// Licensed under the MIT license.
+
+namespace SmartFormat.Core.Settings;
+
+/// <summary>
+/// Determines the filter type for allowed or disallowed characters.
+/// </summary>
+public enum SelectorFilterType    
+{
+    /// <summary>
+    /// Use a list of characters that are allowed. The default characters are<br/>
+    /// alphanumeric characters (upper and lower case), plus '_' and '-'.<br/>
+    /// </summary>
+    Alphanumeric,
+
+    /// <summary>
+    /// All Unicode characters are allowed in a selector, except 68 non-visual characters:
+    /// Control Characters (U+0000–U+001F, U+007F), Format Characters (Category: Cf),
+    /// Directional Formatting (Category: Cf), Invisible Separator, Common Combining Marks (Category: Mn),
+    /// Whitespace Characters (non-glyph spacing).
+    /// <para/>
+    /// {}[]()\.? are characters with special functions that are never allowed.
+    /// </summary>
+    VisualUnicodeChars
+}

Original file line number	Diff line number	Diff line change
`@@ -211,7 +211,7 @@ public void Parser_Error_Action_OutputErrorInResult()`
`211`	`211`	`{`
`212`	`212`	`Parser = new ParserSettings`
`213`	`213`	`{`
`214`		`- SelectorCharFilter = FilterType.Allowlist, // default`
	`214`	`+ SelectorCharFilter = SelectorFilterType.Alphanumeric, // default`
`215`	`215`	`ErrorAction = ParseErrorAction.OutputErrorInResult`
`216`	`216`	`}`
`217`	`217`	`});`
`@@ -471,7 +471,7 @@ public void Escaping_TheEscapingCharacter_ShouldWork()`
`471`	`471`	`[Test]`
`472`	`472`	`public void Parsing_Selector_With_CharFromBlocklist_ShouldThrow()`
`473`	`473`	`{`
`474`		`- var settings = new SmartSettings { Parser = new ParserSettings { SelectorCharFilter = FilterType.Blocklist } };`
	`474`	`+ var settings = new SmartSettings { Parser = new ParserSettings { SelectorCharFilter = SelectorFilterType.VisualUnicodeChars } };`
`475`	`475`	`var parser = GetRegularParser(settings);`
`476`	`476`
`477`	`477`	`// The newline character is in the default blocklist of disallowed characters`
`@@ -609,7 +609,7 @@ public void Selector_WorksWithAllUnicodeChars(string selector)`
`609`	`609`	`// See https://github.com/axuno/SmartFormat/issues/454`
`610`	`610`
`611`	`611`	`// settings must be set before parser instantiation`
`612`		`- var settings = new SmartSettings { Parser = { SelectorCharFilter = FilterType.Blocklist } };`
	`612`	`+ var settings = new SmartSettings { Parser = { SelectorCharFilter = SelectorFilterType.VisualUnicodeChars } };`
`613`	`613`	`const string expected = "The Value";`
`614`	`614`	`// The default formatter with default settings should be able to handle any`
`615`	`615`	`// Unicode characters in selectors except the "magic" disallowed ones`
`@@ -751,7 +751,7 @@ public void ParseHtmlInput_Without_ParserSetting_IsHtml(string input, string sel`
`751`	`751`	`StringFormatCompatibility = false,`
`752`	`752`	`Parser = new ParserSettings`
`753`	`753`	`{`
`754`		`- SelectorCharFilter = FilterType.Blocklist,`
	`754`	`+ SelectorCharFilter = SelectorFilterType.VisualUnicodeChars,`
`755`	`755`	`ErrorAction = ParseErrorAction.ThrowError,`
`756`	`756`	`ParseInputAsHtml = false`
`757`	`757`	`}`
`@@ -780,7 +780,7 @@ public void ParseHtmlInput_Without_ParserSetting_IsHtml(string input, bool shoul`
`780`	`780`	`StringFormatCompatibility = false,`
`781`	`781`	`Parser = new ParserSettings`
`782`	`782`	`{`
`783`		`- SelectorCharFilter = FilterType.Allowlist,`
	`783`	`+ SelectorCharFilter = SelectorFilterType.Alphanumeric,`
`784`	`784`	`ErrorAction = ParseErrorAction.ThrowError,`
`785`	`785`	`ParseInputAsHtml = false`
`786`	`786`	`}`
Original file line number	Diff line number	Diff line change
`@@ -27,21 +27,21 @@ public void ExistingSelectorCharacter_Should_Not_Be_Added()`
`27`	`27`	`});`
`28`	`28`	`}`
`29`	`29`
`30`		`- [TestCase(FilterType.Allowlist)]`
`31`		`- [TestCase(FilterType.Blocklist)]`
`32`		`- public void ControlCharacters_Should_Be_Added_As_SelectorChars(FilterType filterType)`
	`30`	`+ [TestCase(SelectorFilterType.Alphanumeric)]`
	`31`	`+ [TestCase(SelectorFilterType.VisualUnicodeChars)]`
	`32`	`+ public void NonVisualCharacters_Should_Be_AddedBack_As_SelectorChars(SelectorFilterType filterType)`
`33`	`33`	`{`
`34`	`34`	`var settings = new SmartSettings { Parser = { SelectorCharFilter = filterType } };`
`35`		`- var controlChars = ParserSettings.ControlChars().ToList();`
`36`		`- settings.Parser.AddCustomSelectorChars(controlChars);`
	`35`	`+ var nonVisualChars = ParserSettings.NonVisualUnicodeCharacters;`
	`36`	`+ settings.Parser.AddCustomSelectorChars(nonVisualChars);`
`37`	`37`
`38`	`38`	`Assert.Multiple(() =>`
`39`	`39`	`{`
`40`		`- Assert.That(settings.Parser.CustomSelectorChars, Has.Count.EqualTo(controlChars.Count));`
	`40`	`+ Assert.That(settings.Parser.CustomSelectorChars, Has.Count.EqualTo(nonVisualChars.Length));`
`41`	`41`	`foreach (var c in settings.Parser.CustomSelectorChars)`
`42`	`42`	`{`
`43`		`- Assert.That(settings.Parser.GetSelectorChars(), filterType == FilterType.Allowlist ? Does.Contain(c) : Does.Not.Contain(c),`
`44`		`- $"Control char U+{(int) c:X4} should be allowed as selector char.");`
	`43`	`+ Assert.That(settings.Parser.GetSelectorChars(), filterType == SelectorFilterType.Alphanumeric ? Does.Contain(c) : Does.Not.Contain(c),`
	`44`	`+ $"Character U+{(int) c:X4} should be allowed as selector char.");`
`45`	`45`	`}`
`46`	`46`	`});`
`47`	`47`	`}`