Switching from Span to Memory

This is to help the next layer, the parser, do its job a bit easier

Author: Turnerj

src/TurnerSoftware.CascadingStyles/CssToken.cs

@@ -0,0 +1,101 @@
+using System;
+using System.ComponentModel;
+
+#pragma warning disable 0809 // Obsolete member overrides non-obsolete member
+
+namespace TurnerSoftware.CascadingStyles.Parsing
+{
+	public struct CssToken
+	{
+		public readonly ReadOnlyMemory<char> RawValue;
+		public readonly CssTokenType Type;
+		public readonly CssTokenFlag Flags;
+		public readonly ReadOnlyMemory<char> RawSecondaryValue;
+
+		public static CssToken EndOfFile => new(CssTokenType.EndOfFile);
+
+		public CssToken(CssTokenType type)
+		{
+			RawValue = default;
+			Type = type;
+			Flags = CssTokenFlag.None;
+			RawSecondaryValue = default;
+		}
+
+		public CssToken(ReadOnlyMemory<char> rawValue, CssTokenType type, CssTokenFlag flags = CssTokenFlag.None, ReadOnlyMemory<char> secondaryValue = default)
+		{
+			RawValue = rawValue;
+			Type = type;
+			Flags = flags;
+			RawSecondaryValue = secondaryValue;
+		}
+
+		public static bool operator !=(CssToken left, CssToken right) => !(left == right);
+
+		public static bool operator ==(CssToken left, CssToken right) =>
+			left.RawValue.Equals(right) &&
+			left.Type == right.Type &&
+			left.Flags == right.Flags &&
+			left.RawSecondaryValue.Equals(right);
+
+		[Obsolete("Equals() on CssToken will always throw an exception. Use == instead.")]
+		[EditorBrowsable(EditorBrowsableState.Never)]
+		public override bool Equals(object obj) => throw new NotSupportedException();
+		[Obsolete("GetHashCode() on CssToken will always throw an exception.")]
+		[EditorBrowsable(EditorBrowsableState.Never)]
+		public override int GetHashCode() => throw new NotSupportedException();
+	}
+
+	public enum CssTokenType
+	{
+		EndOfFile,
+		Identifier,
+		Function,
+		AtKeyword,
+		Hash,
+		String,
+		BadString,
+		Url,
+		BadUrl,
+		Delimiter,
+		Number,
+		Percentage,
+		Dimension,
+		Whitespace,
+		CDO,
+		CDC,
+		Colon,
+		Semicolon,
+		Comma,
+		LeftSquareBracket,
+		RightSquareBracket,
+		LeftParenthesis,
+		RightParenthesis,
+		LeftCurlyBracket,
+		RightCurlyBracket
+	}
+
+	public enum CssTokenFlag
+	{
+		/// <summary>
+		/// No token flag is set
+		/// </summary>
+		None,
+		/// <summary>
+		/// The default flag for &lt;hash-token&gt;.
+		/// </summary>
+		Hash_Unrestricted,
+		/// <summary>
+		/// Defines that the &lt;hash-token&gt; is a valid ID.
+		/// </summary>
+		Hash_Id,
+		/// <summary>
+		/// Whether a &lt;number-token> or &lt;dimension&gt; is a whole number.
+		/// </summary>
+		Number_Integer,
+		/// <summary>
+		/// Whether a &lt;number-token> or &lt;dimension&gt; is a fraction.
+		/// </summary>
+		Number_Number
+	}
+}

src/TurnerSoftware.CascadingStyles/Parsing/CssToken.cs

@@ -1,6 +1,6 @@
 using System;
 
-namespace TurnerSoftware.CascadingStyles
+namespace TurnerSoftware.CascadingStyles.Parsing
 {
 	/// <summary>
 	/// Provides CSS tokenization support, closely following the <a href="https://drafts.csswg.org/css-syntax/#tokenization">official CSS specification</a>.
@@ -15,15 +15,15 @@ namespace TurnerSoftware.CascadingStyles
 	/// These deviations from the standard allow the tokenizer to be allocation-free in its processing.
 	/// </para>
 	/// </remarks>
-	public ref struct CssReader
+	public struct CssTokenizer
 	{
 		public const char EndOfFile = char.MaxValue;
 
-		private readonly ReadOnlySpan<char> InputStream;
+		private readonly ReadOnlyMemory<char> InputStream;
 		private int RegionStartIndex;
 		private int IndexOffset;
 
-		public CssReader(ReadOnlySpan<char> value)
+		public CssTokenizer(ReadOnlyMemory<char> value)
 		{
 			InputStream = value;
 			RegionStartIndex = 0;
@@ -38,7 +38,7 @@ private char Current
 			{
 				if (CurrentIndex < InputStream.Length)
 				{
-					return InputStream[CurrentIndex];
+					return InputStream.Span[CurrentIndex];
 				}
 
 				return EndOfFile;
@@ -48,14 +48,14 @@ private char Current
 		/// <summary>
 		/// The unconsumed "remaining" input stream data
 		/// </summary>
-		private ReadOnlySpan<char> RemainingInputStream => InputStream[CurrentIndex..];
+		private ReadOnlyMemory<char> RemainingInputStream => InputStream.Slice(CurrentIndex);
 
 		private char Peek(int distance = 1)
 		{
 			var nextIndex = CurrentIndex + distance;
 			if (nextIndex < InputStream.Length)
 			{
-				return InputStream[nextIndex];
+				return InputStream.Span[nextIndex];
 			}
 
 			return EndOfFile;
@@ -65,17 +65,17 @@ private char Peek(int distance = 1)
 		/// Returns the region starting from <see cref="RegionStartIndex"/> to, but not including, <see cref="CurrentIndex"/>.
 		/// </summary>
 		/// <returns></returns>
-		private ReadOnlySpan<char> GetRegion()
+		private ReadOnlyMemory<char> GetRegion()
 		{
-			return InputStream[RegionStartIndex..CurrentIndex];
+			return InputStream.Slice(RegionStartIndex, IndexOffset);
 		}
 
 		/// <summary>
 		/// Finalizes the current region and starts a new region.
 		/// Region is from <see cref="RegionStartIndex"/> to, but not including, <see cref="CurrentIndex"/>.
 		/// </summary>
 		/// <returns></returns>
-		private ReadOnlySpan<char> ConsumeRegion()
+		private ReadOnlyMemory<char> ConsumeRegion()
 		{
 			var result = GetRegion();
 			StartNewCaptureRegion();
@@ -136,7 +136,7 @@ public bool NextToken(out CssToken token)
 		{
 			if (RegionStartIndex == InputStream.Length)
 			{
-				token = default;
+				token = CssToken.EndOfFile;
 				return false;
 			}
 
@@ -248,7 +248,7 @@ public bool NextToken(out CssToken token)
 					}
 					else
 					{
-						token = default;
+						token = CssToken.EndOfFile;
 						return false;
 					}
 					break;
@@ -266,7 +266,7 @@ private void ConsumeCommentsIfAny()
 			// followed by a U+002F SOLIDUS (/), or up to an EOF code point.
 			if (Current == '/' && Peek() == '*')
 			{
-				var matchIndex = RemainingInputStream.IndexOf("*/");
+				var matchIndex = RemainingInputStream.Span.IndexOf("*/");
 				// No closing comment found / it closes at the end of the file
 				if (matchIndex == -1)
 				{
@@ -531,7 +531,7 @@ private CssToken ConsumeIdentifierLikeToToken()
 			var identifier = ConsumeIdentifier();
 
 			// If identifier matches "url" (case-insensitive) and the next input code point is U+0028 LEFT PARENTHESIS, consume it
-			if (identifier.Equals("url", StringComparison.OrdinalIgnoreCase) && Current == '(')
+			if (identifier.Span.Equals("url", StringComparison.OrdinalIgnoreCase) && Current == '(')
 			{
 				ConsumeCurrent();
 
@@ -654,10 +654,12 @@ private CssToken ConsumeUrlToToken()
 		/// <returns></returns>
 		private CssToken ConsumeRemnantsOfBadUrlToToken()
 		{
+			var remainingInputSpan = RemainingInputStream.Span;
+
 			// Repeatedly consume the next input code point from the stream:
 			while (true)
 			{
-				var nextRightParenthesis = RemainingInputStream.IndexOf(')');
+				var nextRightParenthesis = remainingInputSpan.IndexOf(')');
 
 				// If a U+0029 RIGHT PARENTHESIS ()) or EOF is found, consume and return
 				// In this case, if no right parenthesis is found, we can jump to the EOF
@@ -671,7 +673,7 @@ private CssToken ConsumeRemnantsOfBadUrlToToken()
 				// Consume an escaped code point. This allows an escaped right parenthesis ("\)")
 				// to be encountered without ending the <bad-url-token>.
 				// We backtrack from the index of the right parenthesis to check
-				var isEscaped = nextRightParenthesis > 0 && RemainingInputStream[nextRightParenthesis - 1] == '\\';
+				var isEscaped = nextRightParenthesis > 0 && remainingInputSpan[nextRightParenthesis - 1] == '\\';
 
 				// Either way, we consume all the characters we've found
 				ConsumeNext(nextRightParenthesis + 1);
@@ -693,7 +695,7 @@ private CssToken ConsumeRemnantsOfBadUrlToToken()
 		/// </summary>
 		/// <param name="seenDecimal"></param>
 		/// <returns></returns>
-		private void ConsumeNumber(out ReadOnlySpan<char> number, out CssTokenFlag flags)
+		private void ConsumeNumber(out ReadOnlyMemory<char> number, out CssTokenFlag flags)
 		{
 			var startIndex = CurrentIndex;
 
@@ -757,7 +759,7 @@ private void ConsumeNumber(out ReadOnlySpan<char> number, out CssTokenFlag flags
 		/// Assumes that <see cref="Current"/> is the start of the identifier.
 		/// </remarks>
 		/// <returns></returns>
-		private ReadOnlySpan<char> ConsumeIdentifier()
+		private ReadOnlyMemory<char> ConsumeIdentifier()
 		{
 			var startIndex = CurrentIndex;
 			while (true)

…nerSoftware.CascadingStyles/CssReader.cs ….CascadingStyles/Parsing/CssTokenizer.cssrc/TurnerSoftware.CascadingStyles/CssReader.cs renamed to src/TurnerSoftware.CascadingStyles/Parsing/CssTokenizer.cs src/TurnerSoftware.CascadingStyles/CssReader.cs renamed to src/TurnerSoftware.CascadingStyles/Parsing/CssTokenizer.cs

@@ -1,7 +1,7 @@
 using System;
 using System.Text;
 
-namespace TurnerSoftware.CascadingStyles
+namespace TurnerSoftware.CascadingStyles.Parsing
 {
 	public static class TokenizationHelper
 	{
@@ -98,13 +98,13 @@ public static string UnescapeValue(ReadOnlySpan<char> value)
 			return value.ToString();
 		}
 
-		public static string UnescapeValue(this CssToken token) => UnescapeValue(token.RawValue);
+		public static string UnescapeValue(this CssToken token) => UnescapeValue(token.RawValue.Span);
 
-		public static string UnescapeSecondaryValue(this CssToken token) => UnescapeValue(token.RawSecondaryValue);
+		public static string UnescapeSecondaryValue(this CssToken token) => UnescapeValue(token.RawSecondaryValue.Span);
 
 		public static decimal GetNumber(this CssToken token) => token.Type switch
 		{
-			CssTokenType.Number or CssTokenType.Dimension => decimal.Parse(token.RawValue, System.Globalization.NumberStyles.Number),
+			CssTokenType.Number or CssTokenType.Dimension => decimal.Parse(token.RawValue.Span, System.Globalization.NumberStyles.Number),
 			_ => throw new InvalidOperationException("Token is not of a valid type to decode a number from"),
 		};
 	}