Harden YAML lexical boundaries

Use YAML s-white and ordinal token checks, consume legal BOMs, preserve flow scalar folding, and disambiguate anchor and alias colons without regressing the official corpus.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

Author: MariusStorhaug

src/functions/private/ConvertFrom-YamlByteOrderMark.ps1

@@ -0,0 +1,55 @@
+function ConvertFrom-YamlByteOrderMark {
+    <#
+        .SYNOPSIS
+        Consumes byte order marks at YAML stream and explicit document boundaries.
+    #>
+    [CmdletBinding()]
+    [OutputType([string])]
+    param (
+        [Parameter(Mandatory)]
+        [AllowEmptyString()]
+        [string] $Text
+    )
+
+    if ($Text.IndexOf([char] 0xFEFF) -lt 0) {
+        return $Text
+    }
+
+    $builder = [System.Text.StringBuilder]::new($Text.Length)
+    for ($index = 0; $index -lt $Text.Length; $index++) {
+        if ($Text[$index] -cne [char] 0xFEFF) {
+            [void] $builder.Append($Text[$index])
+            continue
+        }
+
+        $atStreamStart = $index -eq 0
+        $atLineStart = $index -gt 0 -and $Text[$index - 1] -ceq "`n"
+        $beforeDirective = $index + 1 -lt $Text.Length -and $Text[$index + 1] -ceq '%'
+        $beforeDocumentStart = $false
+        if ($index + 3 -lt $Text.Length -and
+            $Text.Substring($index + 1, 3).Equals(
+                '---',
+                [System.StringComparison]::Ordinal
+            )) {
+            $afterMarker = $index + 4
+            $beforeDocumentStart = $afterMarker -ge $Text.Length -or
+            (Test-YamlWhiteSpace -Character $Text[$afterMarker]) -or
+            $Text[$afterMarker] -ceq "`n"
+        }
+
+        if ($atStreamStart -or ($atLineStart -and ($beforeDirective -or $beforeDocumentStart))) {
+            continue
+        }
+
+        $before = $Text.Substring(0, $index)
+        $line = ([regex]::Matches($before, "`n")).Count
+        $lastBreak = $before.LastIndexOf("`n", [System.StringComparison]::Ordinal)
+        $column = if ($lastBreak -lt 0) { $before.Length } else { $before.Length - $lastBreak - 1 }
+        $mark = New-YamlMark -Index $index -Line $line -Column $column
+        throw (New-YamlException -Start $mark -End $mark -ErrorId 'YamlInvalidByteOrderMark' -Message (
+                'A YAML byte order mark is only allowed at the start of the stream or an explicit document.'
+            ))
+    }
+
+    return $builder.ToString()
+}

src/functions/private/Find-YamlCommentStart.ps1

@@ -0,0 +1,22 @@
+function Find-YamlCommentStart {
+    <#
+        .SYNOPSIS
+        Finds a comment indicator separated by YAML s-white.
+    #>
+    [CmdletBinding()]
+    [OutputType([int])]
+    param (
+        [Parameter(Mandatory)]
+        [AllowEmptyString()]
+        [string] $Text
+    )
+
+    for ($index = 0; $index -lt $Text.Length; $index++) {
+        if ($Text[$index] -ceq '#' -and (
+                $index -eq 0 -or (Test-YamlWhiteSpace -Character $Text[$index - 1])
+            )) {
+            return $index
+        }
+    }
+    return -1
+}

src/functions/private/Find-YamlMappingColon.ps1

@@ -8,7 +8,9 @@ function Find-YamlMappingColon {
     param (
         [Parameter(Mandatory)]
         [AllowEmptyString()]
-        [string] $Text
+        [string] $Text,
+
+        [switch] $AllowAnchorFallback
     )
 
     if ($Text.IndexOf(':') -lt 0) {
@@ -19,6 +21,7 @@ function Find-YamlMappingColon {
     $singleQuoted = $false
     $doubleQuoted = $false
     $atNodeStart = $true
+    $anchorColonCandidate = -1
     for ($index = 0; $index -lt $Text.Length; $index++) {
         $character = $Text[$index]
         if ($doubleQuoted) {
@@ -39,20 +42,27 @@ function Find-YamlMappingColon {
             }
             continue
         }
-        if ($atNodeStart -and [char]::IsWhiteSpace($character)) {
+        if ($atNodeStart -and (Test-YamlWhiteSpace -Character $character)) {
             continue
         }
         if ($atNodeStart -and $character -eq '&') {
             $index++
-            while ($index -lt $Text.Length -and -not [char]::IsWhiteSpace($Text[$index])) {
+            while ($index -lt $Text.Length -and
+                -not (Test-YamlWhiteSpace -Character $Text[$index]) -and
+                $Text[$index] -notin @(',', '[', ']', '{', '}')) {
+                if (Test-YamlMappingValueIndicator -Text $Text -Index $index) {
+                    $anchorColonCandidate = $index
+                }
                 $index++
             }
             $index--
             continue
         }
         if ($atNodeStart -and $character -eq '*') {
             $index++
-            while ($index -lt $Text.Length -and -not [char]::IsWhiteSpace($Text[$index])) {
+            while ($index -lt $Text.Length -and
+                -not (Test-YamlWhiteSpace -Character $Text[$index]) -and
+                $Text[$index] -notin @(',', '[', ']', '{', '}')) {
                 $index++
             }
             $index--
@@ -67,7 +77,7 @@ function Find-YamlMappingColon {
             } else {
                 $index++
                 while ($index -lt $Text.Length -and
-                    -not [char]::IsWhiteSpace($Text[$index]) -and
+                    -not (Test-YamlWhiteSpace -Character $Text[$index]) -and
                     $Text[$index] -notin @('[', ']', '{', '}', ',')) {
                     $index++
                 }
@@ -94,18 +104,24 @@ function Find-YamlMappingColon {
             ':' {
                 if ($flowDepth -eq 0) {
                     $nextIsSeparator = $index + 1 -ge $Text.Length
-                    $nextIsSeparator = $nextIsSeparator -or [char]::IsWhiteSpace($Text[$index + 1])
+                    $nextIsSeparator = $nextIsSeparator -or
+                    (Test-YamlWhiteSpace -Character $Text[$index + 1])
                     if ($nextIsSeparator) {
                         return $index
                     }
                 }
             }
             '#' {
-                if ($flowDepth -eq 0 -and ($index -eq 0 -or [char]::IsWhiteSpace($Text[$index - 1]))) {
+                if ($flowDepth -eq 0 -and (
+                        $index -eq 0 -or (Test-YamlWhiteSpace -Character $Text[$index - 1])
+                    )) {
                     return -1
                 }
             }
         }
     }
+    if ($AllowAnchorFallback) {
+        return $anchorColonCandidate
+    }
     return -1
 }

src/functions/private/Get-YamlContentWithoutComment.ps1

@@ -11,9 +11,9 @@ function Get-YamlContentWithoutComment {
         [string] $Text
     )
 
-    $comment = [regex]::Match($Text, '(?<!\S)#')
-    if ($comment.Success) {
-        return $Text.Substring(0, $comment.Index).TrimEnd()
+    $comment = Find-YamlCommentStart -Text $Text
+    if ($comment -ge 0) {
+        return $Text.Substring(0, $comment).TrimEnd(' ', "`t")
     }
-    return $Text.TrimEnd()
+    return $Text.TrimEnd(' ', "`t")
 }

src/functions/private/New-YamlReaderContext.ps1

@@ -36,8 +36,9 @@ function New-YamlReaderContext {
         [int] $MaxNumericLength
     )
 
-    Assert-YamlText -Yaml $Yaml
     $text = $Yaml.Replace("`r`n", "`n").Replace("`r", "`n")
+    $text = ConvertFrom-YamlByteOrderMark -Text $text
+    Assert-YamlText -Yaml $text
     $lines = [System.Text.RegularExpressions.Regex]::Split($text, "`n")
     $lineStarts = [int[]]::new($lines.Count)
     $offset = 0

src/functions/private/Read-YamlBlockKey.ps1

@@ -65,7 +65,7 @@ function Read-YamlBlockKey {
     $first = $rest[0]
     if ($first -in @(',', '[', ']', '{', '}', '#', '&', '*', '!', '|', '>', "'", '"', '%', '@', '`') -or
         ($first -in @('-', '?', ':') -and (
-            $rest.Length -gt 1 -and [char]::IsWhiteSpace($rest[1])
+            $rest.Length -gt 1 -and (Test-YamlWhiteSpace -Character $rest[1])
         ))) {
         throw (New-YamlException -Start $start -End $start -ErrorId 'YamlInvalidPlainScalar' -Message (
                 'The first character is not allowed in a YAML plain scalar.'
@@ -82,7 +82,7 @@ function Read-YamlBlockKey {
     $node = New-YamlSyntaxNode -Context $Context -Kind Scalar -Depth $Depth -Start $start -End $end
     Set-YamlParsedNodeProperty -Node $node -Tag $properties.Tag `
         -HasUnknownTag $properties.HasUnknownTag -Anchor $properties.Anchor -Context $Context
-    $node.Value = $rest.TrimEnd()
+    $node.Value = $rest.TrimEnd(' ', "`t")
     $node.Style = 'Plain'
     $node.IsPlainImplicit = [string]::IsNullOrEmpty($properties.Tag) -and
     -not $properties.HasUnknownTag

src/functions/private/Read-YamlBlockMapping.ps1

@@ -77,10 +77,10 @@ function Read-YamlBlockMapping {
         $isExplicit = Test-YamlIndicator -Text $content -Indicator '?'
         if ($isExplicit) {
             $afterQuestion = $content.Substring(1)
-            $leading = $afterQuestion.Length - $afterQuestion.TrimStart().Length
-            $keyText = $afterQuestion.TrimStart()
+            $leading = $afterQuestion.Length - $afterQuestion.TrimStart(' ', "`t").Length
+            $keyText = $afterQuestion.TrimStart(' ', "`t")
             $keyColumn = $contentColumn + 1 + $leading
-            if ([string]::IsNullOrWhiteSpace((Get-YamlContentWithoutComment -Text $keyText))) {
+            if ([string]::IsNullOrEmpty((Get-YamlContentWithoutComment -Text $keyText))) {
                 $Context.LineIndex++
                 Skip-YamlBlockTrivia -Context $Context
                 if ($Context.LineIndex -ge $Context.Lines.Count) {
@@ -104,9 +104,12 @@ function Read-YamlBlockMapping {
                     }
                 }
             } elseif (Test-YamlIndicator -Text $keyText -Indicator '-') {
-                $firstItem = $keyText.Substring(1).TrimStart()
+                $firstItem = $keyText.Substring(1).TrimStart(' ', "`t")
                 $dashColumn = $keyColumn
-                $itemColumn = $dashColumn + 1 + ($keyText.Substring(1).Length - $keyText.Substring(1).TrimStart().Length)
+                $itemColumn = $dashColumn + 1 + (
+                    $keyText.Substring(1).Length -
+                    $keyText.Substring(1).TrimStart(' ', "`t").Length
+                )
                 $key = Read-YamlBlockSequence -Context $Context -Indent $dashColumn -Depth ($Depth + 1) `
                     -FirstItemText $firstItem -FirstItemColumn $itemColumn
             } else {
@@ -130,10 +133,10 @@ function Read-YamlBlockMapping {
             if ($valueIndent -eq $Indent -and
                 (Test-YamlIndicator -Text $valueContent -Indicator ':')) {
                 $valueText = $valueContent.Substring(1)
-                $leading = $valueText.Length - $valueText.TrimStart().Length
-                $valueText = $valueText.TrimStart()
+                $leading = $valueText.Length - $valueText.TrimStart(' ', "`t").Length
+                $valueText = $valueText.TrimStart(' ', "`t")
                 $valueColumn = $Indent + 1 + $leading
-                if ([string]::IsNullOrWhiteSpace((Get-YamlContentWithoutComment -Text $valueText))) {
+                if ([string]::IsNullOrEmpty((Get-YamlContentWithoutComment -Text $valueText))) {
                     $valueLineNumber = $Context.LineIndex
                     $Context.LineIndex++
                     Skip-YamlBlockTrivia -Context $Context
@@ -171,7 +174,7 @@ function Read-YamlBlockMapping {
             continue
         }
 
-        $colon = Find-YamlMappingColon -Text $content
+        $colon = Find-YamlMappingColon -Text $content -AllowAnchorFallback
         if ($colon -lt 0) {
             $mark = New-YamlMark -Index ($Context.LineStarts[$lineNumber] + $contentColumn) `
                 -Line $lineNumber -Column $contentColumn
@@ -192,17 +195,17 @@ function Read-YamlBlockMapping {
                 -Line $lineNumber -Column $contentColumn
             $key = New-YamlEmptyScalar -Context $Context -Depth ($Depth + 1) -Mark $mark
         } else {
-            $keyText = $content.Substring(0, $colon).TrimEnd()
+            $keyText = $content.Substring(0, $colon).TrimEnd(' ', "`t")
             $key = Read-YamlBlockKey -Context $Context -Text $keyText -Line $lineNumber `
                 -Column $contentColumn -Depth ($Depth + 1)
         }
 
         $valueStart = $colon + 1
         $valueSource = $content.Substring($valueStart)
-        $leading = $valueSource.Length - $valueSource.TrimStart().Length
-        $valueText = $valueSource.TrimStart()
+        $leading = $valueSource.Length - $valueSource.TrimStart(' ', "`t").Length
+        $valueText = $valueSource.TrimStart(' ', "`t")
         $valueColumn = $contentColumn + $valueStart + $leading
-        if ([string]::IsNullOrWhiteSpace((Get-YamlContentWithoutComment -Text $valueText))) {
+        if ([string]::IsNullOrEmpty((Get-YamlContentWithoutComment -Text $valueText))) {
             $Context.LineIndex = $lineNumber + 1
             Skip-YamlBlockTrivia -Context $Context
             if ($Context.LineIndex -lt $Context.Lines.Count) {
@@ -242,7 +245,8 @@ function Read-YamlBlockMapping {
             }
             $Context.LineIndex = $lineNumber
             $value = Read-YamlBlockNode -Context $Context -ParentIndent $Indent -Depth ($Depth + 1) `
-                -Segment $valueText -SegmentColumn $valueColumn -AllowIndentlessSequence
+                -Segment $valueText -SegmentColumn $valueColumn -AllowIndentlessSequence `
+                -DisallowCompactMapping
         }
         $node.Entries.Add([pscustomobject]@{ Key = $key; Value = $value })
     }

src/functions/private/Read-YamlBlockNode.ps1

@@ -29,7 +29,9 @@ function Read-YamlBlockNode {
         [AllowEmptyString()]
         [string] $PendingAnchor = '',
 
-        [switch] $AllowIndentlessSequence
+        [switch] $AllowIndentlessSequence,
+
+        [switch] $DisallowCompactMapping
     )
 
     $hasSegment = $PSBoundParameters.ContainsKey('Segment')
@@ -67,8 +69,9 @@ function Read-YamlBlockNode {
     if (
         (-not [string]::IsNullOrEmpty($PendingTag) -or $PendingUnknownTag -or
         -not [string]::IsNullOrEmpty($PendingAnchor)) -and
+        -not $DisallowCompactMapping -and
         -not (Test-YamlIndicator -Text $Segment -Indicator '-') -and
-        ((Find-YamlMappingColon -Text $Segment) -ge 0 -or
+        ((Find-YamlMappingColon -Text $Segment -AllowAnchorFallback) -ge 0 -or
         (Test-YamlIndicator -Text $Segment -Indicator '?'))
     ) {
         return Read-YamlBlockMapping -Context $Context -Indent $SegmentColumn -Depth $Depth `
@@ -78,8 +81,9 @@ function Read-YamlBlockNode {
     if (
         [string]::IsNullOrEmpty($PendingTag) -and -not $PendingUnknownTag -and
         [string]::IsNullOrEmpty($PendingAnchor) -and
+        -not $DisallowCompactMapping -and
         -not (Test-YamlIndicator -Text $Segment -Indicator '-') -and
-        ((Find-YamlMappingColon -Text $Segment) -ge 0 -or
+        ((Find-YamlMappingColon -Text $Segment -AllowAnchorFallback) -ge 0 -or
         (Test-YamlIndicator -Text $Segment -Indicator '?'))
     ) {
         return Read-YamlBlockMapping -Context $Context -Indent $SegmentColumn -Depth $Depth `
@@ -111,7 +115,7 @@ function Read-YamlBlockNode {
     $rest = Get-YamlContentWithoutComment -Text $restSource
     $contentColumn = $SegmentColumn + $properties.Consumed
 
-    if ([string]::IsNullOrWhiteSpace($rest)) {
+    if ([string]::IsNullOrEmpty($rest)) {
         $Context.LineIndex++
         Skip-YamlBlockTrivia -Context $Context
         if ($Context.LineIndex -ge $Context.Lines.Count -or
@@ -154,23 +158,24 @@ function Read-YamlBlockNode {
         }
         $firstSource = $rest.Substring(1)
         if ($firstSource.IndexOf("`t", [System.StringComparison]::Ordinal) -ge 0 -and
-            $firstSource.Trim() -eq '-') {
+            $firstSource.Trim(' ', "`t") -ceq '-') {
             $mark = New-YamlMark -Index ($Context.LineStarts[$lineNumber] + $contentColumn + 1) `
                 -Line $lineNumber -Column ($contentColumn + 1)
             throw (New-YamlException -Start $mark -End $mark -ErrorId 'YamlInvalidIndentation' -Message (
                     'Tabs cannot separate adjacent block sequence indicators.'
                 ))
         }
         $first = $firstSource
-        $leading = $first.Length - $first.TrimStart().Length
-        $first = $first.TrimStart()
+        $leading = $first.Length - $first.TrimStart(' ', "`t").Length
+        $first = $first.TrimStart(' ', "`t")
         return Read-YamlBlockSequence -Context $Context -Indent $contentColumn -Depth $Depth -Tag $tag `
             -HasUnknownTag $unknownTag -Anchor $anchor -FirstItemText $first `
             -FirstItemColumn ($contentColumn + 1 + $leading)
     }
 
-    if ((Find-YamlMappingColon -Text $rest) -ge 0 -or
-        (Test-YamlIndicator -Text $rest -Indicator '?')) {
+    if (-not $DisallowCompactMapping -and (
+            (Find-YamlMappingColon -Text $rest -AllowAnchorFallback) -ge 0 -or
+            (Test-YamlIndicator -Text $rest -Indicator '?'))) {
         return Read-YamlBlockMapping -Context $Context -Indent $contentColumn -Depth $Depth -Tag $tag `
             -HasUnknownTag $unknownTag -Anchor $anchor -FirstText $rest -FirstColumn $contentColumn
     }

src/functions/private/Read-YamlBlockSequence.ps1

@@ -62,11 +62,14 @@ function Read-YamlBlockSequence {
             if (-not (Test-YamlIndicator -Text $content -Indicator '-')) {
                 break
             }
-            $rest = $content.Substring(1).TrimStart()
-            $itemColumn = $Indent + 1 + ($content.Substring(1).Length - $content.Substring(1).TrimStart().Length)
+            $rest = $content.Substring(1).TrimStart(' ', "`t")
+            $itemColumn = $Indent + 1 + (
+                $content.Substring(1).Length -
+                $content.Substring(1).TrimStart(' ', "`t").Length
+            )
         }
 
-        if ([string]::IsNullOrWhiteSpace((Get-YamlContentWithoutComment -Text $rest))) {
+        if ([string]::IsNullOrEmpty((Get-YamlContentWithoutComment -Text $rest))) {
             $line = $Context.LineIndex
             $Context.LineIndex++
             Skip-YamlBlockTrivia -Context $Context

src/functions/private/Read-YamlDirectiveBlock.ps1

@@ -71,7 +71,7 @@ function Read-YamlDirectiveBlock {
             }
             $tagHandles[$handle] = $prefix
         } elseif ($directive -cnotmatch (
-                '^%[A-Za-z0-9]+(?:[ \t]+[^#\s][^#]*?)?(?:[ \t]+#.*)?$'
+                '^%[A-Za-z0-9]+(?:[ \t]+[^# \t][^#]*?)?(?:[ \t]+#.*)?$'
             )) {
             throw (New-YamlException -Start $mark -End $mark `
                     -ErrorId 'YamlInvalidDirective' -Message (