Close final representation edge cases

MariusStorhaug · Copilot · MariusStorhaug · commit a24c58ad84c9 · 2026-07-24T01:04:11.000+02:00
Co-authored-by: Copilot App &lt;223556219+Copilot@users.noreply.github.com&gt;
diff --git a/src/functions/private/Assert-YamlNoByteOrderMark.ps1 b/src/functions/private/Assert-YamlNoByteOrderMark.ps1
@@ -0,0 +1,21 @@
+function Assert-YamlNoByteOrderMark {
+    <#
+        .SYNOPSIS
+        Rejects a raw byte order mark outside a quoted scalar or document prefix.
+    #>
+    [CmdletBinding()]
+    param (
+        [Parameter(Mandatory)]
+        [AllowEmptyString()]
+        [string] $Text,
+
+        [Parameter(Mandatory)]
+        [pscustomobject] $Mark
+    )
+
+    if ($Text.IndexOf([char] 0xFEFF) -ge 0) {
+        throw (New-YamlException -Start $Mark -End $Mark -ErrorId 'YamlInvalidByteOrderMark' -Message (
+                'A raw YAML byte order mark is only allowed in a quoted scalar or document prefix.'
+            ))
+    }
+}
diff --git a/src/functions/private/ConvertFrom-YamlByteOrderMark.ps1 b/src/functions/private/ConvertFrom-YamlByteOrderMark.ps1
@@ -32,14 +32,7 @@ function ConvertFrom-YamlByteOrderMark {
             continue
         }
 
-        $before = $Text.Substring(0, $index)
-        $line = ([regex]::Matches($before, "`n")).Count
-        $lastBreak = $before.LastIndexOf("`n", [System.StringComparison]::Ordinal)
-        $column = if ($lastBreak -lt 0) { $before.Length } else { $before.Length - $lastBreak - 1 }
-        $mark = New-YamlMark -Index $index -Line $line -Column $column
-        throw (New-YamlException -Start $mark -End $mark -ErrorId 'YamlInvalidByteOrderMark' -Message (
-                'A YAML byte order mark is only allowed at the start of the stream or a document prefix.'
-            ))
+        [void] $builder.Append($Text[$index])
     }
 
     return $builder.ToString()
diff --git a/src/functions/private/ConvertTo-YamlQuotedText.ps1 b/src/functions/private/ConvertTo-YamlQuotedText.ps1
@@ -50,7 +50,7 @@ function ConvertTo-YamlQuotedText {
             [void] $builder.Append('\"')
         } elseif ($code -eq 92) {
             [void] $builder.Append('\\')
-        } elseif ($code -lt 0x20 -or $code -eq 0x7F -or
+        } elseif ($code -lt 0x20 -or $code -eq 0x7F -or $code -eq 0xFEFF -or
             $code -eq 0xFFFE -or $code -eq 0xFFFF -or
             ($code -ge 0x80 -and $code -le 0x9F)) {
             [void] $builder.Append(('\u{0:X4}' -f $code))
diff --git a/src/functions/private/Get-YamlEffectiveTag.ps1 b/src/functions/private/Get-YamlEffectiveTag.ps1
@@ -14,8 +14,7 @@ function Get-YamlEffectiveTag {
         [object] $Value
     )
 
-    if (-not [string]::IsNullOrEmpty($Node.Tag) -and
-        -not $Node.Tag.Equals('!', [System.StringComparison]::Ordinal)) {
+    if (-not [string]::IsNullOrEmpty($Node.Tag)) {
         return $Node.Tag
     }
     if ($Node.Kind.Equals('Sequence', [System.StringComparison]::Ordinal)) {
@@ -24,7 +23,7 @@ function Get-YamlEffectiveTag {
     if ($Node.Kind.Equals('Mapping', [System.StringComparison]::Ordinal)) {
         return 'tag:yaml.org,2002:map'
     }
-    if ($Node.Tag.Equals('!', [System.StringComparison]::Ordinal)) {
+    if ($Node.HasUnknownTag) {
         return 'tag:yaml.org,2002:str'
     }
 
diff --git a/src/functions/private/Read-YamlBlockKey.ps1 b/src/functions/private/Read-YamlBlockKey.ps1
@@ -62,6 +62,7 @@ function Read-YamlBlockKey {
         return $node
     }
 
+    Assert-YamlNoByteOrderMark -Text $rest -Mark $start
     $first = $rest[0]
     if ($first -in @(',', '[', ']', '{', '}', '#', '&', '*', '!', '|', '>', "'", '"', '%', '@', '`') -or
         ($first -in @('-', '?', ':') -and (
diff --git a/src/functions/private/Read-YamlBlockMapping.ps1 b/src/functions/private/Read-YamlBlockMapping.ps1
@@ -191,7 +191,8 @@ function Read-YamlBlockMapping {
             $key = Read-YamlBlockKey -Context $Context -Text $keyText -Line $lineNumber `
                 -Column $contentColumn -Depth ($Depth + 1)
         }
-        if ($colon -gt 0 -and (Get-YamlRuneCount -Text $keyText) -gt 1024) {
+        if ($colon -gt 0 -and
+            (Get-YamlRuneCount -Text $content.Substring(0, $colon)) -gt 1024) {
             $mark = New-YamlMark -Index ($Context.LineStarts[$lineNumber] + $contentColumn + $colon) `
                 -Line $lineNumber -Column ($contentColumn + $colon)
             throw (New-YamlException -Start $mark -End $mark -ErrorId 'YamlInvalidImplicitKey' -Message (
diff --git a/src/functions/private/Read-YamlBlockScalar.ps1 b/src/functions/private/Read-YamlBlockScalar.ps1
@@ -97,6 +97,7 @@ function Read-YamlBlockScalar {
     $decodedLength = 0
     while ($Context.LineIndex -lt $Context.Lines.Count) {
         $line = $Context.Lines[$Context.LineIndex]
+        Assert-YamlNoByteOrderMark -Text $line -Mark $start
         if ($line -match '^(?:---|\.\.\.)(?:[ \t]|$)') {
             break
         }
diff --git a/src/functions/private/Read-YamlFlowNode.ps1 b/src/functions/private/Read-YamlFlowNode.ps1
@@ -90,6 +90,7 @@ function Read-YamlFlowNode {
                 Move-YamlCursor -Cursor $Cursor -Context $Context
             }
             $anchor = $Context.Text.Substring($anchorStart, $Cursor.Index - $anchorStart)
+            Assert-YamlNoByteOrderMark -Text $anchor -Mark $start
             if ([string]::IsNullOrEmpty($anchor)) {
                 throw (New-YamlException -Start $start -End $start -ErrorId 'YamlInvalidAnchor' -Message (
                         'A YAML anchor name is missing.'
@@ -137,6 +138,7 @@ function Read-YamlFlowNode {
             Move-YamlCursor -Cursor $Cursor -Context $Context
         }
         $alias = $Context.Text.Substring($aliasStart, $Cursor.Index - $aliasStart)
+        Assert-YamlNoByteOrderMark -Text $alias -Mark $start
         $Context.AliasCount++
         if ($Context.AliasCount -gt $Context.MaxAliases) {
             throw (New-YamlException -Start $start -End $start -ErrorId 'YamlAliasLimitExceeded' -Message (
@@ -544,6 +546,12 @@ function Read-YamlFlowNode {
     }
     while ($Cursor.Index -lt $Context.Text.Length) {
         $character = $Context.Text[$Cursor.Index]
+        if ($character -ceq [char] 0xFEFF) {
+            $mark = New-YamlMark -Index $Cursor.Index -Line $Cursor.Line -Column $Cursor.Column
+            throw (New-YamlException -Start $mark -End $mark -ErrorId 'YamlInvalidByteOrderMark' -Message (
+                    'A raw YAML byte order mark is only allowed in a quoted scalar or document prefix.'
+                ))
+        }
         if ($character -in @(',', '[', ']', '{', '}')) {
             break
         }
diff --git a/src/functions/private/Read-YamlNodeProperty.ps1 b/src/functions/private/Read-YamlNodeProperty.ps1
@@ -84,6 +84,10 @@ function Read-YamlNodeProperty {
                 $position++
             }
             $anchor = $Text.Substring($start, $position - $start)
+            Assert-YamlNoByteOrderMark -Text $anchor -Mark (
+                New-YamlMark -Index ($Context.LineStarts[$Line] + $Column + $start) -Line $Line `
+                    -Column ($Column + $start)
+            )
             if ([string]::IsNullOrEmpty($anchor) -or $anchor.IndexOfAny(@('[', ']', '{', '}', ',')) -ge 0) {
                 $mark = New-YamlMark -Index ($Context.LineStarts[$Line] + $Column + $start - 1) -Line $Line `
                     -Column ($Column + $start - 1)
diff --git a/src/functions/private/Read-YamlPlainScalar.ps1 b/src/functions/private/Read-YamlPlainScalar.ps1
@@ -37,6 +37,7 @@ function Read-YamlPlainScalar {
     $parts = [System.Collections.Generic.List[object]]::new()
     $firstComment = Find-YamlCommentStart -Text $FirstText
     $firstValue = (Get-YamlContentWithoutComment -Text $FirstText).Trim(' ', "`t")
+    Assert-YamlNoByteOrderMark -Text $firstValue -Mark $start
     $firstCharacter = if ($firstValue.Length -gt 0) { $firstValue[0] } else { [char] 0 }
     $forbiddenFirst = $firstCharacter -in @(
         ',', '[', ']', '{', '}', '#', '&', '*', '!', '|', '>', "'", '"', '%', '@', '`'
@@ -91,6 +92,7 @@ function Read-YamlPlainScalar {
             break
         }
         $trimmedContent = $content.Trim(' ', "`t")
+        Assert-YamlNoByteOrderMark -Text $trimmedContent -Mark $start
         $separatorLength = if ($pendingBreaks -gt 0) { $pendingBreaks } else { 1 }
         if ($decodedLength + $separatorLength + $trimmedContent.Length -gt
             $Context.MaxScalarLength) {
diff --git a/src/functions/private/Resolve-YamlTag.ps1 b/src/functions/private/Resolve-YamlTag.ps1
@@ -22,6 +22,7 @@ function Resolve-YamlTag {
             ))
     }
 
+    $isNonSpecific = $Token.Equals('!', [System.StringComparison]::Ordinal)
     if ($Token.StartsWith('!<', [System.StringComparison]::Ordinal)) {
         if (-not $Token.EndsWith('>', [System.StringComparison]::Ordinal) -or $Token.Length -lt 4) {
             throw (New-YamlException -Start $Mark -End $Mark -ErrorId 'YamlInvalidTag' -Message (
@@ -67,7 +68,11 @@ function Resolve-YamlTag {
         }
     }
 
-    $expanded = ConvertFrom-YamlTagUriEscape -Text ($prefix + $suffix) -Mark $Mark -Token $Token
+    $expanded = if ($isNonSpecific) {
+        ''
+    } else {
+        ConvertFrom-YamlTagUriEscape -Text ($prefix + $suffix) -Mark $Mark -Token $Token
+    }
     $expandedLength = $expanded.Length
     if ($expandedLength -gt $Context.MaxTagLength) {
         throw (New-YamlException -Start $Mark -End $Mark -ErrorId 'YamlTagLimitExceeded' -Message (
@@ -98,6 +103,6 @@ function Resolve-YamlTag {
 
     [pscustomobject]@{
         Tag       = $expanded
-        IsUnknown = -not $known
+        IsUnknown = $isNonSpecific -or -not $known
     }
 }
diff --git a/tests/ConvertFrom-Yaml.Tests.ps1 b/tests/ConvertFrom-Yaml.Tests.ps1
@@ -213,6 +213,8 @@ folded: >
             ("foo${bom}bar" | Test-Yaml) | Should -BeFalse
             ("---`n${bom}value" | Test-Yaml) | Should -BeFalse
             ("---`none`n...`n${bom}# comment`ntwo" | Test-Yaml) | Should -BeFalse
+            ('"foo' + $bom + 'bar"' | ConvertFrom-Yaml) | Should -Be "foo${bom}bar"
+            ("'foo${bom}bar'" | ConvertFrom-Yaml) | Should -Be "foo${bom}bar"
         }
     }
 
diff --git a/tests/ConvertTo-Yaml.Tests.ps1 b/tests/ConvertTo-Yaml.Tests.ps1
@@ -171,6 +171,15 @@ Describe 'ConvertTo-Yaml' {
             $result['multiline'] | Should -Be $inputObject.multiline
         }
 
+        It 'escapes byte order marks as quoted scalar content' {
+            $value = "foo$([char] 0xFEFF)bar"
+
+            $yaml = ConvertTo-Yaml -InputObject $value
+
+            $yaml.TrimEnd("`n") | Should -Be '"foo\uFEFFbar"'
+            ($yaml | ConvertFrom-Yaml) | Should -Be $value
+        }
+
         It 'emits only valid YAML characters and rejects malformed UTF-16 input' {
             $noncharacters = ([string] [char] 0xFFFE) + [char] 0xFFFF
             $yaml = ConvertTo-Yaml -InputObject $noncharacters
diff --git a/tests/Test-Yaml.Tests.ps1 b/tests/Test-Yaml.Tests.ps1
@@ -67,11 +67,16 @@ Describe 'Test-Yaml' {
                 Should -Be ($length -le 1024)
         }
 
+        ((('k' * 1023) + ' : value') | Test-Yaml) | Should -BeTrue
+        ((('k' * 1024) + ' : value') | Test-Yaml) | Should -BeFalse
+
     }
 
     It 'resolves non-specific tags before comparing representation keys' {
         ("! x: one`n!!str x: two" | Test-Yaml) | Should -BeFalse
         ("? ! [x]`n: one`n? !!seq [x]`n: two" | Test-Yaml) | Should -BeFalse
+        ("! x: one`n!<!> x: two" | Test-Yaml) | Should -BeTrue
+        ("!<!> x: one`n!<!> x: two" | Test-Yaml) | Should -BeFalse
     }
 
     It 'accepts multiline keys in flow mappings' {

Original file line number	Diff line number	Diff line change
`@@ -32,14 +32,7 @@ function ConvertFrom-YamlByteOrderMark {`
`32`	`32`	`continue`
`33`	`33`	`}`
`34`	`34`
`35`		`- $before = $Text.Substring(0, $index)`
`36`		- $line = ([regex]::Matches($before, "`n")).Count
`37`		- $lastBreak = $before.LastIndexOf("`n", [System.StringComparison]::Ordinal)
`38`		`- $column = if ($lastBreak -lt 0) { $before.Length } else { $before.Length - $lastBreak - 1 }`
`39`		`- $mark = New-YamlMark -Index $index -Line $line -Column $column`
`40`		`- throw (New-YamlException -Start $mark -End $mark -ErrorId 'YamlInvalidByteOrderMark' -Message (`
`41`		`- 'A YAML byte order mark is only allowed at the start of the stream or a document prefix.'`
`42`		`- ))`
	`35`	`+ [void] $builder.Append($Text[$index])`
`43`	`36`	`}`
`44`	`37`
`45`	`38`	`return $builder.ToString()`
Original file line number	Diff line number	Diff line change
`@@ -14,8 +14,7 @@ function Get-YamlEffectiveTag {`
`14`	`14`	`[object] $Value`
`15`	`15`	`)`
`16`	`16`
`17`		`- if (-not [string]::IsNullOrEmpty($Node.Tag) -and`
`18`		`- -not $Node.Tag.Equals('!', [System.StringComparison]::Ordinal)) {`
	`17`	`+ if (-not [string]::IsNullOrEmpty($Node.Tag)) {`
`19`	`18`	`return $Node.Tag`
`20`	`19`	`}`
`21`	`20`	`if ($Node.Kind.Equals('Sequence', [System.StringComparison]::Ordinal)) {`
`@@ -24,7 +23,7 @@ function Get-YamlEffectiveTag {`
`24`	`23`	`if ($Node.Kind.Equals('Mapping', [System.StringComparison]::Ordinal)) {`
`25`	`24`	`return 'tag:yaml.org,2002:map'`
`26`	`25`	`}`
`27`		`- if ($Node.Tag.Equals('!', [System.StringComparison]::Ordinal)) {`
	`26`	`+ if ($Node.HasUnknownTag) {`
`28`	`27`	`return 'tag:yaml.org,2002:str'`
`29`	`28`	`}`
`30`	`29`
Original file line number	Diff line number	Diff line change
`@@ -62,6 +62,7 @@ function Read-YamlBlockKey {`
`62`	`62`	`return $node`
`63`	`63`	`}`
`64`	`64`
	`65`	`+ Assert-YamlNoByteOrderMark -Text $rest -Mark $start`
`65`	`66`	`$first = $rest[0]`
`66`	`67`	if ($first -in @(',', '[', ']', '{', '}', '#', '&', '*', '!', '\|', '>', "'", '"', '%', '@', '`') -or
`67`	`68`	`($first -in @('-', '?', ':') -and (`
Original file line number	Diff line number	Diff line change
`@@ -97,6 +97,7 @@ function Read-YamlBlockScalar {`
`97`	`97`	`$decodedLength = 0`
`98`	`98`	`while ($Context.LineIndex -lt $Context.Lines.Count) {`
`99`	`99`	`$line = $Context.Lines[$Context.LineIndex]`
	`100`	`+ Assert-YamlNoByteOrderMark -Text $line -Mark $start`
`100`	`101`	`if ($line -match '^(?:---\|\.\.\.)(?:[ \t]\|$)') {`
`101`	`102`	`break`
`102`	`103`	`}`