Constrain conformance policy to four cases

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

Author: MariusStorhaug

README.md

@@ -167,19 +167,25 @@ The offline test gate runs the complete released `yaml-test-suite` data corpus
 at commit `6ad3d2c62885d82fc349026c136ef560838fdf3d` (generated from source
 commit `45db50ae`). The pinned archive contains 402 inputs:
 
-- all 94 fixtures marked invalid are rejected;
-- 306 of 308 fixtures marked valid are accepted;
-- the other two valid-syntax fixtures, `2JQS` and `X38W`, are deliberately
-  rejected because this module rejects duplicate mapping keys;
-- 282 fixtures include `in.json`; three belong to invalid inputs, 277 of the
-  279 applicable constructions match exactly, and two use a different
-  documented projection policy.
-
-The two construction-policy differences are `565N`, where this module
-constructs `!!binary` as `byte[]` instead of a Base64 string, and `J7PZ`, where
-the explicitly supported `!!omap` tag becomes an ordered dictionary instead of
-remaining a sequence of one-entry mappings. The deterministic runner reports
-398 passing cases, four policy exclusions, and no unexplained failures.
+| Surface | Pass | PolicyDifference | Fail | NotApplicable |
+| --- | ---: | ---: | ---: | ---: |
+| Syntax and composition | 400 | 2 | 0 | 0 |
+| Representation events | 308 | 0 | 0 | 94 |
+| JSON projection | 277 | 2 | 0 | 123 |
+| `out.yaml` projection | 241 | 0 | 0 | 161 |
+| Emit and round trip | 306 | 0 | 0 | 96 |
+
+All 94 fixtures marked invalid are rejected. The valid `2JQS` and `X38W`
+inputs are syntactically recognized and produce matching representation
+events, then are rejected during load validation because representation
+mapping keys must be unique. They are not unsupported grammar.
+
+The two JSON projection differences are `565N`, where `!!binary` intentionally
+becomes `byte[]` instead of a Base64 string, and `J7PZ`, where legacy `!!omap`
+intentionally becomes `System.Collections.Specialized.OrderedDictionary`
+instead of remaining a sequence of one-entry mappings. No event mismatch is
+classified as policy. The deterministic runner reports no unexplained
+failures.
 
 These results are a pinned compatibility measurement, not a claim that a finite
 corpus proves complete YAML 1.2.2 compliance.

tests/Conformance.Tests.ps1

@@ -32,19 +32,22 @@ Describe 'Released yaml-test-suite corpus accounting' {
         $suiteResults.Count | Should -Be 402
     }
 
-    It 'accounts for syntax acceptance, rejection, and policy differences' {
+    It 'accounts for syntax recognition and representation-key load policy' {
         @($suiteResults | Where-Object SyntaxResult -EQ 'Pass').Count | Should -Be 400
         @($suiteResults | Where-Object SyntaxResult -EQ 'PolicyDifference').Count |
             Should -Be 2
         @($suiteResults | Where-Object SyntaxResult -EQ 'Fail').Count | Should -Be 0
         @($suiteResults | Where-Object SyntaxResult -EQ 'NotApplicable').Count |
             Should -Be 0
-        @(
+        $policyResults = @(
             $suiteResults |
                 Where-Object SyntaxResult -EQ 'PolicyDifference' |
-                Select-Object -ExpandProperty Case |
-                Sort-Object
-        ) | Should -Be @('2JQS', 'X38W')
+                Sort-Object Case
+        )
+        @($policyResults.Case) | Should -Be @('2JQS', 'X38W')
+        @($policyResults.SyntaxReason | Select-Object -Unique) |
+            Should -Be @('RepresentationMappingKeyUniqueness')
+        @($policyResults | Where-Object EventResult -NE 'Pass').Count | Should -Be 0
     }
 
     It 'accounts for parser representation/event comparisons' {
@@ -63,6 +66,16 @@ Describe 'Released yaml-test-suite corpus accounting' {
         @($suiteResults | Where-Object JsonResult -EQ 'Fail').Count | Should -Be 0
         @($suiteResults | Where-Object JsonResult -EQ 'NotApplicable').Count |
             Should -Be 123
+        $policyResults = @(
+            $suiteResults |
+                Where-Object JsonResult -EQ 'PolicyDifference' |
+                Sort-Object Case
+        )
+        @($policyResults.Case) | Should -Be @('565N', 'J7PZ')
+        @($policyResults.JsonReason) | Should -Be @(
+            'BinaryByteArrayProjection',
+            'LegacyOrderedMapProjection'
+        )
     }
 
     It 'accounts for out.yaml representation comparisons' {

tests/Specification.Tests.ps1

@@ -202,5 +202,20 @@ Describe 'Pinned yaml-test-suite reference cases' {
             $result.canonical,
             $result.generic
         ) | Should -BeTrue
+        , $result.canonical | Should -BeOfType [byte[]]
+        , $result.generic | Should -BeOfType [byte[]]
+    }
+
+    It 'projects the legacy ordered map in case J7PZ as an ordered dictionary' {
+        $result = @'
+--- !!omap
+- Mark McGwire: 65
+- Sammy Sosa: 63
+- Ken Griffy: 58
+'@ | ConvertFrom-Yaml
+
+        $result | Should -BeOfType [System.Collections.Specialized.OrderedDictionary]
+        @($result.Keys) | Should -Be @('Mark McGwire', 'Sammy Sosa', 'Ken Griffy')
+        @($result.Values) | Should -Be @(65, 63, 58)
     }
 }

tests/tools/Invoke-YamlTestSuite.ps1

@@ -243,26 +243,19 @@ function ConvertTo-YamlSuiteReferenceSignature {
     return ($output -join ';')
 }
 
-function Get-YamlSuitePolicyReason {
+function Get-YamlSuiteJsonPolicyReason {
     [CmdletBinding()]
     [OutputType([string])]
     param (
-        [string] $YamlText,
-        [string] $Expected,
-        [string] $Actual,
-        [string] $DefaultReason
+        [Parameter(Mandatory)]
+        [string] $Case
     )
 
-    if ($DefaultReason) {
-        return $DefaultReason
-    }
-    if ($YamlText -cmatch '!!(?:binary|omap|pairs|set|timestamp)(?:[ \t\r\n,\[\]\{\}]|$)') {
-        return 'StandardTagProjectionPolicy'
-    }
-    if (($Expected -match 'unsupported:' -or $Actual -match 'unsupported:')) {
-        return 'PowerShellTypePolicy'
+    switch -CaseSensitive ($Case) {
+        '565N' { return 'BinaryByteArrayProjection' }
+        'J7PZ' { return 'LegacyOrderedMapProjection' }
+        default { return '' }
     }
-    return ''
 }
 
 function ConvertFrom-YamlSuiteEventText {
@@ -694,9 +687,10 @@ foreach ($inputFile in $inputFiles) {
             }
             if ($expectsError) {
                 $syntaxResult = 'Pass'
-            } elseif ($_.Exception.Data['YamlErrorId'] -eq 'YamlDuplicateKey') {
+            } elseif ($casePath -cin @('2JQS', 'X38W') -and
+                $_.Exception.Data['YamlErrorId'] -eq 'YamlDuplicateKey') {
                 $syntaxResult = 'PolicyDifference'
-                $syntaxReason = 'DuplicateKeyRejected'
+                $syntaxReason = 'RepresentationMappingKeyUniqueness'
             } else {
                 $syntaxResult = 'Fail'
                 $syntaxReason = [string] $_.Exception.Data['YamlErrorId']
@@ -732,15 +726,8 @@ foreach ($inputFile in $inputFiles) {
             if (Compare-YamlSuiteCanonicalList -Left $actualEvents -Right $expectedEvents) {
                 $eventResult = 'Pass'
             } else {
-                $reason = Get-YamlSuitePolicyReason -YamlText $yaml -Expected ($expectedEvents -join "`n") `
-                    -Actual ($actualEvents -join "`n") -DefaultReason ''
-                if ($reason) {
-                    $eventResult = 'PolicyDifference'
-                    $eventReason = $reason
-                } else {
-                    $eventResult = 'Fail'
-                    $eventReason = 'EventMismatch'
-                }
+                $eventResult = 'Fail'
+                $eventReason = 'EventMismatch'
             }
         }
     }
@@ -765,8 +752,7 @@ foreach ($inputFile in $inputFiles) {
             if ($projectedCanonical -ceq $expectedCanonical) {
                 $jsonResult = 'Pass'
             } else {
-                $reason = Get-YamlSuitePolicyReason -YamlText $yaml -Expected $expectedCanonical `
-                    -Actual $projectedCanonical -DefaultReason ''
+                $reason = Get-YamlSuiteJsonPolicyReason -Case $casePath
                 if ($reason) {
                     $jsonResult = 'PolicyDifference'
                     $jsonReason = $reason
@@ -794,15 +780,8 @@ foreach ($inputFile in $inputFiles) {
                 if ($outCanonical -ceq $projectedCanonical) {
                     $outYamlResult = 'Pass'
                 } else {
-                    $reason = Get-YamlSuitePolicyReason -YamlText $yaml -Expected $projectedCanonical `
-                        -Actual $outCanonical -DefaultReason ''
-                    if ($reason) {
-                        $outYamlResult = 'PolicyDifference'
-                        $outYamlReason = $reason
-                    } else {
-                        $outYamlResult = 'Fail'
-                        $outYamlReason = 'OutYamlConstructionMismatch'
-                    }
+                    $outYamlResult = 'Fail'
+                    $outYamlReason = 'OutYamlConstructionMismatch'
                 }
             } catch {
                 if ($_.Exception.Data.Contains('IsYamlException')) {
@@ -820,7 +799,7 @@ foreach ($inputFile in $inputFiles) {
             $emitResult = 'NotApplicable'
             if ($expectsError) { $emitReason = 'InvalidSyntax' }
         } elseif ($projectionError) {
-            $emitResult = 'PolicyDifference'
+            $emitResult = 'Fail'
             $emitReason = $projectionError
         } elseif ($syntaxResult -eq 'PolicyDifference') {
             $emitResult = 'PolicyDifference'
@@ -855,19 +834,12 @@ foreach ($inputFile in $inputFiles) {
                     if ($roundCanonical -ceq $projectedCanonical -and $roundReference -ceq $projectedReference) {
                         $emitResult = 'Pass'
                     } else {
-                        $reason = Get-YamlSuitePolicyReason -YamlText $yaml -Expected $projectedCanonical `
-                            -Actual $roundCanonical -DefaultReason ''
-                        if ($reason) {
-                            $emitResult = 'PolicyDifference'
-                            $emitReason = $reason
-                        } else {
-                            $emitResult = 'Fail'
-                            $emitReason = 'EmitRoundTripMismatch'
-                        }
+                        $emitResult = 'Fail'
+                        $emitReason = 'EmitRoundTripMismatch'
                     }
                 }
             } catch [System.NotSupportedException] {
-                $emitResult = 'PolicyDifference'
+                $emitResult = 'Fail'
                 $emitReason = 'UnsupportedEmissionType'
             } catch {
                 if ($_.Exception.Data.Contains('IsYamlException')) {