Allow conversion splitters for text extensions (#1778)

* Allow conversion splitters for text extensions

Allow conversion splitters for text extensions
Previously RegisterStreamConverter threw for text extensions, and the
splitter pipeline only handled stream targets, so a converter could not
run against a text extension (eg html).
- Drop the text-extension guard in RegisterStreamConverter
- DoExtensionConversion now takes the Target itself and resolves the
  converter input from either a stream or a text (utf8) target, instead
  of reconstructing an illegal stream Target for text extensions
- Scrub text targets before they are fed to a converter, so derived
  targets (eg rendered images) reflect the scrubbed content
- Move converted-target scrubbing into DoExtensionConversion and pass
  already-converted targets through GetTargets untouched, so each text
  target is scrubbed exactly once (no double scrub of the re-emitted
  source)
Add ExtensionConverterTests.TextSplitter covering a converter registered
against a text extension.

* .

---------

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>

Author: SimonCropp

docs/converter.md

@@ -146,6 +146,45 @@ await VerifyFile("sample.tif");
 <!-- endSnippet -->
 
 
+### Text extensions
+
+A stream converter can also be registered against a text extension. This is useful when a text document needs derived targets, for example rendering html to an image for visual verification.
+
+The text target is scrubbed before being passed to the converter, so any derived targets (for example a rendered image) reflect the scrubbed content.
+
+For a custom text extension, register it as text via `FileExtensions.AddTextExtension`. Built-in text extensions (for example `html` or `csv`) do not require this.
+
+<!-- snippet: RegisterStreamConverterTextExtension -->
+<a id='snippet-RegisterStreamConverterTextExtension'></a>
+```cs
+// "texttoconvert" is a custom text extension, so register it as text first.
+// For built-in text extensions (eg html or csv) this step is not required.
+FileExtensions.AddTextExtension("texttoconvert");
+
+// The input text is scrubbed before being passed to the converter, so any
+// derived targets (eg a rendered image) reflect the scrubbed content.
+VerifierSettings.RegisterStreamConverter(
+    "texttoconvert",
+    async (_, stream, _) =>
+        new(
+            null,
+            [
+                new("texttoconvert", await stream.ReadStringBuilderWithFixedLines()),
+                new("txt", "derived from text")
+            ]));
+```
+<sup><a href='/src/Verify.Tests/Converters/ExtensionConverterTests.cs#L6-L24' title='Snippet source file'>snippet source</a> | <a href='#snippet-RegisterStreamConverterTextExtension' title='Start of snippet'>anchor</a></sup>
+<!-- endSnippet -->
+
+<!-- snippet: TextExtensionConverterVerify -->
+<a id='snippet-TextExtensionConverterVerify'></a>
+```cs
+Verify("the source text", "texttoconvert");
+```
+<sup><a href='/src/Verify.Tests/Converters/ExtensionConverterTests.cs#L30-L32' title='Snippet source file'>snippet source</a> | <a href='#snippet-TextExtensionConverterVerify' title='Start of snippet'>anchor</a></sup>
+<!-- endSnippet -->
+
+
 ### Cleanup
 
 If cleanup needs to occur after verification a callback can be passes to `ConversionResult`:

docs/mdsource/converter.source.md

@@ -50,6 +50,19 @@ snippet: RegisterStreamConverterExtension
 snippet: FileConverterExtensionVerify
 
 
+### Text extensions
+
+A stream converter can also be registered against a text extension. This is useful when a text document needs derived targets, for example rendering html to an image for visual verification.
+
+The text target is scrubbed before being passed to the converter, so any derived targets (for example a rendered image) reflect the scrubbed content.
+
+For a custom text extension, register it as text via `FileExtensions.AddTextExtension`. Built-in text extensions (for example `html` or `csv`) do not require this.
+
+snippet: RegisterStreamConverterTextExtension
+
+snippet: TextExtensionConverterVerify
+
+
 ### Cleanup
 
 If cleanup needs to occur after verification a callback can be passes to `ConversionResult`:

src/Verify.Tests/Converters/ExtensionConverterTests.TextSplitter.verified.texttoconvert

@@ -0,0 +1 @@
+the source text

src/Verify.Tests/Converters/ExtensionConverterTests.TextSplitter.verified.txt

@@ -0,0 +1 @@
+derived from text

src/Verify.Tests/Converters/ExtensionConverterTests.cs

@@ -1,5 +1,36 @@
 public class ExtensionConverterTests
 {
+    [ModuleInitializer]
+    public static void TextSplitterInit()
+    {
+        #region RegisterStreamConverterTextExtension
+
+        // "texttoconvert" is a custom text extension, so register it as text first.
+        // For built-in text extensions (eg html or csv) this step is not required.
+        FileExtensions.AddTextExtension("texttoconvert");
+
+        // The input text is scrubbed before being passed to the converter, so any
+        // derived targets (eg a rendered image) reflect the scrubbed content.
+        VerifierSettings.RegisterStreamConverter(
+            "texttoconvert",
+            async (_, stream, _) =>
+                new(
+                    null,
+                    [
+                        new("texttoconvert", await stream.ReadStringBuilderWithFixedLines()),
+                        new("txt", "derived from text")
+                    ]));
+
+        #endregion
+    }
+
+    // a conversion splitter registered against a text extension
+    [Fact]
+    public Task TextSplitter() =>
+        #region TextExtensionConverterVerify
+        Verify("the source text", "texttoconvert");
+        #endregion
+
     [ModuleInitializer]
     public static void RecursiveInit() =>
         VerifierSettings.RegisterStreamConverter(

src/Verify.Tests/InnerVerifyChecksTests/VerifyChecksTests.GetExtensions.verified.txt

@@ -6,6 +6,7 @@
   nuspec,
   props,
   staticComparerExt,
+  texttoconvert,
   txt,
   xml
 ]

src/Verify/Compare/Png/PngSsimComparer.cs

@@ -13,7 +13,8 @@ internal static Task<CompareResult> Compare(Stream received, Stream verified, do
         var receivedImage = PngDecoder.Decode(received);
         var verifiedImage = PngDecoder.Decode(verified);
 
-        if (receivedImage.Width != verifiedImage.Width || receivedImage.Height != verifiedImage.Height)
+        if (receivedImage.Width != verifiedImage.Width ||
+            receivedImage.Height != verifiedImage.Height)
         {
             return Task.FromResult(CompareResult.NotEqual(
                 $"PNG dimensions differ. Received: {receivedImage.Width}x{receivedImage.Height}, Verified: {verifiedImage.Width}x{verifiedImage.Height}"));

src/Verify/Splitters/Settings_Extension.cs

@@ -23,11 +23,6 @@ public static void RegisterStreamConverter(
     {
         InnerVerifier.ThrowIfVerifyHasBeenRun();
         Guards.AgainstBadExtension(extension);
-        if (FileExtensions.IsTextExtension(extension))
-        {
-            throw new("RegisterStreamConverter is only supported for non-text extensions");
-        }
-
         extensionConverters[extension] = conversion;
     }
 }

src/Verify/Verifier/InnerVerifier_Inner.cs

@@ -52,46 +52,54 @@ async Task<VerifyResult> VerifyInner(object? root, Func<Task>? cleanup, IEnumera
     {
         List<Target> list = [..targets, ..VerifierSettings.GetFileAppenders(settings)];
         var cleanup = () => Task.CompletedTask;
-        if (doExtensionConversion)
-        {
-            var result = new List<Target>();
-            foreach (var target in list)
-            {
-                if (!target.PerformConversion ||
-                    !VerifierSettings.HasStreamConverter(target.Extension))
-                {
-                    result.Add(target);
-                    continue;
-                }
-
-                var (info, converted, itemCleanup) = await DoExtensionConversion(target.Extension, target.StreamData, null, target.Name);
-                cleanup = cleanup.Then(itemCleanup);
-                if (info != null)
-                {
-                    result.Add(
-                        new(
-                            settings.TxtOrJson,
-                            JsonFormatter.AsJson(
-                                settings,
-                                counter,
-                                info)));
-                }
-
-                result.AddRange(converted);
-            }
 
-            list = result;
+        // When doExtensionConversion is false the targets have already been run through
+        // conversion and scrubbing (the only caller is the post-conversion stream path),
+        // so pass them through untouched to avoid double scrubbing.
+        if (!doExtensionConversion)
+        {
+            return (list, cleanup);
         }
 
+        var result = new List<Target>();
         foreach (var target in list)
         {
-            if (target.TryGetStringBuilder(out var builder))
+            if (!target.PerformConversion ||
+                !VerifierSettings.HasStreamConverter(target.Extension))
             {
-                ApplyScrubbers.ApplyForExtension(target.Extension, builder, settings, counter);
+                Scrub(target);
+                result.Add(target);
+                continue;
             }
+
+            var (info, converted, itemCleanup) = await DoExtensionConversion(target, null);
+            cleanup = cleanup.Then(itemCleanup);
+            if (info != null)
+            {
+                Target infoTarget = new(
+                    settings.TxtOrJson,
+                    JsonFormatter.AsJson(
+                        settings,
+                        counter,
+                        info));
+                Scrub(infoTarget);
+                result.Add(infoTarget);
+            }
+
+            // converted targets are scrubbed within DoExtensionConversion
+            result.AddRange(converted);
         }
 
-        return (list, cleanup);
+        return (result, cleanup);
+    }
+
+    // Scrubs a text target in place. Stream (binary) targets are left untouched.
+    void Scrub(in Target target)
+    {
+        if (target.TryGetStringBuilder(out var builder))
+        {
+            ApplyScrubbers.ApplyForExtension(target.Extension, builder, settings, counter);
+        }
     }
 
     bool TryGetRootTarget(object? root,bool ignoreNullRoot, [NotNullWhen(true)] out Target? target)

src/Verify/Verifier/InnerVerifier_Stream.cs

@@ -90,7 +90,8 @@ public async Task<VerifyResult> VerifyStream(Stream? stream, string extension, o
         {
             if (VerifierSettings.HasStreamConverter(extension))
             {
-                var (newInfo, converted, cleanup) = await DoExtensionConversion(extension, stream, info, null);
+                var initial = await GetTarget(stream, extension);
+                var (newInfo, converted, cleanup) = await DoExtensionConversion(initial, info);
 
                 return await VerifyInner(newInfo, cleanup, converted, false, true);
             }
@@ -122,10 +123,15 @@ static async Task<Target> GetTarget(Stream stream, string extension)
         return new(extension, stream);
     }
 
-    //TODO: possibly pass in the target here
-    async Task<(object? info, List<Target> targets, Func<Task> cleanup)> DoExtensionConversion(string extension, Stream stream, object? info, string? name)
+    async Task<(object? info, List<Target> targets, Func<Task> cleanup)> DoExtensionConversion(Target initial, object? info)
     {
-        var cleanup = stream.DisposeAsyncEx;
+        var cleanup = () => Task.CompletedTask;
+        // the source stream of a stream target is owned here, so dispose it once consumed
+        if (initial.IsStream)
+        {
+            cleanup = cleanup.Then(initial.StreamData.DisposeAsyncEx);
+        }
+
         var infos = new List<object>();
         if (info != null)
         {
@@ -135,19 +141,37 @@ static async Task<Target> GetTarget(Stream stream, string extension)
         var targets = new List<Target>();
 
         var queue = new Queue<Target>();
-        queue.Enqueue(new(extension, stream, name));
+        queue.Enqueue(initial);
 
         while (queue.Count > 0)
         {
             var target = queue.Dequeue();
 
             if (!VerifierSettings.TryGetStreamConverter(target.Extension, out var conversion))
             {
+                // terminal target: scrub text before it is finalized
+                Scrub(target);
                 targets.Add(target);
                 continue;
             }
 
-            var targetStream = target.StreamData;
+            // scrub text before conversion so derived targets (eg rendered images) reflect the scrubbed content
+            Scrub(target);
+
+            Stream targetStream;
+            if (target.IsStream)
+            {
+                targetStream = target.StreamData;
+            }
+            else
+            {
+                // a text target is fed to the converter as a utf8 stream
+                target.TryGetStringBuilder(out var builder);
+                var memory = new MemoryStream(Encoding.UTF8.GetBytes(builder!.ToString()));
+                cleanup = cleanup.Then(memory.DisposeAsyncEx);
+                targetStream = memory;
+            }
+
             var result = await conversion(target.Name, targetStream, settings.Context);
             if (result.Cleanup != null)
             {
@@ -163,7 +187,8 @@ static async Task<Target> GetTarget(Stream stream, string extension)
 
             foreach (var resultTarget in resultTargets)
             {
-                // if the same extension is returned. no need to re process
+                // if the same extension is returned. no need to re process.
+                // its content derives from the already scrubbed input, so it is not scrubbed again
                 if (resultTarget.Extension == target.Extension)
                 {
                     targets.Add(resultTarget);