Ignore style/script elements by default, allow opting for other elements too

script/style are typically not part of the content of a page, yet they can be quite large blocks of text that would consume memory unnecessarily.

We should skip both by default, and allow opting in for those as well as potentially other elements via settings.

This commit reworks the way we deal with settings so all overloads have a consistent and non-duplicated behavior.

Defaulting the HTML 4.01 transitional DTD also allows for much cleaner object model, so we opt in for that too.

Fixes #8

Author: kzu

readme.md

@@ -10,8 +10,8 @@ Implements CSS selectors for XLinq.
 # Usage
 
 ```csharp
-using Devlooped.Web;
 using System.Xml.Linq;
+using Devlooped.Web;
 
 XDocument page = HtmlDocument.Load("page.html")
 IEnumerable<XElement> elements = page.CssSelectElements("div.menuitem");

src/Html/Html.csproj

@@ -8,19 +8,17 @@
     <PackageProjectUrl>https://clarius.org/html</PackageProjectUrl>
     <PackageReadmeFile>readme.md</PackageReadmeFile>
     <Description>Read HTML as XML using XLinq.</Description>
+    <LangVersion>Preview</LangVersion>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1">
-      <PrivateAssets>all</PrivateAssets>
-      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-    </PackageReference>
+    <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1" PrivateAssets="all" />
     <PackageReference Include="Microsoft.Xml.SgmlReader" Version="1.8.25" />
     <PackageReference Include="NuGetizer" Version="0.8.0" PrivateAssets="all" />
   </ItemGroup>
 
   <ItemGroup>
     <None Include="..\..\readme.md" PackagePath="readme.md" />
   </ItemGroup>
-  
+
 </Project>

src/Html/HtmlDocument.cs

@@ -12,6 +12,23 @@ namespace Devlooped.Html;
 /// </summary>
 public static class HtmlDocument
 {
+    const string DefaultPublicIdentifier = "-//W3C//DTD XHTML 1.0 Transitional//EN";
+    const string DefaultSystemLiteral = "http://www.w3.org/TR/html4/loose.dtd";
+
+    /// <summary>
+    /// Create a new <see cref="XDocument"/> and initialize its underlying XML tree using
+    /// the passed <see cref="Stream"/> parameter.
+    /// </summary>
+    /// <param name="stream">
+    /// A <see cref="Stream"/> containing the raw HTML to read into the newly
+    /// created <see cref="XDocument"/>.
+    /// </param>
+    /// <returns>
+    /// A new <see cref="XDocument"/> containing the contents of the passed in
+    /// <see cref="Stream"/>.
+    /// </returns>
+    public static XDocument Load(Stream stream) => Load(stream, HtmlReaderSettings.Default);
+
     /// <overloads>
     /// The Load method provides multiple strategies for creating a new
     /// <see cref="XDocument"/> and initializing it from a data source containing
@@ -35,11 +52,22 @@ public static class HtmlDocument
     /// An <see cref="XDocument"/> initialized with the contents of the file referenced
     /// in the passed in uri parameter.
     /// </returns>
-    public static XDocument Load(string uri)
-    {
-        using var stream = (Stream)new XmlUrlResolver().GetEntity(new Uri(uri), string.Empty, typeof(Stream));
-        return Load(stream);
-    }
+    public static XDocument Load(string uri) => Load(uri, HtmlReaderSettings.Default);
+
+    /// <summary>
+    /// Create a new <see cref="XDocument"/> and initialize its underlying XML tree using
+    /// the passed <see cref="TextReader"/> parameter.  Optionally whitespace handling
+    /// can be preserved.
+    /// </summary>
+    /// <param name="textReader">
+    /// A <see cref="TextReader"/> containing the raw HTML to read into the newly
+    /// created <see cref="XDocument"/>.
+    /// </param>
+    /// <returns>
+    /// A new <see cref="XDocument"/> containing the contents of the passed in
+    /// <see cref="TextReader"/>.
+    /// </returns>
+    public static XDocument Load(TextReader textReader) => Load(textReader, HtmlReaderSettings.Default);
 
     /// <summary>
     /// Create a new <see cref="XDocument"/> and initialize its underlying XML tree using
@@ -49,12 +77,13 @@ public static XDocument Load(string uri)
     /// A <see cref="Stream"/> containing the raw HTML to read into the newly
     /// created <see cref="XDocument"/>.
     /// </param>
+    /// <param name="settings">The settings for the HTML load process.</param>
     /// <returns>
     /// A new <see cref="XDocument"/> containing the contents of the passed in
     /// <see cref="Stream"/>.
     /// </returns>
-    public static XDocument Load(Stream stream)
-        => Load(new StreamReader(stream));
+    public static XDocument Load(Stream stream, HtmlReaderSettings settings)
+        => Load(new StreamReader(stream), settings);
 
     /// <overloads>
     /// The Load method provides multiple strategies for creating a new
@@ -81,48 +110,13 @@ public static XDocument Load(Stream stream)
     /// in the passed in uri parameter.
     /// </returns>
     public static XDocument Load(string uri, HtmlReaderSettings settings)
-    {
-        using var stream = (Stream)new XmlUrlResolver().GetEntity(new Uri(uri), string.Empty, typeof(Stream));
-        return Load(stream, settings);
-    }
-
-    /// <summary>
-    /// Create a new <see cref="XDocument"/> and initialize its underlying XML tree using
-    /// the passed <see cref="Stream"/> parameter.
-    /// </summary>
-    /// <param name="stream">
-    /// A <see cref="Stream"/> containing the raw HTML to read into the newly
-    /// created <see cref="XDocument"/>.
-    /// </param>
-    /// <param name="settings">The settings for the HTML load process.</param>
-    /// <returns>
-    /// A new <see cref="XDocument"/> containing the contents of the passed in
-    /// <see cref="Stream"/>.
-    /// </returns>
-    public static XDocument Load(Stream stream, HtmlReaderSettings settings)
-        => Load(new StreamReader(stream), settings);
-
-    /// <summary>
-    /// Create a new <see cref="XDocument"/> and initialize its underlying XML tree using
-    /// the passed <see cref="TextReader"/> parameter.  Optionally whitespace handling
-    /// can be preserved.
-    /// </summary>
-    /// <param name="textReader">
-    /// A <see cref="TextReader"/> containing the raw HTML to read into the newly
-    /// created <see cref="XDocument"/>.
-    /// </param>
-    /// <returns>
-    /// A new <see cref="XDocument"/> containing the contents of the passed in
-    /// <see cref="TextReader"/>.
-    /// </returns>
-    public static XDocument Load(TextReader textReader)
     {
         using var reader = new SgmlReader
         {
-            InputStream = textReader,
+            Href = uri,
         };
 
-        return XDocument.Load(new IgnoreXmlNsReader(reader));
+        return XDocument.Load(Configure(reader, settings));
     }
 
     /// <summary>
@@ -144,12 +138,9 @@ public static XDocument Load(TextReader textReader, HtmlReaderSettings settings)
         using var reader = new SgmlReader
         {
             InputStream = textReader,
-            CaseFolding = settings.CaseFolding,
-            WhitespaceHandling = settings.WhitespaceHandling,
-            TextWhitespace = settings.TextWhitespace,
         };
 
-        return XDocument.Load(settings.IgnoreXmlNamespaces ? new IgnoreXmlNsReader(reader) : reader);
+        return XDocument.Load(Configure(reader, settings));
     }
 
     /// <overloads>
@@ -165,8 +156,7 @@ public static XDocument Load(TextReader textReader, HtmlReaderSettings settings)
     /// An <see cref="XDocument"/> containing an XML tree initialized from the
     /// passed in HTML string.
     /// </returns>
-    public static XDocument Parse(string html)
-        => Load(new StringReader(html));
+    public static XDocument Parse(string html) => Parse(html, HtmlReaderSettings.Default);
 
     /// <overloads>
     /// Create a new <see cref="XDocument"/> from a string containing
@@ -187,6 +177,27 @@ public static XDocument Parse(string html)
     public static XDocument Parse(string html, HtmlReaderSettings settings)
         => Load(new StringReader(html), settings);
 
+    static XmlReader Configure(SgmlReader reader, HtmlReaderSettings settings)
+    {
+        reader.DocType = "html";
+        reader.PublicIdentifier = DefaultPublicIdentifier;
+        reader.SystemLiteral = DefaultSystemLiteral;
+
+        reader.CaseFolding = settings.CaseFolding;
+        reader.WhitespaceHandling = settings.WhitespaceHandling;
+        reader.TextWhitespace = settings.TextWhitespace;
+
+        XmlReader result = reader;
+
+        if (settings.IgnoreXmlNamespaces)
+            result = new IgnoreXmlNsReader(result);
+
+        if (settings.SkipElements.Length > 0)
+            result = new SkipElementsReader(result, settings.SkipElements);
+
+        return result;
+    }
+
     /// <summary>
     /// Removes all XML namespaces, since for HTML content it's typically 
     /// irrelevant.
@@ -242,5 +253,28 @@ public override bool MoveToNextAttribute()
 
         bool IsLocalXmlNs => Prefix == "xmlns";
     }
+
+    /// <summary>
+    /// Removes all XML namespaces, since for HTML content it's typically 
+    /// irrelevant.
+    /// </summary>
+    class SkipElementsReader : XmlWrappingReader
+    {
+        readonly HashSet<string> skipElements;
+
+        public SkipElementsReader(XmlReader baseReader, string[] skipElements) : base(baseReader)
+        {
+            this.skipElements = new HashSet<string>(skipElements, StringComparer.OrdinalIgnoreCase);
+        }
+
+        public override bool Read()
+        {
+            var read = base.Read();
+            if (read && base.NodeType == XmlNodeType.Element && skipElements.Contains(LocalName))
+                base.Skip();
+
+            return read;
+        }
+    }
 }
 

src/Html/HtmlReaderSettings.cs

@@ -10,6 +10,18 @@ namespace Devlooped.Html;
 /// </summary>
 public sealed class HtmlReaderSettings
 {
+    /// <summary>
+    /// Default settings when reading HTML, which are:
+    /// <see cref="CaseFolding.ToLower" />, <see cref="IgnoreXmlNamespaces"/>=true  
+    /// and <see cref="SkipElements"/>=["script", "style"].
+    /// </summary>
+    public static HtmlReaderSettings Default { get; } = new HtmlReaderSettings
+    {
+        CaseFolding = CaseFolding.ToLower,
+        IgnoreXmlNamespaces = true,
+        SkipElements = new string[] { "script", "style" },
+    };
+
     /// <summary>
     /// HTML is case insensitive, so you can choose between converting
     /// to lower case or upper case tags. "None" means that the case is left
@@ -22,6 +34,12 @@ public sealed class HtmlReaderSettings
     /// </summary>
     public bool IgnoreXmlNamespaces { get; set; } = true;
 
+    /// <summary>
+    /// Elements that should be skipped when reading the HTML so they are 
+    /// not loaded into the resulting XML document. Defaults to no elements.
+    /// </summary>
+    public string[] SkipElements { get; set; } = Array.Empty<string>();
+
     /// <summary>
     /// Specifies how leading and trailing whitespace is handled.
     /// Note that this is a <see cref="FlagsAttribute"/>-enum.

src/Tests/HtmlTests.cs

@@ -1,3 +1,5 @@
+using System.Xml;
+using System.Xml.Linq;
 using System.Xml.XPath;
 using Devlooped.Html;
 
@@ -6,56 +8,95 @@ namespace Devlooped.Tests;
 public record HtmlTests(ITestOutputHelper Output)
 {
     [Fact]
-    public void IncludesScripts()
+    public void Render()
     {
-        var doc = HtmlDocument.Load(new Uri("file://" + new FileInfo("wikipedia.html").FullName).AbsoluteUri);
+        var doc = HtmlDocument.Load(File("sample.html"));
+
+        Output.WriteLine(doc.ToString());
+    }
+
+    [Fact]
+    public void ExcludesScriptsByDefault()
+    {
+        var doc = HtmlDocument.Load(File("wikipedia.html"));
+
+        Assert.Empty(doc.XPathSelectElements("//script"));
+    }
+
+    [Fact]
+    public void IncludeScriptsExplicitSettings()
+    {
+        var doc = HtmlDocument.Load(File("wikipedia.html"), new HtmlReaderSettings());
 
         Assert.NotEmpty(doc.XPathSelectElements("//script"));
     }
 
     [Fact]
-    public void IncludesStyles()
+    public void ExcludesStylesByDefault()
     {
-        var doc = HtmlDocument.Load(new Uri("file://" + new FileInfo("wikipedia.html").FullName).AbsoluteUri);
+        var doc = HtmlDocument.Load(File("wikipedia.html"));
+
+        Assert.Empty(doc.XPathSelectElements("//style"));
+    }
+
+    [Fact]
+    public void IncludeStylesExplicitSettings()
+    {
+        var doc = HtmlDocument.Load(File("wikipedia.html"), new HtmlReaderSettings());
 
         Assert.NotEmpty(doc.XPathSelectElements("//style"));
     }
 
     [Fact]
-    public void RemovesXmlNamespaces()
+    public void ExcludesXmlNamespacesByDefault()
     {
-        var doc = HtmlDocument.Load(new Uri("file://" + new FileInfo("sample.xhtml").FullName).AbsoluteUri);
+        var doc = HtmlDocument.Load(File("sample.xhtml"));
 
         Assert.NotEmpty(doc.XPathSelectElements("//h1"));
     }
 
     [Fact]
-    public void HtmlSettings()
+    public void IncludeXmlNamespacesExplicitly()
+    {
+        var doc = HtmlDocument.Load(File("sample.xhtml"), new HtmlReaderSettings { IgnoreXmlNamespaces = false });
+        var resolver = new XmlNamespaceManager(new NameTable());
+        resolver.AddNamespace("xh", "http://www.w3.org/1999/xhtml");
+
+        Assert.NotEmpty(doc.XPathSelectElements("//xh:h1", resolver));
+        // Won't match because the elements will have the XHTML namespace
+        Assert.Empty(doc.XPathSelectElements("//h1"));
+    }
+
+    [Fact]
+    public void CanChangeToUpperCaseHtml()
     {
-        var doc = HtmlDocument.Load(new Uri("file://" + new FileInfo("wikipedia.html").FullName).AbsoluteUri,
+        var doc = HtmlDocument.Load(File("wikipedia.html"),
             new HtmlReaderSettings
             {
                 CaseFolding = Sgml.CaseFolding.ToUpper,
-                TextWhitespace = Sgml.TextWhitespaceHandling.TrimBoth,
-                WhitespaceHandling = System.Xml.WhitespaceHandling.None
             });
 
         // The source has lowercase elements
         var central = doc.XPathSelectElement("/HTML/BODY/DIV/H1/SPAN");
 
         Assert.NotNull(central);
-
-        // The source contains leading and trailing whitespaces.
-        Assert.Equal("Wikipedia", central!.Value);
     }
 
     [Fact]
-    public void OptOutOfXmlNamespacesRemoval()
+    public void HtmlSettings()
     {
-        var doc = HtmlDocument.Load(new Uri("file://" + new FileInfo("sample.xhtml").FullName).AbsoluteUri,
-            new HtmlReaderSettings { IgnoreXmlNamespaces = false });
+        var doc = HtmlDocument.Load(File("wikipedia.html"),
+            new HtmlReaderSettings
+            {
+                TextWhitespace = Sgml.TextWhitespaceHandling.TrimBoth,
+                WhitespaceHandling = WhitespaceHandling.None
+            });
 
-        // Won't match because the elements will have the XHTML namespace
-        Assert.Empty(doc.XPathSelectElements("//h1"));
+        var central = doc.XPathSelectElement("/html/body/div/h1/span");
+
+        // The source contains leading and trailing whitespaces.
+        Assert.Equal("Wikipedia", central?.Value);
     }
+
+    string File(string path) => new Uri("file://" + new FileInfo(path).FullName).AbsoluteUri;
 }