Skip to content

Commit 7acd166

Browse files
kzukzu
authored
HTML > XML + CSS
2 parents 0005c65 + e817dc0 commit 7acd166

18 files changed

Lines changed: 3407 additions & 40 deletions

‎.netconfig‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,3 +162,8 @@
162162
sha = 9a1b07589b9bde93bc12528e9325712a32dec418
163163
etag = b54216ac431a83ce5477828d391f02046527e7f6fffd21da1d03324d352c3efb
164164
weak
165+
[file "src/Web/System/Xml/XmlWrappingReader.cs"]
166+
url = https://github.com/devlooped/catbag/blob/main/System/Xml/XmlWrappingReader.cs
167+
sha = b1f3e12a7107dc81de53fd0a962bd4a149ab1ef7
168+
etag = b2c97f61df993f05a7d6e3627ab10e7933528ad33d91be4ac16323756c522b6b
169+
weak

Css.sln renamed to Web.sln

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,12 @@ VisualStudioVersion = 17.0.31612.314
55
MinimumVisualStudioVersion = 10.0.40219.1
66
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tests", "src\Tests\Tests.csproj", "{2E8FE01D-35EB-49E2-B693-490F4ABFDD5D}"
77
EndProject
8-
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Css", "src\Css\Css.csproj", "{442D46AA-C4DC-4AEE-826B-CE98A9C6F837}"
8+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Web", "src\Web\Web.csproj", "{442D46AA-C4DC-4AEE-826B-CE98A9C6F837}"
99
EndProject
1010
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{F2D75BE9-587E-46CC-BFE4-D5BA538E325A}"
1111
ProjectSection(SolutionItems) = preProject
1212
.editorconfig = .editorconfig
13-
global.json = global.json
13+
.netconfig = .netconfig
1414
readme.md = readme.md
1515
EndProjectSection
1616
EndProject

‎assets/icon-32.png‎

1.03 KB
Loading

‎assets/icon.svg‎

Lines changed: 8 additions & 0 deletions
Loading

‎readme.md‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,39 @@
1-
![Icon](https://raw.githubusercontent.com/devlooped/css/main/assets/img/icon-32.png) XLinq to Css
1+
![Icon](https://raw.githubusercontent.com/devlooped/css/main/assets/img/icon-32.png) HTML => XML + CSS with XLinq 🤘
22
============
33

44
[![Version](https://img.shields.io/nuget/vpre/Devlooped.Xml.Css.svg?color=royalblue)](https://www.nuget.org/packages/Devlooped.Xml.Css)
55
[![Downloads](https://img.shields.io/nuget/dt/Devlooped.Xml.Css.svg?color=green)](https://www.nuget.org/packages/Devlooped.Xml.Css)
66
[![License](https://img.shields.io/github/license/devlooped/css.svg?color=blue)](https://github.com/devlooped/css/blob/main/license.txt)
77

8-
Implements CSS selectors for XLinq.
8+
Read HTML as XML and query it with CSS over XLinq.
9+
10+
No need to learn an entirely new object model for a page 🤘.
11+
This makes it the most productive and lean library for web
12+
scraping using the latest and greatest that .NET can offer.
913

1014
# Usage
1115

1216
```csharp
13-
using Devlooped.Xml.Css;
17+
using System.Xml.Linq;
18+
using Devlooped.Web;
1419

15-
var page = XDocument.Load("page.xhtml")
20+
XDocument page = HtmlDocument.Load("page.html")
1621
IEnumerable<XElement> elements = page.CssSelectElements("div.menuitem");
1722

18-
XElement title = page.CssSelectElement("div[role=alert]");
23+
XElement title = page.CssSelectElement("html head meta[name=title]");
1924
```
2025

26+
By default, `HtmlDocument.Load` will skip non-content elements `script` and
27+
`style`, turn all element names into lower case, and ignore all XML namespaces
28+
(useful when loading XHTML, for example) for easier querying. These options
29+
as well as granular whitespace handling can be configured using the overloads
30+
receiving an `HtmlReaderSettings`.
31+
32+
The underlying parsing is performed by the amazing [SgmlReader](https://www.nuget.org/packages/Microsoft.Xml.SgmlReader)
33+
library by Microsoft's [Chris Lovett](http://lovettsoftware.com/).
34+
35+
## CSS
36+
2137
At the moment, supports the following CSS selector features:
2238

2339
- [Type selector](https://www.w3.org/TR/selectors-3/#type-selectors)

‎src/Tests/HtmlTests.cs‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
using System.Xml;
2+
using System.Xml.Linq;
3+
using System.Xml.XPath;
4+
using Devlooped.Web;
5+
6+
namespace Devlooped.Tests;
7+
8+
public record HtmlTests(ITestOutputHelper Output)
9+
{
10+
[Fact]
11+
public void Render()
12+
{
13+
var doc = HtmlDocument.Load(File("sample.html"));
14+
15+
Output.WriteLine(doc.ToString());
16+
}
17+
18+
[Fact]
19+
public void ExcludesScriptsByDefault()
20+
{
21+
var doc = HtmlDocument.Load(File("wikipedia.html"));
22+
23+
Assert.Empty(doc.XPathSelectElements("//script"));
24+
}
25+
26+
[Fact]
27+
public void IncludeScriptsExplicitSettings()
28+
{
29+
var doc = HtmlDocument.Load(File("wikipedia.html"), new HtmlReaderSettings());
30+
31+
Assert.NotEmpty(doc.XPathSelectElements("//script"));
32+
}
33+
34+
[Fact]
35+
public void ExcludesStylesByDefault()
36+
{
37+
var doc = HtmlDocument.Load(File("wikipedia.html"));
38+
39+
Assert.Empty(doc.XPathSelectElements("//style"));
40+
}
41+
42+
[Fact]
43+
public void IncludeStylesExplicitSettings()
44+
{
45+
var doc = HtmlDocument.Load(File("wikipedia.html"), new HtmlReaderSettings());
46+
47+
Assert.NotEmpty(doc.XPathSelectElements("//style"));
48+
}
49+
50+
[Fact]
51+
public void ExcludesXmlNamespacesByDefault()
52+
{
53+
var doc = HtmlDocument.Load(File("sample.xhtml"));
54+
55+
Assert.NotEmpty(doc.XPathSelectElements("//h1"));
56+
}
57+
58+
[Fact]
59+
public void IncludeXmlNamespacesExplicitly()
60+
{
61+
var doc = HtmlDocument.Load(File("sample.xhtml"), new HtmlReaderSettings { IgnoreXmlNamespaces = false });
62+
var resolver = new XmlNamespaceManager(new NameTable());
63+
resolver.AddNamespace("xh", "http://www.w3.org/1999/xhtml");
64+
65+
Assert.NotEmpty(doc.XPathSelectElements("//xh:h1", resolver));
66+
// Won't match because the elements will have the XHTML namespace
67+
Assert.Empty(doc.XPathSelectElements("//h1"));
68+
}
69+
70+
[Fact]
71+
public void CanChangeToUpperCaseHtml()
72+
{
73+
var doc = HtmlDocument.Load(File("wikipedia.html"),
74+
new HtmlReaderSettings
75+
{
76+
CaseFolding = Sgml.CaseFolding.ToUpper,
77+
});
78+
79+
// The source has lowercase elements
80+
var central = doc.XPathSelectElement("/HTML/BODY/DIV/H1/SPAN");
81+
82+
Assert.NotNull(central);
83+
}
84+
85+
[Fact]
86+
public void HtmlSettings()
87+
{
88+
var doc = HtmlDocument.Load(File("wikipedia.html"),
89+
new HtmlReaderSettings
90+
{
91+
TextWhitespace = Sgml.TextWhitespaceHandling.TrimBoth,
92+
WhitespaceHandling = WhitespaceHandling.None
93+
});
94+
95+
var central = doc.XPathSelectElement("/html/body/div/h1/span");
96+
97+
// The source contains leading and trailing whitespaces.
98+
Assert.Equal("Wikipedia", central?.Value);
99+
}
100+
101+
string File(string path) => new Uri("file://" + new FileInfo(path).FullName).AbsoluteUri;
102+
}

‎src/Tests/Tests.csproj‎

Lines changed: 4 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,8 @@
22

33
<PropertyGroup>
44
<TargetFramework>net6.0</TargetFramework>
5+
<ImplicitUsings>enable</ImplicitUsings>
56
<AssemblyName>Devlooped.Tests</AssemblyName>
6-
<RootNamespace>Devlooped</RootNamespace>
7-
<NoWarn>xUnit1013</NoWarn>
87
</PropertyGroup>
98

109
<ItemGroup>
@@ -17,33 +16,15 @@
1716
<ItemGroup>
1817
<Using Include="Xunit" />
1918
<Using Include="Xunit.Abstractions" />
20-
<Using Include="Devlooped.Xml.Css" />
21-
<Import Include="@(Using)" />
22-
<!--
23-
<AdditionalFiles Include="page.html" SourceItemType="Html" />
24-
<CompilerVisibleItemMetadata Include="AdditionalFiles" MetadataName="SourceItemType" />
25-
-->
19+
<Using Include="Devlooped.Web" />
2620
</ItemGroup>
2721

2822
<ItemGroup>
29-
<ProjectReference Include="..\Css\Css.csproj" />
23+
<ProjectReference Include="..\Web\Web.csproj" />
3024
</ItemGroup>
3125

3226
<ItemGroup>
33-
<None Update="page.html" Generator="MSBuild:ForceCompile" CopyToOutputDirectory="PreserveNewest"/>
34-
<None Update="xunit.runner.json" CopyToOutputDirectory="PreserveNewest"/>
27+
<None Update="@(None)" CopyToOutputDirectory="PreserveNewest" />
3528
</ItemGroup>
3629

37-
<PropertyGroup>
38-
<CoreCompileDependsOn>ForceCompile;$(CoreCompileDependsOn)</CoreCompileDependsOn>
39-
</PropertyGroup>
40-
41-
<Target Name="ForceCompile" BeforeTargets="CoreCompile"
42-
Inputs="page.html" Outputs="$(BaseIntermediateOutputPath)page.g.cs">
43-
<WriteLinesToFile File="$(BaseIntermediateOutputPath)page.g.cs" Lines="" />
44-
<ItemGroup>
45-
<Compile Include="$(BaseIntermediateOutputPath)page.g.cs" />
46-
</ItemGroup>
47-
</Target>
48-
4930
</Project>

‎src/Tests/sample.html‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
<html lang="en" class="no-js">
2+
<head>
3+
<meta charset="utf-8">
4+
<title>Wikipedia</title>
5+
<script>
6+
// some script
7+
</script>
8+
<style>
9+
/* some style */
10+
</style>
11+
<meta name="viewport" content="initial-scale=1,user-scalable=yes">
12+
<link rel="shortcut icon" href="/static/favicon/wikipedia.ico">
13+
<link rel="license" href="//creativecommons.org/licenses/by-sa/3.0/">
14+
</head>
15+
<body>
16+
<script>
17+
// more script
18+
</script>
19+
<style>
20+
/* some style */
21+
</style>
22+
<h1>
23+
Wikipedia
24+
</h1>
25+
</body>
26+
</html>

‎src/Tests/sample.xhtml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
2+
<html lang="en" xmlns="http://www.w3.org/1999/xhtml">
3+
<body>
4+
<h1>
5+
Wikipedia
6+
</h1>
7+
</body>
8+
</html>

0 commit comments

Comments
 (0)