Skip to content

Commit 16e8215

Browse files
zTgxzTgx
authored
Merge pull request #19 from vectorlessflow/dev
Dev
2 parents 1e79ac7 + f8ed886 commit 16e8215

10 files changed

Lines changed: 1757 additions & 5 deletions

File tree

‎Cargo.toml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[package]
22
name = "vectorless"
3-
version = "0.1.17"
3+
version = "0.1.18"
44
edition = "2024"
55
authors = ["zTgx <beautifularea@gmail.com>"]
66
description = "Hierarchical, reasoning-native document intelligence engine"
@@ -83,6 +83,9 @@ rand = "0.8"
8383
# BM25 scoring
8484
bm25 = { version = "2.3.2", features = ["parallelism"] }
8585

86+
# HTML parsing
87+
scraper = "0.22"
88+
8689
[dev-dependencies]
8790
tempfile = "3.10"
8891
tokio-test = "0.4"

‎examples/html_parser.rs‎

Lines changed: 291 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,291 @@
1+
// Copyright (c) 2026 vectorless developers
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
//! HTML Parser Example.
5+
//!
6+
//! This example demonstrates how to parse HTML documents using vectorless.
7+
//!
8+
//! # Features
9+
//!
10+
//! - Parses HTML5 documents
11+
//! - Extracts heading hierarchy (h1-h6)
12+
//! - Extracts content from paragraphs, lists, tables
13+
//! - Extracts metadata from <head> (title, description, etc.)
14+
//!
15+
//! # Usage
16+
//!
17+
//! ```bash
18+
//! cargo run --example html_parser
19+
//! ```
20+
21+
use vectorless::parser::{DocumentParser, HtmlConfig, HtmlParser};
22+
23+
#[tokio::main]
24+
async fn main() -> vectorless::Result<()> {
25+
println!("=== HTML Parser Example ===\n");
26+
27+
// 1. Basic HTML parsing
28+
println!("--- Step 1: Basic HTML Parsing ---\n");
29+
demo_basic_parsing().await?;
30+
31+
// 2. Parsing with metadata
32+
println!("\n--- Step 2: HTML with Metadata ---\n");
33+
demo_metadata_parsing().await?;
34+
35+
// 3. Complex HTML structure
36+
println!("\n--- Step 3: Complex HTML Structure ---\n");
37+
demo_complex_structure().await?;
38+
39+
// 4. Configuration options
40+
println!("\n--- Step 4: Configuration Options ---\n");
41+
demo_configuration().await?;
42+
43+
// 5. Integration with Engine
44+
println!("\n--- Step 5: Integration with Engine ---\n");
45+
demo_engine_integration();
46+
47+
println!("\n=== Done ===");
48+
Ok(())
49+
}
50+
51+
/// Demonstrate basic HTML parsing.
52+
async fn demo_basic_parsing() -> vectorless::Result<()> {
53+
let parser = HtmlParser::new();
54+
let html = r#"
55+
<!DOCTYPE html>
56+
<html>
57+
<head><title>Basic Document</title></head>
58+
<body>
59+
<h1>Main Title</h1>
60+
<p>This is the introduction paragraph.</p>
61+
62+
<h2>Section 1</h2>
63+
<p>Content for section 1.</p>
64+
65+
<h2>Section 2</h2>
66+
<p>Content for section 2.</p>
67+
<h3>Subsection 2.1</h3>
68+
<p>Detailed content here.</p>
69+
</body>
70+
</html>
71+
"#;
72+
73+
let result = parser.parse(html).await?;
74+
75+
println!("Document: {}", result.meta.name);
76+
println!("Nodes extracted: {}\n", result.nodes.len());
77+
78+
for node in &result.nodes {
79+
println!(" {} {} (level {})",
80+
"•".repeat(node.level),
81+
node.title,
82+
node.level
83+
);
84+
if !node.content.is_empty() {
85+
let preview: String = node.content.chars().take(50).collect();
86+
println!(" Content: {}...", preview);
87+
}
88+
}
89+
90+
Ok(())
91+
}
92+
93+
/// Demonstrate parsing HTML with metadata.
94+
async fn demo_metadata_parsing() -> vectorless::Result<()> {
95+
let parser = HtmlParser::new();
96+
let html = r#"
97+
<!DOCTYPE html>
98+
<html>
99+
<head>
100+
<title>Technical Documentation</title>
101+
<meta name="description" content="Complete guide to the API">
102+
<meta name="author" content="Documentation Team">
103+
<meta name="keywords" content="API, REST, documentation">
104+
<meta property="og:description" content="Open Graph description">
105+
</head>
106+
<body>
107+
<h1>API Reference</h1>
108+
<p>Introduction to the API.</p>
109+
</body>
110+
</html>
111+
"#;
112+
113+
let result = parser.parse(html).await?;
114+
115+
println!("Metadata extracted:");
116+
println!(" Title: {}", result.meta.name);
117+
println!(" Description: {:?}", result.meta.description);
118+
println!(" Format: {:?}", result.meta.format);
119+
println!(" Lines: {}", result.meta.line_count);
120+
121+
Ok(())
122+
}
123+
124+
/// Demonstrate parsing complex HTML structure.
125+
async fn demo_complex_structure() -> vectorless::Result<()> {
126+
let parser = HtmlParser::new();
127+
let html = r#"
128+
<!DOCTYPE html>
129+
<html>
130+
<body>
131+
<h1>Complex Document</h1>
132+
133+
<h2>Lists</h2>
134+
<ul>
135+
<li>First item</li>
136+
<li>Second item</li>
137+
<li>Third item</li>
138+
</ul>
139+
140+
<ol>
141+
<li>Step one</li>
142+
<li>Step two</li>
143+
<li>Step three</li>
144+
</ol>
145+
146+
<h2>Table</h2>
147+
<table>
148+
<tr><th>Name</th><th>Value</th></tr>
149+
<tr><td>Option A</td><td>100</td></tr>
150+
<tr><td>Option B</td><td>200</td></tr>
151+
</table>
152+
153+
<h2>Code Block</h2>
154+
<pre><code>fn main() {
155+
println!("Hello, World!");
156+
}</code></pre>
157+
158+
<h2>Blockquote</h2>
159+
<blockquote>
160+
This is a quoted text from another source.
161+
It can span multiple lines.
162+
</blockquote>
163+
</body>
164+
</html>
165+
"#;
166+
167+
let result = parser.parse(html).await?;
168+
169+
println!("Nodes with complex content:\n");
170+
for node in &result.nodes {
171+
println!(" [Level {}] {}", node.level, node.title);
172+
if node.content.contains("•") || node.content.contains("1.") {
173+
println!(" → Contains list content");
174+
}
175+
if node.content.contains("|") {
176+
println!(" → Contains table content");
177+
}
178+
if node.content.contains("```") {
179+
println!(" → Contains code block");
180+
}
181+
if node.content.contains(">") {
182+
println!(" → Contains blockquote");
183+
}
184+
}
185+
186+
Ok(())
187+
}
188+
189+
/// Demonstrate configuration options.
190+
async fn demo_configuration() -> vectorless::Result<()> {
191+
// Default configuration
192+
let _default_parser = HtmlParser::new();
193+
println!("Default config:");
194+
println!(" - max_heading_level: 6");
195+
println!(" - include_code_blocks: true");
196+
println!(" - merge_small_nodes: true");
197+
println!(" - min_content_length: 50\n");
198+
199+
// Custom configuration
200+
let config = HtmlConfig::new()
201+
.with_max_heading_level(3) // Only h1-h3
202+
.with_code_blocks(false) // Exclude code
203+
.with_min_content_length(20) // Smaller threshold
204+
.with_default_title("Overview");
205+
206+
let custom_parser = HtmlParser::with_config(config);
207+
println!("Custom config:");
208+
println!(" - max_heading_level: 3");
209+
println!(" - include_code_blocks: false");
210+
println!(" - min_content_length: 20");
211+
println!(" - default_title: \"Overview\"\n");
212+
213+
// Parse with custom config
214+
let html = r#"
215+
<html>
216+
<body>
217+
<h1>Title</h1>
218+
<p>Short.</p>
219+
<h4>This heading is ignored (level > 3)</h4>
220+
<p>This content goes to parent.</p>
221+
</body>
222+
</html>
223+
"#;
224+
225+
let result = custom_parser.parse(html).await?;
226+
println!("Nodes with max_level=3: {}", result.nodes.len());
227+
228+
// Show preset configs
229+
println!("\nPreset configurations:");
230+
let simple = HtmlConfig::simple();
231+
println!(" HtmlConfig::simple():");
232+
println!(" - merge_small_nodes: {}", simple.merge_small_nodes);
233+
println!(" - min_content_length: {}", simple.min_content_length);
234+
235+
let no_code = HtmlConfig::no_code_blocks();
236+
println!(" HtmlConfig::no_code_blocks():");
237+
println!(" - include_code_blocks: {}", no_code.include_code_blocks);
238+
239+
Ok(())
240+
}
241+
242+
/// Demonstrate integration with Engine.
243+
fn demo_engine_integration() {
244+
println!("Integration with Engine:\n");
245+
246+
println!("```rust");
247+
println!("use vectorless::{{EngineBuilder, IndexContext}};");
248+
println!("use vectorless::parser::DocumentFormat;");
249+
println!();
250+
println!("# #[tokio::main]");
251+
println!("# async fn main() -> vectorless::Result<()> {{");
252+
println!(" let engine = EngineBuilder::new()");
253+
println!(" .with_workspace(\"./workspace\")");
254+
println!(" .build()");
255+
println!(" .await?;");
256+
println!();
257+
println!(" // Method 1: From HTML file");
258+
println!(" let doc_id = engine.index(");
259+
println!(" IndexContext::from_path(\"./documentation.html\")");
260+
println!(" ).await?;");
261+
println!();
262+
println!(" // Method 2: From HTML content");
263+
println!(" let html = r#\"");
264+
println!("<html>");
265+
println!("<head><title>My Doc</title></head>");
266+
println!("<body>");
267+
println!(" <h1>Introduction</h1>");
268+
println!(" <p>Content here...</p>");
269+
println!("</body>");
270+
println!("</html>");
271+
println!("\"#;");
272+
println!();
273+
println!(" let doc_id = engine.index(");
274+
println!(" IndexContext::from_content(html, DocumentFormat::Html)");
275+
println!(" .with_name(\"my-document\")");
276+
println!(" ).await?;");
277+
println!();
278+
println!(" // Query the indexed document");
279+
println!(" let result = engine.query(&doc_id, \"What is the introduction?\").await?;");
280+
println!(" println!(\"{{}}\", result.content);");
281+
println!();
282+
println!(" Ok(())");
283+
println!("}}");
284+
println!("```\n");
285+
286+
println!("Supported file extensions:");
287+
println!(" - .html, .htm → HTML format");
288+
println!(" - .md, .markdown → Markdown format");
289+
println!(" - .pdf → PDF format");
290+
println!(" - .docx → Word document");
291+
}

0 commit comments

Comments
 (0)