Merge pull request #60 from pixlie/feature/real-world-tests

brainless · web-flow · commit 6df47385c6f5 · 2025-07-08T21:46:22.000+05:30
feat: add real-world integration tests (#56)
diff --git a/Cargo.toml b/Cargo.toml
@@ -86,3 +86,4 @@ chrono = { version = "0.4", features = ["serde"] }
 
 [dev-dependencies]
 tempfile = "3.0"
+serial_test = "3.0"
diff --git a/src/html_parser.rs b/src/html_parser.rs
@@ -42,6 +42,68 @@ impl HtmlNode {
 
         None
     }
+
+    /// Find elements by CSS-like path (ignoring IDs)
+    /// Example: "html body center table tbody tr td table tbody tr.athing.submission td.title"
+    pub fn find_by_path(&self, path: &str) -> Vec<&HtmlNode> {
+        let path_parts: Vec<&str> = path.split_whitespace().collect();
+        if path_parts.is_empty() {
+            return vec![];
+        }
+
+        let mut results = Vec::new();
+        self.find_by_path_recursive(&path_parts, 0, &mut results);
+        results
+    }
+
+    fn find_by_path_recursive<'a>(
+        &'a self,
+        path_parts: &[&str],
+        depth: usize,
+        results: &mut Vec<&'a HtmlNode>,
+    ) {
+        if depth >= path_parts.len() {
+            return;
+        }
+
+        let current_part = path_parts[depth];
+
+        // Check if current node matches the current path part
+        if self.matches_path_part(current_part) {
+            if depth == path_parts.len() - 1 {
+                // This is the final part, add to results
+                results.push(self);
+            } else {
+                // Continue searching in children for the next part
+                for child in &self.children {
+                    child.find_by_path_recursive(path_parts, depth + 1, results);
+                }
+            }
+        }
+
+        // Also check children for the current part (to handle non-matching intermediate nodes)
+        for child in &self.children {
+            child.find_by_path_recursive(path_parts, depth, results);
+        }
+    }
+
+    fn matches_path_part(&self, part: &str) -> bool {
+        // Parse part like "tr.athing.submission" or just "td"
+        if let Some(dot_pos) = part.find('.') {
+            let tag = &part[..dot_pos];
+            let classes_str = &part[dot_pos + 1..];
+            let required_classes: Vec<&str> = classes_str.split('.').collect();
+
+            // Check tag matches and all required classes are present
+            self.tag == tag
+                && required_classes
+                    .iter()
+                    .all(|class| self.classes.contains(&class.to_string()))
+        } else {
+            // Just a tag name
+            self.tag == part
+        }
+    }
 }
 
 pub struct HtmlParser {
@@ -360,4 +422,75 @@ mod tests {
         assert!(!parser.is_same_domain("https://other.com/page", "example.com"));
         assert!(!parser.is_same_domain("https://notexample.com/page", "example.com"));
     }
+
+    #[test]
+    fn test_find_by_path() {
+        let parser = HtmlParser::new();
+        let html = r#"<html>
+            <body>
+                <center>
+                    <table>
+                        <tbody>
+                            <tr>
+                                <td>
+                                    <table>
+                                        <tbody>
+                                            <tr class="athing submission">
+                                                <td class="title">First Item</td>
+                                            </tr>
+                                            <tr class="athing submission">
+                                                <td class="title">Second Item</td>
+                                            </tr>
+                                        </tbody>
+                                    </table>
+                                </td>
+                            </tr>
+                        </tbody>
+                    </table>
+                </center>
+            </body>
+        </html>"#;
+        let tree = parser.parse(html);
+
+        // Test finding elements by path
+        let results = tree.find_by_path(
+            "html body center table tbody tr td table tbody tr.athing.submission td.title",
+        );
+        assert_eq!(results.len(), 2);
+        assert_eq!(results[0].content, "First Item");
+        assert_eq!(results[1].content, "Second Item");
+
+        // Test finding by simple tag path
+        let body_results = tree.find_by_path("html body");
+        assert_eq!(body_results.len(), 1);
+        assert_eq!(body_results[0].tag, "body");
+
+        // Test non-existent path
+        let empty_results = tree.find_by_path("html body div.nonexistent");
+        assert_eq!(empty_results.len(), 0);
+    }
+
+    #[test]
+    fn test_matches_path_part() {
+        let node = HtmlNode::new(
+            "tr".to_string(),
+            vec!["athing".to_string(), "submission".to_string()],
+            None,
+            String::new(),
+        );
+
+        // Test matching with classes
+        assert!(node.matches_path_part("tr.athing.submission"));
+        assert!(node.matches_path_part("tr.athing"));
+        assert!(node.matches_path_part("tr"));
+
+        // Test non-matching
+        assert!(!node.matches_path_part("td.athing"));
+        assert!(!node.matches_path_part("tr.nonexistent"));
+
+        // Test simple tag matching
+        let simple_node = HtmlNode::new("div".to_string(), vec![], None, String::new());
+        assert!(simple_node.matches_path_part("div"));
+        assert!(!simple_node.matches_path_part("span"));
+    }
 }
diff --git a/tests/README.md b/tests/README.md
@@ -0,0 +1,109 @@
+# Real-World Integration Tests
+
+This directory contains integration tests that crawl real websites to verify SmartCrawler's complete functionality in real-world scenarios, including domain-level duplicate filtering.
+
+## Prerequisites
+
+Before running these tests, you need to have a WebDriver server running:
+
+### Option 1: GeckoDriver (Firefox) - Recommended
+```bash
+# Install geckodriver and start it
+geckodriver
+```
+
+### Option 2: ChromeDriver
+```bash
+# Download ChromeDriver and start it on port 4444
+chromedriver --port=4444
+```
+
+### Option 3: Docker
+```bash
+# Run Chrome in headless mode with WebDriver
+docker run -d -p 4444:4444 selenium/standalone-chrome:latest
+```
+
+## Running the Tests
+
+The real-world tests are ignored by default to prevent them from running during normal development. To run them explicitly:
+
+```bash
+# Run all real-world tests (they run serially to avoid WebDriver conflicts)
+cargo test real_world -- --ignored
+
+# Run a specific test
+cargo test test_hacker_news_submissions -- --ignored
+cargo test test_mykin_ai_team_member -- --ignored
+
+# Test WebDriver connection
+cargo test test_webdriver_connection -- --ignored
+```
+
+### Serial Execution
+
+**Important**: Real-world tests are configured to run serially (one at a time) using the `serial_test` crate. This prevents WebDriver session conflicts that occur when multiple tests try to use the same WebDriver port (4444) simultaneously.
+
+The `#[serial]` attribute ensures that:
+- Tests won't interfere with each other's WebDriver sessions
+- Each test gets exclusive access to the WebDriver instance
+- Tests are more reliable and predictable
+
+## Test Descriptions
+
+### `test_hacker_news_submissions`
+- **URL**: https://news.ycombinator.com/
+- **Purpose**: Verifies the complete SmartCrawler pipeline including domain-level duplicate filtering
+- **Pipeline**: Full 3-phase crawling with link discovery, root URL prioritization, and duplicate analysis
+- **Path**: `html body center table tbody tr td table tbody tr.athing.submission td.title`
+- **Expected**: Exactly 30 elements matching this path (including filtered duplicates)
+- **Features Tested**: Link discovery, domain duplicate detection, filtered content identification
+
+### `test_mykin_ai_team_member`
+- **URL**: https://mykin.ai/company
+- **Purpose**: Verifies the complete SmartCrawler pipeline on a modern web framework
+- **Pipeline**: Full 3-phase crawling with link discovery, root URL prioritization, and duplicate analysis
+- **Path**: `html.w-mod-js.w-mod-ix body div.page-wrapper main.main-wrapper section.section_team div.padding-global div.container-medium div.team_collection.is-desktop.w-dyn-list div.team_collection-list.w-dyn-items div.w-dyn-item a.team_card.w-inline-block div.team_content h4`
+- **Expected**: At least one element containing "Kasper Juul" (excluding filtered duplicates)
+- **Features Tested**: Complex CSS class matching, content filtering, modern web app crawling
+
+## Manual Verification
+
+If a test fails, you should manually verify the webpage structure:
+
+1. Visit the URL in your browser
+2. Inspect the HTML structure using browser developer tools
+3. Check if the CSS path still matches the expected elements
+4. Update the test path if the website structure has changed
+
+## SmartCrawler Pipeline Features Tested
+
+The real-world tests exercise the complete SmartCrawler functionality:
+
+### Phase 1: Preparation Stage
+- **Link Discovery**: Extracts same-domain links from initial page
+- **Root URL Prioritization**: Automatically adds domain root URLs
+- **URL Collection**: Gathers up to 3 URLs per domain for analysis
+
+### Phase 2: Processing Stage  
+- **Sequential Processing**: Processes URLs in priority order (user → root → discovered)
+- **Content Extraction**: Retrieves HTML source and parses into structured tree
+- **Status Tracking**: Monitors fetch success/failure for each URL
+
+### Phase 3: Analysis Stage
+- **Domain Duplicate Detection**: Identifies common elements across domain pages
+- **Content Filtering**: Marks duplicate patterns as `[FILTERED DUPLICATE]`
+- **Result Preparation**: Provides clean, filtered HTML trees for analysis
+
+## Notes
+
+- These tests depend on external websites and may fail if:
+  - The websites are down or inaccessible
+  - The website structure has changed
+  - Network connectivity issues
+  - WebDriver is not running or accessible
+- Element IDs are ignored in path matching to make tests more robust against dynamic content
+- Tests include verbose output to help with debugging when they fail
+- **Serial execution**: Tests run one at a time to prevent WebDriver session conflicts - this is automatic and requires no special flags
+- **Full pipeline testing**: Each test exercises the complete 3-phase SmartCrawler workflow
+- **Duplicate filtering**: Tests verify that domain-level filtering correctly identifies and marks duplicate content
diff --git a/tests/real_world_tests.rs b/tests/real_world_tests.rs

Original file line number	Diff line number	Diff line change
`@@ -86,3 +86,4 @@ chrono = { version = "0.4", features = ["serde"] }`
`86`	`86`
`87`	`87`	`[dev-dependencies]`
`88`	`88`	`tempfile = "3.0"`
	`89`	`+serial_test = "3.0"`