Web-Scraper/config.yaml at main · Hacknova49/Web-Scraper · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
# Web Scraper Configuration
scraper:
  # Request settings
  timeout: 30
  max_retries: 3
  retry_delay: 1
  rate_limit: 1  # seconds between requests

  # Headers
  headers:
    User-Agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
    Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"
    Accept-Language: "en-US,en;q=0.5"
    Accept-Encoding: "gzip, deflate"
    Connection: "keep-alive"
    Upgrade-Insecure-Requests: "1"

# Example scraping targets
targets:
  example_news:
    base_url: "https://example-news-site.com"
    selectors:
      title: "h1.article-title"
      content: "div.article-content"
      author: "span.author-name"
      date: "time.publish-date"
    pagination:
      enabled: true
      next_button: "a.next-page"
      max_pages: 10

  example_ecommerce:
    base_url: "https://example-shop.com/products"
    selectors:
      product_name: "h2.product-title"
      price: "span.price"
      rating: "div.rating"
      availability: "span.stock-status"
    pagination:
      enabled: true
      next_button: "a[aria-label='Next']"
      max_pages: 5

# Output settings
output:
  format: "csv"  # csv, json, xlsx
  filename: "scraped_data"
  include_timestamp: true