quotes-pagination-scraper/quote_pagination.py at main · sudip-python-dev/quotes-pagination-scraper · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
import requests
from bs4 import BeautifulSoup
import csv

rows = []
page_num = 1

while True:
    url = f"https://quotes.toscrape.com/page/{page_num}/"

    print(f"Current scraping URL: {url}")

    response = requests.get(url)

    soup = BeautifulSoup(response.text, "html.parser")

    quotes = soup.find_all("div", class_="quote")

    if not quotes:
        print("No more quotes found.")
        print(f"Total pages scraped: {page_num-1}")
        break

    for tag in quotes:
        quote = tag.find(
            "span",
            class_="text"
        ).text.strip()

        author = tag.find(
            "small",
            class_="author"
        ).text.strip()

        tag_list = [
            t.text for t in tag.find_all(
                "a",
                class_="tag"
            )
        ]

        tags = ", ".join(tag_list)

        row = {
            "Quote": quote,
            "Author": author,
            "Tags": tags
        }

        rows.append(row)

    page_num += 1

print("Writing data to CSV file...")

with open(
    "quotes_pagination_data.csv",
    "w",
    newline="",
    encoding="utf-8"
) as file:

    writer = csv.DictWriter(
        file,
        fieldnames=[
            "Quote",
            "Author",
            "Tags"
        ]
    )

    writer.writeheader()
    writer.writerows(rows)

print("CSV file has been written!")