doneDeal/DataGathering.py at master · rdoreilly/doneDeal · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
import requests
import re
from bs4 import BeautifulSoup

# URL we are looking for
url = "https://www.donedeal.ie/cars/Ford/C-MAX?start=1"
headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'}
response = requests.get(url, headers=headers)

queryURLS = []
# PULL THE 1st card-collection from the page as there is a 2nd with advertisments.
# More info on regex to be found below
# https://stackoverflow.com/questions/499345/regular-expression-to-extract-url-from-an-html-link
soup = BeautifulSoup(response.content, 'html.parser')
for tag in soup.findAll('a', href=True, class_="card__link"):
    queryURLS.append(tag['href'])
#print(queryURLS)


count=0
#Pull data from each of the pages
for url in queryURLS:
    print(url)
    response = requests.get(url, headers=headers)
    soup = BeautifulSoup(response.content, 'html.parser')
    #file = open("data/car"+str(count)+".html","w")
    count += 1

    #title = soup.findAll("h1", {"itemprop":"name"})
    #print(title)

    #content = soup.prettify()
    #print(content)
    #file.write(content)
    #file.close()


    #print(soup.prettify())

    title = soup.findAll("title")
    print(title)
    #price = soup.findAll("span", {"class" : "price ng-binding"})
    #print(price)
    #county = soup.findAll('span', class_="county-disp ng-binding")

    #print(price)
    #print(county)


    #main-content page-row ng-scope
    #< h1 itemprop = "name" ng-bind-html = "adView.ad.header" class ="ng-binding" > Ford C-Max < / h1 >
    #<span class="county-disp ng-binding">Offaly</span>
    #<span class ="price ng-binding" > 999 < / span >

#    class ="cad-content divider"

#class="cad-info-container space-top-10"