HTMLScraper/scrapeTest1.py at master · CWJamieson/HTMLScraper · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
import requests;
from lxml import html;
import unicodedata


print ("hello world")

page = requests.get("Generic URL")
tree = html.fromstring(page.content)

#copied from chrome
title = tree.xpath('//span[@id="titletextonly"]/text()')

#what I copied from chrome

#price = tree.xpath('//*[@id="sortable-results"]/div[1]/p[1]/span/span[2]/span[1]/text()')
#rooms = tree.xpath('//*[@id="sortable-results"]/div[1]/p[1]/span/span[2]/span[2]/text()')

#what I changed it too

price = tree.xpath('//span[@class="price"]/text()')
rooms = tree.xpath('//span[@class="housing"]/text()')

start = tree.xpath('//span[@class="rangeFrom"]/text()')
end = tree.xpath('//span[@class="rangeTo"]/text()')

length = int(end[0]) - int(start[0]) +1


for i in range(0,length):

    curTitle=str(title[i])
    curPrice=str(price[i])
    curRooms=str(rooms[i])
    curTitle = curTitle.strip()
    curPrice = curPrice.strip()
    curRooms = curRooms.strip()
    if(curRooms[0] =='/'):
        curRooms = curRooms[2:]
    if(curRooms[len(curRooms)-1] == '-'):
        curRooms = curRooms[:2]
    print("Title:",curTitle,", price:",curPrice,", # of rooms:",curRooms)

input("Press enter to exit")