-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathscrape.py
More file actions
319 lines (202 loc) · 8.27 KB
/
scrape.py
File metadata and controls
319 lines (202 loc) · 8.27 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
# scrape
# nær í html töflur frá Numbeo og vistar í gagnagrunn
import requests
from bs4 import BeautifulSoup
import pandas as pd
import re
import sqlite3
def toNumeric(df):
names = df.columns.tolist()
names.remove("City")
names.remove("Rank")
for columnName in names:
df[columnName] = pd.to_numeric(df[columnName])
def replaceEuro(df):
for x in df.columns.tolist():
df[x] = df[x].str.replace('€' , '')
# use: s = beforeComma(string)
# before: string is a string that includes a comma
# after: s is the part of string that's before the comma
def beforeComma(string):
return string.split(",")[0]
# use: list = getHTMLTabless(url)
# before: the url contains HTML tables
# after: df is a list of dataframes with data from tables from url, with column names as headers from tables from url
def getHTMLTables(url):
res = []
r = requests.get(url)
soup = BeautifulSoup(r.text, 'lxml') # spurning með 'lxml', fer eftir töflunni
tables = soup.find_all('table')
#print(tables[2])
print(len(tables))
nrOfTables = 0
for table in tables:
# first nrOfTables tables have been added to res as data frames
headers = table.find_all('tr')[0].find_all('th')
nrOfColumns = len( headers ) # number of column names
columns = [[] for _ in headers]
if len(columns) > 0:
# data is only loaded into the tables with keys
# data is stored in the columns
nrOfRows = 0
# how many rows have been traversed
for row in table.find_all('tr')[1:]:
# first nrOfRows rows have been added to c[0..nrOfRows] where c are in columns
rowData = row.find_all('td')
nrOfRows += 1
if len(rowData) >0:
for columnNr in range(nrOfColumns):
if len(rowData[columnNr].contents) > 1:
# some html stuff in this column
# í tilfellinu þar sem þetta er 'range':
# tek það sem er á milli <span class="barTextLeft"> og </span>
# og <span class="barTextRight"> og </span>
# og greini á milli með "-"
lowest = rowData[columnNr].contents[1].string
highest = rowData[columnNr].contents[-1].string
if(lowest is not None and highest is not None):
# we can take substring og lowest
lowest = rowData[columnNr].contents[1].string[1:]
columns[columnNr].append(lowest + "-" + highest)
else:
columns[columnNr].append("")
else:
columns[columnNr].append( str(rowData[columnNr].string) )
# data in table has been loaded into columns
keys = []
for header in headers:
keys.append(header.string)
res.append(pd.DataFrame(dict(zip(keys, columns))))
# data frame with data from table with keys as key has been added to res
nrOfTables += 1
return res
# use: x = fixNames(cityNames)
# before: cityNames is a list of strings
# after: x is cityNames with '(' and ')' removed and spaces replaced with '-', for scraping in Numbeo
def fixNames(cityNames):
res = []
for name in cityNames:
res.append(name.replace(" ", "-").replace("(","").replace(")",""))
for n, name in enumerate(res):
if name == 'The-Hague-Den-Haag':
res[n] = 'The-Hague-Den-Haag-Netherlands'
return res
def afterComma(string):
if string is not None:
if len(string.split("-")) > 1:
return string.split("-")[1]
#### Numbeo
# náum í verðlagið fyrir allar borgirnar og lífsgæðin fyrir hverja borg
### Evropa, merkt með E í endann
# næ fyrst í cost of living index töfluna, fæ þaðan nöfnin (dálkur "city")
costIndexE = getHTMLTables("https://www.numbeo.com/cost-of-living/region_rankings.jsp?title=2018®ion=150")[0]
# costIndexE er dataframe með gögnunum úr costofliving fyrir Evrópu
qualityE = getHTMLTables("https://www.numbeo.com/quality-of-life/region_rankings.jsp?title=2018®ion=150")[0]
# qualityE er listi af data frames með quality of life indexana af Numbeo fyrir hverja borg í cityNamesE
cityNamesE = list(map(beforeComma, costIndexE["City"]))
# nöfnin á borgunum í Evrópu af Numbeo eru í cityNamesE
costE = []
# verðlagsgögnin fyrir borgirnar í cityNamesE verða í listanum af data frames costE
index = 0
numberOfCities = len(cityNamesE)
scrapeFailures = [] # price tables that didn't come through
for name in fixNames(cityNamesE[0:numberOfCities]):
# verðlags gögnin fyrir fyrstu index borgirnar í cityNamesE eru í fyrstu index röðunum í costE
url = "https://www.numbeo.com/cost-of-living/in/" + name + "?displayCurrency=EUR"
print(cityNamesE[index])
costTables = getHTMLTables(url)
if(len(costTables) > 0):
costTables = costTables[0]
costTables.columns = ["Goods", "Prices", "Range"]
costE.append(costTables)
else:
scrapeFailures.append(name)
scrapeFailures.append(costTables)
## Hreinsa burt töflurnar sem eru óþarfi ?
index += 1
costEdict = dict(zip(cityNamesE, costE))
# costE is a dictionary with names of cities attached
# # data framein sett upp til að bæt amegi í gagnagrunn.
nColumns = len(costIndexE.columns)
cNames = costIndexE.columns.tolist()
cNames.remove("City")
cNames.remove("Rank")
for columnName in cNames:
costIndexE[columnName] = pd.to_numeric(costIndexE[columnName])
toNumeric(qualityE)
for df in costE:
replaceEuro(df)
# evrutáknin ættu að vera farin úr töflunum í costTables
con = sqlite3.connect("C:/Users/Valdi/Desktop/Ferdasja sumar/database.db")
# # data frameið sett í sqlite gagnagrunn:
# # dfin þurfa að hafa sömu dálkaheiti og töflurnar.
costIndexE.to_sql("CostOfLivingIndex", con, if_exists="replace")
qualityE.to_sql("QualityOfLifeIndex", con, if_exists="replace")
# # # indexatöflurnar hafa verið settar í sql
# Gögnin sett í gagnagrunn:
# removes first and last letter of string
def removeFirst(string):
return string[1:-1]
print(removeFirst("yobo"))
with con:
cur = con.cursor()
cityNumber = 0
for city in fixNames(cityNamesE[0:numberOfCities]):
# cityNumber cities have been added to the db. price and price range
prices = costE[cityNumber]["Prices"].values.tolist()
prices = list(map(removeFirst,prices))
prices.insert(0,city)
print(prices)
range = costE[cityNumber]["Range"]
cityNumber += 1
lowRange = list(map( lambda string: string.split("-")[0], range) )
highRange = list(map(lambda x: afterComma(x), range))
lowRange.insert(0,city)
lowRange.insert(1,0)
highRange.insert(0,city)
highRange.insert(1,1)
if len(pricesAttributes) + 1 == len(prices):
# + 1 útaf city
questionMarks = "?,"*(len(prices)-1)
questionMarks += "?"
values = tuple(prices)
cur.execute("INSERT into Prices VALUES(" + questionMarks + ")", values)
# verðgögnin komin inn í Prices
if len(lowRange) == len(highRange):
# hendi fyrst út None sem er fyrir average salary því það er ekkert range
# auka dálkur fyrir 'high/low"
questionMarks += ",?"
values= tuple(lowRange)
# eyðum út
cur.execute("INSERT into PriceRange VALUES(" + questionMarks + ")", values)
#highRange = [x for x in highRange if x is not 'None']
values = tuple(highRange)
cur.execute("INSERT into PriceRange VALUES(" + questionMarks + ")", values)
# range gögnin komin inn í PriceRange
else:
print("unequal lengths, not all attributes ")
# # næ hér í altcurrency töfluna
# newCoins = getHTMLTables("https://coinmarketcap.com/new/")
# aukning = newCoins[0]["% 24h"]
# # er með strengi -37.06 %, vil fjarlæga aftasta, fá x[-1]
# aukning = list(map(lambda x: x[:-2],aukning))
# aukning = list(filter(None, aukning))
# #print(aukning)
# aukning = list(map(float,aukning))
# import numpy
# print(numpy.mean(aukning))
# # allCoins = getHTMLTables("https://coinmarketcap.com/historical/20170709/")
# # aukning = allCoins[0]["% 24h"]
# # aukning = list(map(lambda x: x[:-2],aukning))
# # aukning = list(filter(None, aukning))
# # #print(aukning)
# # aukning = list(map(float,aukning))
# # print(numpy.mean(aukning[0:100]))
# allCoins = getHTMLTables("https://coinmarketcap.com")
# aukning = allCoins[0]["% Change (24h)"]
# aukning = list(map(lambda x: x[:-2],aukning))
# aukning = list(filter(None, aukning))
# #print(aukning)
# aukning = list(map(float,aukning))
# print(numpy.argmax(aukning))
# #print(numpy.mean(aukning[0:100]))