python-crawler/test_crawler.py at master · gabrielabud/python-crawler · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
import requests
import crawler

def test_get_content(requests_mock):
  requests_mock.get("https://www.yoyowallet.com/", content=b"html dom properties")
  assert crawler.get_content('https://www.yoyowallet.com/') == "html dom properties".encode()

def test_get_href():
  html_href = """<!DOCTYPE html>
  <html>
    <head>
      <meta charset="UTF-8">
    </head>
    <body>
      <a href="/retailers/index.html">BBC News</a>
    </body>
  </html>"""
  html_no_href = """<!DOCTYPE html>
  <html>
    <head>
      <meta charset="UTF-8">
      <title>title</title>
    </head>
    <body>
      <img src="https://unsplash.com/photos/8uXthE3xeBI"  alt="mountains"  />
    </body>
  </html>"""
  assert crawler.get_href(html_href) == ["/retailers/index.html"]
  assert crawler.get_href(html_no_href) == []

def test_get_other_assets():
  html_no_other_assets = """<!DOCTYPE html>
  <html>
    <head>
      <meta charset="UTF-8">
    </head>
    <body>
      <a href="/retailers/index.html">BBC News</a>
    </body>
  </html>"""
  html_other_assets = """<!DOCTYPE html>
  <html>
    <head>
      <meta charset="UTF-8">
      <link rel="stylesheet" type="text/css" href="theme.css">
      <script src="myscripts.js"></script>
    </head>
    <body>
      <img src="https://unsplash.com/photos/8uXthE3xeBI"  alt="mountains"  />
    </body>
  </html>"""
  assert crawler.get_other_assets(html_other_assets) == ["stylesheet", "myscripts.js", "https://unsplash.com/photos/8uXthE3xeBI"]
  assert crawler.get_other_assets(html_no_other_assets) == []


def test_get_html():
  html_sample = """<!DOCTYPE html>
  <html>
    <head>
      <meta charset="UTF-8">
      <link rel="stylesheet" type="text/css" href="theme.css">
      <script src="myscripts.js"></script>
    </head>
    <body>
      <img src="https://unsplash.com/photos/8uXthE3xeBI"  alt="mountains"  />
       <a href="/retailers/index.html">BBC News</a>
    </body>
  </html>"""
  assert crawler.get_html(html_sample) == {
    "href": ["/retailers/index.html"],
    "other": ["stylesheet", "myscripts.js",
    "https://unsplash.com/photos/8uXthE3xeBI"],
    "all": ["/retailers/index.html", "stylesheet", "myscripts.js", "https://unsplash.com/photos/8uXthE3xeBI"]
    }

def test_get_root_url():
  page_to_crawl = "https://www.yoyowallet.com/"
  assert crawler.get_root_url(page_to_crawl) == "yoyowallet.com"

def test_clean_links():
  all_links = [
    '/retailers/index.html',
    '/caterers/index.html',
    '/banks/index.html',
    'https://blog.yoyowallet.com/',
    'https://support.yoyowallet.com/hc/en-gb/categories/200398989-Help-Centre',
    '/careers.html',
    '/assets.html',
    'https://support.yoyowallet.com/hc/en-gb/categories/201132913-Legal-and-T-Cs',
    '/',
    'http://www.twitter.com/yoyowallet',
    'http://www.facebook.com/yoyowallet',
    'https://www.linkedin.com/company/yoyo-wallet'
    ]
  cleaned_links = {
    '/retailers/index.html',
    '/assets.html',
    '/banks/index.html',
    '/careers.html',
    '/',
    '/caterers/index.html'
    }
  assert crawler.clean_links(all_links) == cleaned_links


def test_crawl():
  sitemap = [
    '/',
    '/about.html',
    '/assets.html',
    '/banks/index.html',
    '/basket-data.html',
    '/careers.html',
    '/case-studies/caffe-nero-case-study.html',
    '/caterers/index.html',
    '/cookies.html',
    '/epos.html',
    '/get-in-touch.html',
    '/reports/caterers-report.html',
    '/retailers/index.html'
    ]
  page_to_crawl = "https://www.yoyowallet.com/"
  assert crawler.crawl(page_to_crawl)['sitemap'] == sitemap