Web_Scraper_Project/test.py at main · JimboTran12/Web_Scraper_Project · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
import pytest
import requests
from bs4 import BeautifulSoup
from module_1.page_reader import FileReader
from module_2.article_processor import HtmlProcessor
from module_3.open_ai_api import AI

from run import main

def test_url_extraction(): #Test if the urls are extracted from the input file
    reader = FileReader('Data/raw/input.txt')
    pages = reader.get_pages()
    assert isinstance(pages[0], requests.models.Response) #The type of the object should be a response object

def test_html_page_retrieval(): #Test if the html pages are retrieved from the urls
    reader = FileReader('Data/raw/input.txt')
    pages = reader.get_pages()
    assert len(pages) == 10  #10 pages should be loaded because of the 10 urls in the input file

def test_content_extraction(): #Test if the content is extracted from the html pages
    reader = FileReader('Data/raw/input.txt')
    pages = reader.get_pages()
    processor = HtmlProcessor(pages, "Data/processed")
    with open('Data/processed/url1.txt', 'r') as f:
        content = f.read()

    assert isinstance(content, str) and len(content) > 0#The content should be a string and should not be empty

def  test_html_processor_get_headline(): #Test if the headline is extracted from the html pages
    reader = FileReader('Data/raw/input.txt')
    pages = reader.get_pages()
    processor = HtmlProcessor(pages, "Data/processed")
    processor.process()
    with open('Data/processed/url1.txt', 'r') as f:
        content = f.readline()
    assert content == 'COST-FRIENDLY COLLEGE DATES FOR VALENTINE’S DAY\n'

def test_output_file_creation():#Test if the output files are created
    reader = FileReader('Data/raw/input.txt')
    pages = reader.get_pages()
    processor = HtmlProcessor(pages, 'Data/processed')
    processor.process()
    output_filenames = processor.get_output_filenames()
    assert output_filenames == ['Data/processed/url1.txt', 'Data/processed/url2.txt', 'Data/processed/url3.txt', 'Data/processed/url4.txt', 'Data/processed/url5.txt', 'Data/processed/url6.txt', 'Data/processed/url7.txt', 'Data/processed/url8.txt', 'Data/processed/url9.txt', 'Data/processed/url10.txt']

def test_file_reader_init(): #Test if the file reader is initialized
    reader = FileReader('Data/raw/input.txt')
    assert reader.input_file_name == 'Data/raw/input.txt'

def test_html_processor_init(): #Test if the html processor is initialized
    reader = FileReader('Data/raw/input.txt')
    pages = reader.get_pages()
    processor = HtmlProcessor(pages, "Data/processed")
    assert processor.output_dir == "Data/processed"

def test_ai_init(): #Test if the AI is initialized
    reader = FileReader('Data/raw/input.txt')
    pages = reader.get_pages()
    processor = HtmlProcessor(pages, "Data/processed")
    ai = AI(processor.get_output_filenames(), "Data/generated")

    assert ai.output_dir == "Data/generated" #Test if the output directory is set correctly