scraper.py

import requests
from database import Job, engine
from sqlalchemy.orm import Session
import schedule
import time
import json
from colorama import init, Fore, Style
from bs4 import BeautifulSoup

init(autoreset=True)

URL = "https://remotive.com/api/remote-jobs"

#   Loading values from config.json
with open("config.json", "r") as f:
    config = json.load(f)

categories = config["categories"]
limit = config["limit_per_category"]
ALLOWED_LOCATIONS = config["allowed_locations"]

#   Checks if any allowed keyword appears in the location string
def is_allowed_location(location: str) -> bool:
    location_lower = location.lower()
    return any(keyword in location_lower for keyword in ALLOWED_LOCATIONS)

#   Base scraper function logic
def scrape_remotive(category):

    params = {"category": category, "limit": limit}
    response = requests.get(URL, params=params)
    response_status = response.status_code

    if response_status == 200:
        json_data = response.json()
        jobs = json_data["jobs"]
        print(f"{Fore.GREEN}Status: {response_status}")

        scraped_jobs = []

        for job in jobs:
            clean_dict = {
                "title": job["title"] if job["title"] else "Not specified",
                "company": job["company_name"] if job["company_name"] else "Not specified",
                "category": category,
                "location": job["candidate_required_location"] if job["candidate_required_location"] else "Not specified",
                "salary": job["salary"] if job["salary"] else "Not specified",
                "tags": ", ".join(job["tags"]) if job["tags"] else "Not specified",
                "url": job["url"] if job["url"] else "Not specified"
            }

            #   Only add scraped job if it matches predetermined employment locations
            if is_allowed_location(job["candidate_required_location"]):
                scraped_jobs.append(clean_dict)

        print(f"Jobs Found: {len(scraped_jobs)}") 

        # for job in scraped_jobs:
        #     for key, value in job.items():
        #         print(f"{key:12} {value}")
            
        #     print("---------------------------------------")

        #   When inserting rows. Must start a session
        with Session(engine) as session:

            #   Instead of making multiple database requests. Pull all the urls in one database request and compare to that
            #   Fetches all the URLs from the database
            existing_urls = {row.url for row in session.query(Job.url).all()}

            #   Checks if the scrapped job is part of the previously pulled set of urls
            #   If it isn't covert to a Job object and add it to the list of new job objects
            new_jobs = [Job(**job) for job in scraped_jobs if job["url"] not in existing_urls]

            session.add_all(new_jobs)
            session.commit()

        print(f"Saved {len(new_jobs)} new jobs, {len(scraped_jobs) - len(new_jobs)} duplicates skipped")

    else:
        print(f"{Fore.RED}Request Failed with status: {response_status}")

def scrape_wwr(url):
    headers = {"User-Agent": "Mozilla/5.0"}
    response = requests.get(url, headers=headers,timeout=10)
    response_status = response.status_code

    if response_status == 200:
        print(f"{Fore.GREEN}Status: {response_status}")
        soup = BeautifulSoup(response.text, "html.parser")

        listings = soup.find_all("li", class_ = "new-listing-container")

        # DEBUGGING STUFF
        # print(f"Total listings found: {len(listings)}")
        # print(soup.prettify()[:2000])

        scraped_jobs = [] 
        category = url.split("/")[-1]

        for listing in listings:
            title = listing.find(class_ = "new-listing__header__title__text")
            company = listing.find(class_ = "new-listing__company-name")
            link = listing.find("a", class_ = "listing-link--unlocked")

            title_text   = title.text.strip()   if title   else "Not specified"
            company_text = company.text.strip() if company else "Not specified"
            job_url      = "https://weworkremotely.com" + link["href"] if link else None

            if not job_url:
                continue

            clean_dict = {
                "title":    title_text,
                "company":  company_text,
                "category": category,
                "location": "Remote",       # WWR is all remote
                "salary":   "Not specified",
                "tags":     "Not specified",
                "url":      job_url
            }

            if is_allowed_location(clean_dict["location"]):
                scraped_jobs.append(clean_dict)

        print(f"Jobs Found: {len(scraped_jobs)}") 

        with Session(engine) as session:

            #   Instead of making multiple database requests. Pull all the urls in one database request and compare to that
            #   Fetches all the URLs from the database
            existing_urls = {row.url for row in session.query(Job.url).all()}

            #   Checks if the scrapped job is part of the previously pulled set of urls
            #   If it isn't covert to a Job object and add it to the list of new job objects
            new_jobs = [Job(**job) for job in scraped_jobs if job["url"] not in existing_urls]

            session.add_all(new_jobs)
            session.commit()

        print(f"Saved {len(new_jobs)} new jobs, {len(scraped_jobs) - len(new_jobs)} duplicates skipped")
    else:
        print(f"{Fore.RED}WWR request failed: {response.status_code}")
    
#   Scrape for each job title
wwr_categories = config["wwr_categories"]

def scrape_all():
    for category in categories:
        print(f"{Fore.CYAN}{Style.BRIGHT}--- Scraping Remotive: {category} ---")
        scrape_remotive(category)
    
    for wwr_url in wwr_categories:
        category_slug = wwr_url.split("/")[-1]
        print(f"{Fore.CYAN}{Style.BRIGHT}--- Scraping WWR: {category_slug} ---")
        scrape_wwr(wwr_url)

scrape_all()
print(f"{Fore.YELLOW}Scheduler Running...    (press Ctrl+C to stop)")

schedule.every(1).hours.do(scrape_all)
while True:
    schedule.run_pending()
    time.sleep(60)