scraper.py

import requests
from bs4 import BeautifulSoup
import sys

def scrape_article(url):
    headers = {"User-Agent": "Mozilla/5.0"}
    response = requests.get(url, headers=headers)

    if response.status_code != 200:
        print(f"Failed to fetch page. Status code: {response.status_code}")
        return

    soup = BeautifulSoup(response.text, "html.parser")

    # Title
    title = soup.find("title")
    print("\n--- TITLE ---")
    print(title.text.strip() if title else "Title not found")

    # Author (tries common meta tags)
    author = soup.find("meta", {"name": "author"})
    print("\n--- AUTHOR ---")
    print(author["content"] if author else "Author not found")

    # Date (tries common meta tags)
    date = soup.find("meta", {"name": "date"}) or soup.find("meta", {"property": "article:published_time"})
    print("\n--- DATE ---")
    print(date["content"] if date else "Date not found")

    # Article content (grabs all paragraph text)
    print("\n--- CONTENT ---")
    paragraphs = soup.find_all("p")
    for p in paragraphs:
        text = p.text.strip()
        if len(text) > 40:  # skip short/nav paragraphs
            print(text)
            print()

if __name__ == "__main__":
    if len(sys.argv) < 2:
        print("Usage: python scraper.py <article_url>")
    else:
        scrape_article(sys.argv[1])