site.py

import requests
from bs4 import BeautifulSoup # type: ignore
from urllib.parse import urljoin, urlparse
import time

# Configuration
BASE_URL = "https://scet.ac.in"
DOMAIN = urlparse(BASE_URL).netloc

HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
}

# Statistics
visited = set()
to_visit = [BASE_URL]
failed = set()

def is_valid(url):
    """Check if the URL is within the target domain and is likely an HTML page."""
    parsed = urlparse(url)
    # Stay within domain
    if parsed.netloc != DOMAIN and not parsed.netloc.endswith(f".{DOMAIN}"):
        return False
    
    # Exclude common non-html file extensions
    excluded_extensions = [
        '.pdf', '.jpg', '.jpeg', '.png', '.gif', '.bmp', '.svg', 
        '.zip', '.tar', '.gz', '.7z', '.rar',
        '.docx', '.xlsx', '.pptx', '.doc', '.xls', '.ppt',
        '.mp3', '.mp4', '.wav', '.avi', '.mov',
        '.css', '.js', '.xml', '.json'
    ]
    path = parsed.path.lower()
    if any(path.endswith(ext) for ext in excluded_extensions):
        return False
        
    return True

def clean_url(url):
    """Remove fragments and trailing slashes for normalization."""
    parsed = urlparse(url)
    # Remove fragment
    cleaned = parsed._replace(fragment="").geturl()
    # Remove trailing slash
    if cleaned.endswith("/") and len(urlparse(cleaned).path) > 1:
        cleaned = cleaned.rstrip("/")
    return cleaned

print(f"Starting crawl of {BASE_URL}...")

while to_visit:
    url = to_visit.pop(0)
    url = clean_url(url)
    
    if url in visited or url in failed:
        continue

    print(f"Crawling: {url} (Found: {len(visited)}, Remaining: {len(to_visit)})")
    
    try:
        r = requests.get(url, headers=HEADERS, timeout=10)
        visited.add(url)
        
        # Only parse HTML content
        content_type = r.headers.get('Content-Type', '').lower()
        if 'text/html' not in content_type:
            continue

        soup = BeautifulSoup(r.text, "html.parser")
        for link in soup.find_all("a", href=True):
            full_url = urljoin(url, link["href"])
            full_url = clean_url(full_url)
            
            if is_valid(full_url) and full_url not in visited and full_url not in to_visit:
                to_visit.append(full_url)
        
        # Be nice to the server
        time.sleep(0.1)

    except Exception as e:
        print(f"Failed to crawl {url}: {e}")
        failed.add(url)

# Save results to file
output_file = "sitemap.txt"
with open(output_file, "w") as f:
    for url in sorted(list(visited)):
        f.write(f"{url}\n")

print("-" * 30)
print(f"Crawl complete!")
print(f"Total pages found: {len(visited)}")
print(f"Failed URLs: {len(failed)}")
print(f"Results saved to: {output_file}")