script.py

import requests
from urllib.parse import urljoin
import os
import PyPDF2
import pdfplumber
import logging
from urllib.parse import urlparse

# Set up logging
logger = logging.getLogger("PyPDF2")
logger.setLevel(logging.ERROR)
logging.getLogger('pdfminer').setLevel(logging.ERROR)
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

def extract_text_with_fallback(pdf_path):
    """
    Extract text from PDF using PyPDF2 with pdfplumber as fallback.

    Args:
        pdf_path (str): Path to PDF file

    Returns:
        str: Extracted text from PDF
    """
    try:
        # Try PyPDF2 first
        text = extract_text_pypdf2(pdf_path)
        if not text.strip():
            # If PyPDF2 returns empty text, try pdfplumber
            text = extract_text_pdfplumber(pdf_path)
        return text
    except Exception as e:
        logger.error(f"Failed to extract text from {pdf_path}: {str(e)}")
        return ""


def extract_text_pypdf2(pdf_path):
    with open(pdf_path, 'rb') as file:
        reader = PyPDF2.PdfReader(file)
        return " ".join(page.extract_text() for page in reader.pages)


def extract_text_pdfplumber(pdf_path):
    with pdfplumber.open(pdf_path) as pdf:
        return " ".join(page.extract_text() for page in pdf.pages)


def search_string_in_pdfs(download_folder, search_string):
    """
    Search for string in downloaded PDFs.

    Args:
        download_folder (str): Directory containing PDFs
        search_string (str): Text to search for
    """
    if not os.path.exists(download_folder):
        logger.error(f"Download folder '{download_folder}' not found")
        return

    found_count = 0
    for pdf_file in os.listdir(download_folder):
        if pdf_file.lower().endswith('.pdf'):
            pdf_path = os.path.join(download_folder, pdf_file)
            #logger.info(f"Searching {pdf_file}")
            text = extract_text_with_fallback(pdf_path)

            if search_string.lower() in text.lower():
                found_count += 1
                logger.info(f"Match found in: {pdf_file}")

    logger.info(f"Search completed. Found {found_count} matches.")


def find_pdf_links(html_content, base_url):
    """
    Extract PDF links from HTML content.

    Args:
        html_content (str): HTML content of the page
        base_url (str): Base URL for relative links

    Returns:
        list: List of PDF URLs
    """
    from bs4 import BeautifulSoup
    soup = BeautifulSoup(html_content, 'html.parser')
    pdf_links = []

    for link in soup.find_all('a', href=True):
        href = link['href']
        if href.lower().endswith('.pdf'):
            pdf_links.append(urljoin(base_url, href))

    return pdf_links


def download_pdfs(url_list_file, download_folder):
    """
    Download PDFs from URLs listed in a text file.

    Args:
        url_list_file (str): Path to file containing URLs
        download_folder (str): Directory to save downloaded PDFs
    """
    # Create the download folder if it doesn't exist
    os.makedirs(download_folder, exist_ok=True)

    try:
        with open(url_list_file, 'r') as file:
            urls = file.readlines()
    except FileNotFoundError:
        logger.error(f"URL list file '{url_list_file}' not found")
        return

    for url in urls:
        url = url.strip()
        try:
            response = requests.get(url, timeout=30)
            response.raise_for_status()

            # Check if the response is a PDF
            if response.headers.get('Content-Type') == 'application/pdf':
                download_pdf(url, download_folder)
            else:
                logger.info(f"No PDF found at {url}, Content-Type: {response.headers.get('Content-Type')}")

        except requests.RequestException as e:
            logger.error(f"Failed to process URL {url}: {str(e)}")


def download_pdf(pdf_url, download_folder):
    """
    Download a PDF file from a given URL.

    Args:
        pdf_url (str): URL of the PDF file
        download_folder (str): Directory to save the PDF
    """
    try:
        pdf_response = requests.get(pdf_url, timeout=30)
        pdf_response.raise_for_status()

        # Extract a unique filename from the URL
        parsed_url = urlparse(pdf_url)
        numeric_part = extract_numeric_part(parsed_url.path)  # Extract numeric part from the URL
        unique_name = f"document_{numeric_part}.pdf"  # Create a unique filename

        # Construct the full path for saving the PDF
        pdf_name = os.path.join(download_folder, unique_name)

        # Ensure the filename is valid (sanitize only the filename part)
        pdf_name = sanitize_filename(pdf_name)

        # Save the PDF to the specified folder
        with open(pdf_name, 'wb') as pdf_file:
            pdf_file.write(pdf_response.content)
        logger.info(f"Downloaded: {pdf_name}")
    except requests.RequestException as e:
        logger.error(f"Failed to download PDF from {pdf_url}: {str(e)}")


def extract_numeric_part(path):
    """
    Extract the numeric part from the URL path.

    Args:
        path (str): The URL path

    Returns:
        str: The numeric part extracted from the path
    """
    # Split the path and find the numeric part
    parts = path.split('/')
    for part in parts:
        if part.isdigit():  # Check if the part is numeric
            return part
    return "unknown"  # Fallback if no numeric part is found


def sanitize_filename(filename):
    """
    Sanitize the filename to ensure it is valid.

    Args:
        filename (str): Original filename

    Returns:
        str: Sanitized filename
    """
    # Split the path into directory and filename
    directory, file_name = os.path.split(filename)

    # Replace invalid characters in the filename with underscores
    sanitized_file_name = ''.join(c if c.isalnum() or c in (' ', '.', '_') else '_' for c in file_name)

    # Return the full path with the sanitized filename
    return os.path.join(directory, sanitized_file_name)

def generate_urls_to_file(base_url, start, end, output_file):
    """
    Generate a list of URLs with a range of numerical values and write them to a file.

    Args:
        base_url (str): The base URL to which the numeric values will be appended.
        start (int): The starting numeric value.
        end (int): The ending numeric value.
        output_file (str): The file to which the URLs will be written.
    """
    with open(output_file, 'w') as file:
        for num in range(start, end + 1):
            url = f"{base_url}/{num}"
            file.write(url + '\n')  # Write each URL to the file

# Example usage
base_url = "https://www.hudsonwi.gov/DocumentCenter/View"
start_value = 2500
end_value = 3500
output_file = "urls.txt"

#generate_urls_to_file(base_url, start_value, end_value, output_file)

#print(f"URLs have been written to {output_file}.")

def main():
    #download_pdfs('urls.txt', 'pdfs')
    search_string_in_pdfs('pdfs', 'hendel+')

main()