main.py

import sys
from crawl import crawl_site_async
import asyncio
from json_report import write_json_report


async def main():
    if len(sys.argv) < 2:
        print("no website provided")
        sys.exit(1)
    elif len(sys.argv) > 4:
        print("too many arguments provided")
        sys.exit(1)
    else:
        BASE_URL = sys.argv[1]
        MAX_CONCURRENCY = int(sys.argv[2])
        MAX_PAGES = int(sys.argv[3])
        print("starting crawl of:", BASE_URL)
        page_data = await crawl_site_async(BASE_URL, MAX_CONCURRENCY, MAX_PAGES)
        print(f"There are {len(page_data)} pages that have been scraped.")
        write_json_report(page_data)
        # for page in page_data.values():
        #     if page is None:
        #         continue
        #     print(f"- {page['url']}: {len(page['outgoing_links'])} outgoing links")


if __name__ == "__main__":
    asyncio.run(main())