example-web-crawler/crawler.aro at main · arolang/example-web-crawler · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
(* ============================================================
   ARO Web Crawler - Crawl Logic

   Handles the CrawlPage event to fetch URLs, track crawled pages,
   and trigger link extraction.
   ============================================================ *)

(Crawl Page: CrawlPage Handler) {
    (* Typed event extraction - validates against CrawlPageEvent schema *)
    Extract the <event-data: CrawlPageEvent> from the <event>.

    Log "Crawling: ${<event-data: url>}" to the <console> when <env: DEBUG> == "1".

    (* Fetch the page *)
    Request the <response> from the <event-data: url>.
    Extract the <html> from the <response: body>.

    (* Single-parse mode: title + markdown + links from one SwiftSoup parse.
       Earlier version ran ParseHtml twice on the same HTML body. *)
    ParseHtml the <page: page> from the <html>.
    Extract the <title> from the <page: title>.
    Extract the <markdown-content> from the <page: markdown>.
    Extract the <link-list> from the <page: links>.

    (* Depth bookkeeping. <depth> is this page's own level (the seed is 1);
       any links found here live one level deeper. <max> is the configured
       limit (0 = unlimited). We keep descending only while unlimited or this
       page sits strictly above the limit — so --depth 1 saves the seed but
       extracts no links, --depth 2 extracts the seed's links then stops. *)
    Extract the <depth> from the <event-data: depth>.
    Extract the <max-depth> from the <event-data: max>.
    Compute the <child-depth> from <depth> + 1.
    Compute the <descend> from <max-depth> == 0 or <depth> < <max-depth>.

    Log "Depth ${<depth>}/${<max-depth>} — descend: ${<descend>}" to the <console> when <env: DEBUG> == "1".

    (* Save the markdown content to file *)
    Emit a <SavePage: event> with { url: <event-data: url>, title: <title>, content: <markdown-content>, base: <event-data: base> }.

    (* Hand the pre-extracted links to the normalize pipeline. Children carry
       <child-depth>; the emit is suppressed once we hit the depth limit. *)
    Emit a <ExtractLinks: event> with { url: <event-data: url>, links: <link-list>, base: <event-data: base>, depth: <child-depth>, max: <max-depth> } when <descend>.

    Return an <OK: status> for the <crawl>.
}