main.aro

(* ============================================================
   ARO Web Crawler - Application Entry Point

   Reads the --url command-line parameter, seeds the queue, and
   runs the drain loop that fires CrawlPage events at the top of
   the call stack — keeping the runtime task pool from saturating
   on large sites.
   ============================================================ *)

(Application-Start: Web Crawler) {
    Log "Starting Web Crawler..." to the <console>.

    Extract the <start-url> from the <parameter: url>.
    Log "Starting URL: ${<start-url>}" to the <console>.

    (* Resolve the optional --depth parameter into a max-depth sentinel.
       0 = unlimited (crawl every reachable sub-page, the default). --depth 1
       crawls the seed page alone; --depth 2 the seed plus the links found on
       it; and so on. Unset / non-positive / non-numeric all fall through to 0,
       mirroring the MAX_URL_LENGTH convention.

       We read the whole <parameter> dict rather than <parameter: depth>
       directly: the runtime raises a hard error for an absent *named*
       parameter, but a missing key on the dict just falls through to the dict
       itself — which the numeric case below rejects, landing on 0. *)
    Extract the <all-params> from the <parameter>.
    Extract the <depth-raw: depth> from the <all-params>.
    (* Stringify before matching: `match` only runs its regex cases against a
       string. A supplied --depth is an Int; an absent one leaves <depth-raw>
       as the whole params dict — both render to a string the cases can test,
       and only a bare positive integer reaches the numeric branch. *)
    Create the <depth-str> with "${<depth-raw>}".
    match <depth-str> {
        case /^[1-9][0-9]*$/ {
            Transform the <max-depth: int> from the <depth-str>.
        }
        case /./ {
            Create the <max-depth> with 0.
        }
        case /^$/ {
            Create the <max-depth> with 0.
        }
    }
    Log "Crawl depth limit: ${<max-depth>} (0 = unlimited)" to the <console>.

    Create the <output-path> with "./output".
    Make the <output-dir> to the <directory: output-path>.
    Log "Output directory created" to the <console>.

    (* Seed the queue. QueueUrl Handler stores into crawled-repository;
       the crawled-repository Observer mirrors the new entry into
       pending-repository, where the drain loop will pick it up. *)
    Emit a <QueueUrl: event> with { url: <start-url>, base: <start-url>, depth: 1, max: <max-depth> }.

    (* Drain loop --------------------------------------------------
       ARO has no `while`, so we iterate a counted range. Each pass
       snapshots pending-repository, clears it, then emits CrawlPage
       for every snapshotted request. New URLs discovered while those
       CrawlPages run land in pending-repository and are picked up on
       the next pass. When the site is fully crawled, remaining passes
       see an empty snapshot and no-op.

       This loop runs at the *top* of the call stack — Emit CrawlPage
       returns before the next Emit fires, so the synchronous-Emit
       recursion that used to chain Observer → CrawlPage → ExtractLinks
       → ... → QueueUrl → Observer → CrawlPage no longer pins the
       executor pool.

       MAX_DRAIN_ITERATIONS bounds total drain passes (default 1000 —
       any connected page graph deeper than this is almost certainly
       a crawler trap). *)
    Extract the <max-iters-str> from the <env: MAX_DRAIN_ITERATIONS>.
    match <max-iters-str> {
        case /^[1-9][0-9]*$/ {
            Transform the <max-iters: int> from the <max-iters-str>.
        }
        case /./ {
            Create the <max-iters> with 1000.
        }
        case /^$/ {
            Create the <max-iters> with 1000.
        }
    }

    for <pass> from 0 to <max-iters> {
        Retrieve the <pending-list> from the <pending-repository>.
        Delete the <drained> from the <pending-repository>.
        for each <req> in <pending-list> {
            Emit a <CrawlPage: event> with { url: <req: url>, base: <req: base>, depth: <req: depth>, max: <req: max> }.
        }
    }

    Return an <OK: status> for the <startup>.
}

(Application-End: Success) {
    Log "🥁 Web Crawler completed!" to the <console>.
    Log the <metrics: table> to the <console>.
    Return an <OK: status> for the <shutdown>.
}