example-web-crawler/openapi.yaml at main · arolang/example-web-crawler · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
openapi: 3.0.3
info:
  title: ARO Crawler Events
  version: 1.0.0
  description: Event schemas for the ARO web crawler application

# No HTTP paths - this is an event-driven application
paths: {}

components:
  schemas:
    # CrawlPage event: triggers page fetching
    CrawlPageEvent:
      type: object
      required:
        - url
        - base
        - depth
        - max
      properties:
        url:
          type: string
          description: The URL to crawl
        base:
          type: string
          description: The base domain for filtering
        depth:
          type: integer
          description: This page's crawl depth (seed = 1)
        max:
          type: integer
          description: Maximum crawl depth (0 = unlimited)

    # SavePage event: triggers file storage
    SavePageEvent:
      type: object
      required:
        - url
        - title
        - content
      properties:
        url:
          type: string
          description: The page URL
        title:
          type: string
          description: The page title
        content:
          type: string
          description: The markdown content
        base:
          type: string
          description: The base domain

    # ExtractLinks event: fans out pre-extracted links into NormalizeUrl
    # NOTE: payload carries the already-parsed link list, NOT the HTML body.
    # Passing the full HTML used to keep the page (~500 KB) alive in every
    # in-flight handler — for a site with thousands of pages that dominated
    # heap. Parsing happens once inside CrawlPage Handler.
    ExtractLinksEvent:
      type: object
      required:
        - url
        - links
        - depth
        - max
      properties:
        url:
          type: string
          description: The source page URL
        links:
          type: array
          items:
            type: string
          description: Pre-extracted href values from the page
        base:
          type: string
          description: The base domain for filtering
        depth:
          type: integer
          description: Depth to assign the extracted links (parent depth + 1)
        max:
          type: integer
          description: Maximum crawl depth (0 = unlimited)

    # NormalizeUrl event: triggers URL normalization
    NormalizeUrlEvent:
      type: object
      required:
        - raw
        - base
        - depth
        - max
      properties:
        raw:
          type: string
          description: The raw href value
        base:
          type: string
          description: The base domain
        depth:
          type: integer
          description: Depth of the link being normalized
        max:
          type: integer
          description: Maximum crawl depth (0 = unlimited)

    # FilterUrl event: triggers URL filtering
    FilterUrlEvent:
      type: object
      required:
        - url
        - base
        - depth
        - max
      properties:
        url:
          type: string
          description: The normalized URL
        base:
          type: string
          description: The base domain for filtering
        depth:
          type: integer
          description: Depth of the URL being filtered
        max:
          type: integer
          description: Maximum crawl depth (0 = unlimited)

    # QueueUrl event: triggers URL queuing
    QueueUrlEvent:
      type: object
      required:
        - url
        - base
        - depth
        - max
      properties:
        url:
          type: string
          description: The URL to queue
        base:
          type: string
          description: The base domain
        depth:
          type: integer
          description: Depth of the URL being queued
        max:
          type: integer
          description: Maximum crawl depth (0 = unlimited)

    # CrawlRequest: stored in repository
    CrawlRequest:
      type: object
      required:
        - id
        - url
        - base
        - depth
        - max
      properties:
        id:
          type: string
          description: Hash of URL for deduplication
        url:
          type: string
          description: The URL to crawl
        base:
          type: string
          description: The base domain
        depth:
          type: integer
          description: Depth of this URL (seed = 1)
        max:
          type: integer
          description: Maximum crawl depth (0 = unlimited)