Crawler 101

[priyanshum143]

This PR contains the code for Crawler, as of now crawler is working fine without any issue.

[github-actions]

Remaining comments which cannot be posted as a review comment to avoid GitHub Rate Limit

black-format

---

[black-format] _{reported by reviewdog 🐶}

Search_Engine/src/search_engine/utils/string_utils.py

Lines 39 to 40 in 7cb35b2

	fragment = ''

---

[black-format] _{reported by reviewdog 🐶}

Search_Engine/src/search_engine/utils/string_utils.py

Line 44 in 7cb35b2

---

[black-format] _{reported by reviewdog 🐶}

Search_Engine/src/search_engine/utils/string_utils.py

Line 58 in 7cb35b2

return hashlib.sha256(content.encode('utf-8')).hexdigest()

---

[black-format] _{reported by reviewdog 🐶}

Search_Engine/src/search_engine/crawler.py

Line 33 in 7cb35b2

---

[black-format] _{reported by reviewdog 🐶}

Search_Engine/src/search_engine/crawler.py

Line 41 in 7cb35b2

timeout=30.0

---

[black-format] _{reported by reviewdog 🐶}

Search_Engine/src/search_engine/crawler.py

Line 48 in 7cb35b2

logger.debug(f"Initializing URL frontier with seed URLs: {CommonVariables.SEED_URLS}")

---

[black-format] _{reported by reviewdog 🐶}

Search_Engine/src/search_engine/crawler.py

Line 51 in 7cb35b2

---

[black-format] _{reported by reviewdog 🐶}

Search_Engine/src/search_engine/crawler.py

Line 74 in 7cb35b2

---

[black-format] _{reported by reviewdog 🐶}

Search_Engine/src/search_engine/crawler.py

Line 101 in 7cb35b2

logger.debug(f"Queue is full. Current size: {queue_size}, Max limit: {CommonVariables.MAX_LIMIT}")

---

[black-format] _{reported by reviewdog 🐶}

Search_Engine/src/search_engine/crawler.py

Line 119 in 7cb35b2

logger.debug(f"New queue size: {self.url_frontier.qsize()}/{CommonVariables.MAX_LIMIT}")

---

[black-format] _{reported by reviewdog 🐶}

Search_Engine/src/search_engine/crawler.py

Lines 123 to 125 in 7cb35b2

	logger.debug(f"Could not add {urls_len - urls_to_add} URLs due to queue capacity limit")
	async def _parse_response_and_make_page_model(self, responses: List[httpx.Response]) -> None:

---

[black-format] _{reported by reviewdog 🐶}

Search_Engine/src/search_engine/crawler.py

Line 129 in 7cb35b2

---

[black-format] _{reported by reviewdog 🐶}

Search_Engine/src/search_engine/crawler.py

Line 141 in 7cb35b2

with open(CommonVariables.JSONL_FILE_PATH, 'a', encoding='utf-8') as f:

---

[black-format] _{reported by reviewdog 🐶}

Search_Engine/src/search_engine/crawler.py

Line 152 in 7cb35b2

logger.debug(f"Status code for URL [{url}] is {response.status_code}\n content: {response.content}")

---

[black-format] _{reported by reviewdog 🐶}

Search_Engine/src/search_engine/crawler.py

Lines 156 to 158 in 7cb35b2

	content_type = response.headers.get('Content-Type', '').lower()
	if 'xml' in content_type:
	parser = 'xml'

---

[black-format] _{reported by reviewdog 🐶}

Search_Engine/src/search_engine/crawler.py

Line 160 in 7cb35b2

parser = 'html.parser'

---

[black-format] _{reported by reviewdog 🐶}

Search_Engine/src/search_engine/crawler.py

Line 177 in 7cb35b2

links=links

---

[black-format] _{reported by reviewdog 🐶}

Search_Engine/src/search_engine/crawler.py

Line 184 in 7cb35b2

f.write(json.dumps(asdict(page_model), ensure_ascii=False) + '\n')