feat(Agencia Gov): Criação do spider e scraper do portal Agência Gov

Makefile

@@ -82,6 +82,9 @@ scrape_uol:
 
 scrape_folha:
 	docker compose exec scraper python scrape_no_openai.py --platform folha.uol.com.br
+
+scrape_agenciagov:
+	docker compose exec scraper python scrape_no_openai.py --platform agenciagov.ebc.com.br
 
 # Crawler para todos os portais ou específicos
 crawl:
@@ -112,6 +115,9 @@ crawl_ig:
 crawl_folha:
 	docker compose run scraper python crawl.py folhaspider
 
+crawl_agenciagov:
+	docker compose run scraper python crawl.py agenciagovspider
+
 # Workflow completo de coleta de URLs
 crawl_all_working:
 	@echo "Executando crawl de todos os portais funcionais..."
@@ -123,6 +129,7 @@ crawl_all_working:
 	@make crawl_aliadosBrasil
 	@make crawl_ig
 	@make crawl_folha
+	@make crawl_agenciagov
 	@echo "Crawl de todos os portais concluído!"
 
 # Workflow completo de scraping
@@ -136,6 +143,7 @@ scrape_all_working:
 	@make scrape_r7
 	@make scrape_uol
 	@make scrape_folha
+	@make scrape_agenciagov
 	@echo "Scraping de todos os portais concluído!"
 
 # Pipeline completo: crawl + scrape
@@ -182,6 +190,7 @@ help:
 	@echo "  make crawl_aliadosBrasil - Coleta URLs do portal AliadosBrasil"
 	@echo "  make crawl_ig          - Coleta URLs do portal IG"
 	@echo "  make crawl_folha       - Coleta URLs do portal Folha"
+	@echo "  make crawl_agenciagov    - Coleta URLs do portal Agência Gov"
 	@echo ""
 	@echo "=== COMANDOS DE SCRAPING (Extração de Anúncios) ==="
 	@echo "  make scrape_all_working - Executa scraping de todos os portais funcionais"
@@ -193,6 +202,7 @@ help:
 	@echo "  make scrape_r7          - Scraping do portal R7"
 	@echo "  make scrape_uol         - Scraping do portal UOL"
 	@echo "  make scrape_folha       - Scraping do portal Folha"
+	@echo "  make scrape_agenciagov    - Scraping do portal Agência Gov"
 	@echo ""
 	@echo "=== WORKFLOWS COMPLETOS ==="
 	@echo "  make pipeline_complete  - Executa crawl + scraping de todos os portais"

create_db.py

@@ -42,6 +42,7 @@
         ("Gazeta do Povo", "https://www.gazetadopovo.com.br/", "gazetadopovo"),
         ("Mais Goiás", "https://www.maisgoias.com.br/", "maisgoias"),
         ("Aliados Brasil", "https://www.aliadosbrasiloficial.com.br/", "aliadosbrasil"),
+        ("Agência Gov", "https://agenciagov.ebc.com.br/", "agenciagov"),
     ]
     portals_to_add = []
     for portal in portals:

plays/__init__.py

@@ -11,6 +11,7 @@
 from plays.gazetaDoPovo import GazetaDoPovoPlay
 from plays.maisGoias import MaisGoias
 from plays.aliadosBrasil import AliadosBrasilPlay
+from plays.agenciaGov import AgenciaGovPlay
 
 __all__ = [
     ClicRBSPlay,
@@ -26,4 +27,5 @@
     GazetaDoPovoPlay,
     MaisGoias,
     AliadosBrasilPlay,
+    AgenciaGovPlay,
 ]

plays/agenciaGov.py

@@ -0,0 +1,77 @@
+import time
+
+from playwright.sync_api import sync_playwright
+
+from plays.base import BasePlay
+from plays.items import EntryItem
+from plog import logger
+
+
+class AgenciaGovPlay(BasePlay):
+    name = "agenciagov"
+
+    @classmethod
+    def match(cls, url):
+        return "agenciagov.ebc.com.br" in url
+
+    def pre_run(self):
+        pass
+
+    def run(self) -> EntryItem:
+        with sync_playwright() as p:
+            browser = self.launch_browser(p, viewport={"width": 1920, "height": 1080})
+            page = browser.new_page()
+            logger.info(f"[{self.name}] Opening URL '{self.url}'...")
+            page.goto(self.url, timeout=180_000)
+
+            # Extract article title
+            entry_title = ""
+            try:
+                entry_title = page.locator("h1").first.inner_text()
+            except Exception as e:
+                logger.warning(f"[{self.name}] Failed to extract title: {str(e)}")
+
+            # Extract description/subtitle
+            description = ""
+            try:
+                description = page.locator(
+                    "p.resumo-noticia-conteudo"
+                ).first.inner_text()
+            except Exception as e:
+                logger.warning(f"[{self.name}] Failed to extract description: {str(e)}")
+
+            # Extract article body
+            body = ""
+            try:
+                body_element = page.locator("div.texto-conteudo")
+                if body_element.count() > 0:
+                    page.evaluate(
+                        "document.querySelectorAll('p.callout').forEach(el => el.remove())"
+                    )
+                    body = body_element.first.inner_text().strip()
+                else:
+                    logger.warning(f"[{self.name}] Could not find body element")
+            except Exception as e:
+                logger.warning(f"[{self.name}] Failed to extract body: {str(e)}")
+
+            # Extract tags/categories
+            tags = []
+            try:
+                tag_elements = page.locator("nav#category a.link-category")
+                for i in range(tag_elements.count()):
+                    tag_text = tag_elements.nth(i).inner_text().strip()
+                    if tag_text:
+                        tags.append(tag_text)
+            except Exception as e:
+                logger.warning(f"[{self.name}] Failed to extract tags: {str(e)}")
+
+            return EntryItem(
+                title=entry_title,
+                url=self.url,
+                description=description,
+                body=body,
+                tags=tags,
+            )
+
+    def post_run(self, output):
+        return output

spiders/__init__.py

@@ -12,6 +12,7 @@
 from spiders.gazetaDoPovo import GazetaDoPovoSpider
 from spiders.maisGoias import MaisGoiasSpider
 from spiders.aliadosBrasil import AliadosBrasilSpider
+from spiders.agenciaGov import AgenciaGovSpider
 
 __all__ = [
     BaseSpider,
@@ -28,4 +29,5 @@
     GazetaDoPovoSpider,
     MaisGoiasSpider,
     AliadosBrasilSpider,
+    AgenciaGovSpider,
 ]

spiders/agenciaGov.py

@@ -0,0 +1,106 @@
+import scrapy
+
+from spiders.base import BaseSpider
+from spiders.items import URLItem
+from urllib.parse import urlparse
+
+
+class AgenciaGovSpider(BaseSpider):
+    name = "agenciagovspider"
+    start_urls = ["https://agenciagov.ebc.com.br/noticias"]
+    allowed_domains = ["agenciagov.ebc.com.br"]
+
+    custom_settings = {
+        **BaseSpider.custom_settings,
+        "COOKIES_ENABLED": True,
+        "DOWNLOAD_DELAY": 3,
+        "DEFAULT_REQUEST_HEADERS": {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+            "Accept-Language": "pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7",
+            "Accept-Encoding": "gzip, deflate, br",
+            "Connection": "keep-alive",
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+            "Cache-Control": "max-age=0",
+        },
+        "DOWNLOAD_HANDLERS": {
+            "https": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler",
+        },
+        "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+        "PLAYWRIGHT_BROWSER_TYPE": "firefox",
+    }
+
+    def allow_url(self, url: str) -> bool:
+        # Validate domain
+        if not url or "agenciagov.ebc.com.br" not in url:
+            return False
+
+        p = urlparse(url)
+        path = p.path.rstrip("/")
+
+        # Blacklist sections
+        blacklist = {
+            "/sobre",
+            "/ouvidoria",
+            "/distribuicao",
+            "/sitemap",
+            "/login",
+            "/RSS",
+            "/rss.xml",
+            "/atom.xml",
+            "/@@search",
+        }
+
+        if path in blacklist:
+            self.logger.info(f"Blacklisted URL: {url}")
+            return False
+
+        # Validate news patterns
+        if "/noticias/202" not in path:
+            return False
+
+        # Validate segments
+        segments = [seg for seg in path.split("/") if seg]
+        if len(segments) < 3:
+            self.logger.info(f"URL with less than 3 segments: {url}")
+            return False
+
+        slug = segments[-1]
+        # Long slugs by hyphens or by character length
+        if slug.count("-") >= 3 or len(slug) > 30:
+            return True
+
+        return False
+
+    def start_requests(self):
+        for url in self.start_urls:
+            yield scrapy.Request(
+                url,
+                callback=self.parse,
+                dont_filter=True,
+                meta={
+                    "playwright": True,
+                    "playwright_include_page": True,
+                    "playwright_page_goto_kwargs": {"wait_until": "networkidle"},
+                },
+            )
+
+    def parse(self, response):
+        page = response.meta.get("playwright_page")
+        if page:
+            page.close()
+
+        # Extract all links from page
+        links = response.css("a::attr(href)").getall()
+        self.logger.info(f"Found {len(links)} links under homepage")
+
+        for raw in links:
+            if not raw:
+                continue
+
+            # Build absolute URL, then strip fragments/queries
+            full = response.urljoin(raw)
+            full = full.split("#", 1)[0].split("?", 1)[0]
+
+            if self.allow_url(full):
+                yield URLItem(url=full)