diff --git a/Makefile b/Makefile index 12663c4..7f81e4c 100644 --- a/Makefile +++ b/Makefile @@ -82,6 +82,9 @@ scrape_uol: scrape_folha: docker compose exec scraper python scrape_no_openai.py --platform folha.uol.com.br + +scrape_agenciagov: + docker compose exec scraper python scrape_no_openai.py --platform agenciagov.ebc.com.br # Crawler para todos os portais ou específicos crawl: @@ -112,6 +115,9 @@ crawl_ig: crawl_folha: docker compose run scraper python crawl.py folhaspider +crawl_agenciagov: + docker compose run scraper python crawl.py agenciagovspider + # Workflow completo de coleta de URLs crawl_all_working: @echo "Executando crawl de todos os portais funcionais..." @@ -123,6 +129,7 @@ crawl_all_working: @make crawl_aliadosBrasil @make crawl_ig @make crawl_folha + @make crawl_agenciagov @echo "Crawl de todos os portais concluído!" # Workflow completo de scraping @@ -136,6 +143,7 @@ scrape_all_working: @make scrape_r7 @make scrape_uol @make scrape_folha + @make scrape_agenciagov @echo "Scraping de todos os portais concluído!" # Pipeline completo: crawl + scrape @@ -182,6 +190,7 @@ help: @echo " make crawl_aliadosBrasil - Coleta URLs do portal AliadosBrasil" @echo " make crawl_ig - Coleta URLs do portal IG" @echo " make crawl_folha - Coleta URLs do portal Folha" + @echo " make crawl_agenciagov - Coleta URLs do portal Agência Gov" @echo "" @echo "=== COMANDOS DE SCRAPING (Extração de Anúncios) ===" @echo " make scrape_all_working - Executa scraping de todos os portais funcionais" @@ -193,6 +202,7 @@ help: @echo " make scrape_r7 - Scraping do portal R7" @echo " make scrape_uol - Scraping do portal UOL" @echo " make scrape_folha - Scraping do portal Folha" + @echo " make scrape_agenciagov - Scraping do portal Agência Gov" @echo "" @echo "=== WORKFLOWS COMPLETOS ===" @echo " make pipeline_complete - Executa crawl + scraping de todos os portais" diff --git a/create_db.py b/create_db.py index 617b85b..af7658e 100644 --- a/create_db.py +++ b/create_db.py @@ -42,6 +42,7 @@ ("Gazeta do Povo", "https://www.gazetadopovo.com.br/", "gazetadopovo"), ("Mais Goiás", "https://www.maisgoias.com.br/", "maisgoias"), ("Aliados Brasil", "https://www.aliadosbrasiloficial.com.br/", "aliadosbrasil"), + ("Agência Gov", "https://agenciagov.ebc.com.br/", "agenciagov"), ] portals_to_add = [] for portal in portals: diff --git a/plays/__init__.py b/plays/__init__.py index 338fe28..5520075 100644 --- a/plays/__init__.py +++ b/plays/__init__.py @@ -11,6 +11,7 @@ from plays.gazetaDoPovo import GazetaDoPovoPlay from plays.maisGoias import MaisGoias from plays.aliadosBrasil import AliadosBrasilPlay +from plays.agenciaGov import AgenciaGovPlay __all__ = [ ClicRBSPlay, @@ -26,4 +27,5 @@ GazetaDoPovoPlay, MaisGoias, AliadosBrasilPlay, + AgenciaGovPlay, ] diff --git a/plays/agenciaGov.py b/plays/agenciaGov.py new file mode 100644 index 0000000..57416a1 --- /dev/null +++ b/plays/agenciaGov.py @@ -0,0 +1,77 @@ +import time + +from playwright.sync_api import sync_playwright + +from plays.base import BasePlay +from plays.items import EntryItem +from plog import logger + + +class AgenciaGovPlay(BasePlay): + name = "agenciagov" + + @classmethod + def match(cls, url): + return "agenciagov.ebc.com.br" in url + + def pre_run(self): + pass + + def run(self) -> EntryItem: + with sync_playwright() as p: + browser = self.launch_browser(p, viewport={"width": 1920, "height": 1080}) + page = browser.new_page() + logger.info(f"[{self.name}] Opening URL '{self.url}'...") + page.goto(self.url, timeout=180_000) + + # Extract article title + entry_title = "" + try: + entry_title = page.locator("h1").first.inner_text() + except Exception as e: + logger.warning(f"[{self.name}] Failed to extract title: {str(e)}") + + # Extract description/subtitle + description = "" + try: + description = page.locator( + "p.resumo-noticia-conteudo" + ).first.inner_text() + except Exception as e: + logger.warning(f"[{self.name}] Failed to extract description: {str(e)}") + + # Extract article body + body = "" + try: + body_element = page.locator("div.texto-conteudo") + if body_element.count() > 0: + page.evaluate( + "document.querySelectorAll('p.callout').forEach(el => el.remove())" + ) + body = body_element.first.inner_text().strip() + else: + logger.warning(f"[{self.name}] Could not find body element") + except Exception as e: + logger.warning(f"[{self.name}] Failed to extract body: {str(e)}") + + # Extract tags/categories + tags = [] + try: + tag_elements = page.locator("nav#category a.link-category") + for i in range(tag_elements.count()): + tag_text = tag_elements.nth(i).inner_text().strip() + if tag_text: + tags.append(tag_text) + except Exception as e: + logger.warning(f"[{self.name}] Failed to extract tags: {str(e)}") + + return EntryItem( + title=entry_title, + url=self.url, + description=description, + body=body, + tags=tags, + ) + + def post_run(self, output): + return output diff --git a/spiders/__init__.py b/spiders/__init__.py index ecb5e1f..69c9eea 100644 --- a/spiders/__init__.py +++ b/spiders/__init__.py @@ -12,6 +12,7 @@ from spiders.gazetaDoPovo import GazetaDoPovoSpider from spiders.maisGoias import MaisGoiasSpider from spiders.aliadosBrasil import AliadosBrasilSpider +from spiders.agenciaGov import AgenciaGovSpider __all__ = [ BaseSpider, @@ -28,4 +29,5 @@ GazetaDoPovoSpider, MaisGoiasSpider, AliadosBrasilSpider, + AgenciaGovSpider, ] diff --git a/spiders/agenciaGov.py b/spiders/agenciaGov.py new file mode 100644 index 0000000..3bb8e48 --- /dev/null +++ b/spiders/agenciaGov.py @@ -0,0 +1,106 @@ +import scrapy + +from spiders.base import BaseSpider +from spiders.items import URLItem +from urllib.parse import urlparse + + +class AgenciaGovSpider(BaseSpider): + name = "agenciagovspider" + start_urls = ["https://agenciagov.ebc.com.br/noticias"] + allowed_domains = ["agenciagov.ebc.com.br"] + + custom_settings = { + **BaseSpider.custom_settings, + "COOKIES_ENABLED": True, + "DOWNLOAD_DELAY": 3, + "DEFAULT_REQUEST_HEADERS": { + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "Accept-Language": "pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7", + "Accept-Encoding": "gzip, deflate, br", + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36", + "Cache-Control": "max-age=0", + }, + "DOWNLOAD_HANDLERS": { + "https": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler", + }, + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "PLAYWRIGHT_BROWSER_TYPE": "firefox", + } + + def allow_url(self, url: str) -> bool: + # Validate domain + if not url or "agenciagov.ebc.com.br" not in url: + return False + + p = urlparse(url) + path = p.path.rstrip("/") + + # Blacklist sections + blacklist = { + "/sobre", + "/ouvidoria", + "/distribuicao", + "/sitemap", + "/login", + "/RSS", + "/rss.xml", + "/atom.xml", + "/@@search", + } + + if path in blacklist: + self.logger.info(f"Blacklisted URL: {url}") + return False + + # Validate news patterns + if "/noticias/202" not in path: + return False + + # Validate segments + segments = [seg for seg in path.split("/") if seg] + if len(segments) < 3: + self.logger.info(f"URL with less than 3 segments: {url}") + return False + + slug = segments[-1] + # Long slugs by hyphens or by character length + if slug.count("-") >= 3 or len(slug) > 30: + return True + + return False + + def start_requests(self): + for url in self.start_urls: + yield scrapy.Request( + url, + callback=self.parse, + dont_filter=True, + meta={ + "playwright": True, + "playwright_include_page": True, + "playwright_page_goto_kwargs": {"wait_until": "networkidle"}, + }, + ) + + def parse(self, response): + page = response.meta.get("playwright_page") + if page: + page.close() + + # Extract all links from page + links = response.css("a::attr(href)").getall() + self.logger.info(f"Found {len(links)} links under homepage") + + for raw in links: + if not raw: + continue + + # Build absolute URL, then strip fragments/queries + full = response.urljoin(raw) + full = full.split("#", 1)[0].split("?", 1)[0] + + if self.allow_url(full): + yield URLItem(url=full)