feat(Agora no Vale): Criação do spider e scraper do portal de notícias Agora no Vale

Makefile

@@ -88,6 +88,9 @@ scrape_uol:
 scrape_folha:
 	docker compose exec scraper python scrape_no_openai.py --platform folha.uol.com.br
 
+scrape_agoranovale:
+	docker compose exec scraper python scrape_no_openai.py --platform agoranovale.com.br
+
 scrape_jornaldaparaiba:
 	docker compose exec scraper python scrape_no_openai.py --platform jornaldaparaiba.com.br
 
@@ -121,6 +124,9 @@ crawl_ig:
 crawl_folha:
 	docker compose run scraper python crawl.py folhaspider
 
+crawl_agoranovale:
+	docker compose run scraper python crawl.py agoranovalespider
+
 crawl_jornaldaparaiba:
 	docker compose run scraper python crawl.py jornaldaparaiba
 
@@ -197,6 +203,7 @@ help:
 	@echo "  make crawl_aliadosBrasil - Coleta URLs do portal AliadosBrasil"
 	@echo "  make crawl_ig          - Coleta URLs do portal IG"
 	@echo "  make crawl_folha       - Coleta URLs do portal Folha"
+	@echo "  make crawl_agoranovale - Coleta URLs do portal Agora no Vale"
 	@echo ""
 	@echo "=== COMANDOS DE SCRAPING (Extração de Anúncios) ==="
 	@echo "  make scrape_all_working - Executa scraping de todos os portais funcionais"
@@ -208,6 +215,7 @@ help:
 	@echo "  make scrape_r7          - Scraping do portal R7"
 	@echo "  make scrape_uol         - Scraping do portal UOL"
 	@echo "  make scrape_folha       - Scraping do portal Folha"
+	@echo "  make scrape_agoranovale - Scraping do portal Agora no Vale"
 	@echo ""
 	@echo "=== WORKFLOWS COMPLETOS ==="
 	@echo "  make pipeline_complete  - Executa crawl + scraping de todos os portais"
@@ -255,4 +263,4 @@ all:
 
 # Parar todos os serviços
 down:
-	docker compose down
+	docker-compose down

create_db.py

@@ -2,14 +2,7 @@
 from sqlalchemy import create_engine
 from sqlalchemy.orm import Session
 
-from models import (
-    Advertisement,
-    Portal,
-    Entry,
-    QueueStatus,
-    URLQueue,
-)
-
+from models import Advertisement, Entry, Portal, QueueStatus, URLQueue
 
 if __name__ == "__main__":
     engine = create_engine(config("DATABASE_URL"))
@@ -42,6 +35,7 @@
         ("Gazeta do Povo", "https://www.gazetadopovo.com.br/", "gazetadopovo"),
         ("Mais Goiás", "https://www.maisgoias.com.br/", "maisgoias"),
         ("Aliados Brasil", "https://www.aliadosbrasiloficial.com.br/", "aliadosbrasil"),
+        ("Agora no Vale", "https://www.agoranovale.com.br/", "agoranovale"),
         ("Jornal da Paraíba", "https://jornaldaparaiba.com.br/", "jornaldaparaiba"),
     ]
     portals_to_add = []

plays/__init__.py

@@ -11,6 +11,7 @@
 from plays.gazetaDoPovo import GazetaDoPovoPlay
 from plays.maisGoias import MaisGoias
 from plays.aliadosBrasil import AliadosBrasilPlay
+from plays.agoraNoVale import AgoraNoValePlay
 from plays.jornaldaparaiba import JornalDaParaibaPlay
 
 __all__ = [
@@ -27,5 +28,6 @@
     GazetaDoPovoPlay,
     MaisGoias,
     AliadosBrasilPlay,
+    AgoraNoValePlay,
     JornalDaParaibaPlay,
 ]

plays/agoraNoVale.py

@@ -0,0 +1,161 @@
+from playwright.sync_api import sync_playwright
+
+from plays.base import BasePlay
+from plays.items import EntryItem
+from plog import logger
+
+
+class AgoraNoValePlay(BasePlay):
+    name = "agoranovale"
+
+    @classmethod
+    def match(cls, url):
+        return "agoranovale.com.br" in url
+
+    def pre_run(self):
+        pass
+
+    def run(self) -> EntryItem:
+        with sync_playwright() as p:
+            browser = self.launch_browser(p, viewport={"width": 1600, "height": 1000})
+            page = browser.new_page()
+
+            # Bloquear recursos pesados/ads para acelerar
+            blocked_domains = (
+                "doubleclick.net",
+                "googlesyndication.com",
+                "google-analytics.com",
+                "analytics.google.com",
+                "clarity.ms",
+                "tailtarget.com",
+                "flashtalking.com",
+                "googletagmanager.com",
+            )
+
+            def block_ads(route, request):
+                try:
+                    if request.resource_type in {"image", "media", "font", "stylesheet"}:
+                        return route.abort()
+                    if any(d in request.url for d in blocked_domains):
+                        return route.abort()
+                except Exception:
+                    pass
+                return route.continue_()
+
+            page.route("**/*", block_ads)
+            page.set_default_navigation_timeout(60_000)
+            page.set_default_timeout(15_000)
+
+            logger.info(f"[{self.name}] Opening URL '{self.url}'...")
+            page.goto(self.url, timeout=60_000, wait_until="domcontentloaded")
+
+            # Sincroniza com o conteúdo principal sem esperar recursos de terceiros
+            try:
+                page.wait_for_selector("h1", timeout=15_000)
+            except Exception:
+                pass
+
+            # Título
+            title = ""
+            for selector in [
+                "h1.entry-title",
+                ".post-title h1",
+                "article header h1",
+                "h1",
+            ]:
+                try:
+                    el = page.locator(selector)
+                    if el.count() > 0:
+                        title = el.first.inner_text().strip()
+                        if title:
+                            logger.info(f"[{self.name}] Title from '{selector}'")
+                            break
+                except Exception:
+                    continue
+            if not title:
+                try:
+                    title = page.title().strip()
+                except Exception:
+                    title = ""
+
+            # Se a página retornou um placeholder "Ops!", a notícia não existe mais
+            if title.strip().lower() == "ops!":
+                logger.warning(f"[{self.name}] Página 'Ops!' detectada. Ignorando URL {self.url}")
+                raise Exception("Notícia indisponível ('Ops!')")
+
+            # Descrição
+            description = ""
+            for selector in [
+                ".entry-subtitle",
+                "header .subtitle",
+
+            ]:
+                try:
+                    el = page.locator(selector)
+                    if el.count() > 0:
+                        description = el.first.inner_text().strip()
+                        if description:
+                            logger.info(f"[{self.name}] Description from '{selector}'")
+                            break
+                except Exception:
+                    continue
+
+
+            # Corpo da notícia
+            body = ""
+            for selector in [
+                "article .entry-content",
+                "article .content p",
+                "article p",
+                "div[itemprop='articleBody']",
+                "article .post-content",
+                "article .post-content p .post-content h1, .post-content h2, .post-content h3, .post-content ol, .post-content ul",
+
+            ]:
+                try:
+                    els = page.locator(selector)
+                    count = els.count()
+                    if count > 0:
+                        if count > 1:
+                            parts = []
+                            for i in range(count):
+                                t = els.nth(i).inner_text().strip()
+                                if t:
+                                    parts.append(t)
+                            body = "\n\n".join(parts).strip()
+                        else:
+                            body = els.first.inner_text().strip()
+                        if body and len(body) > 100:
+                            logger.info(f"[{self.name}] Body from '{selector}' (len={len(body)})")
+                            break
+                except Exception:
+                    continue
+
+            # Tags
+            tags = []
+            for selector in [
+                ".post-tags a",
+                ".tags a",
+                ".entry-tags a",
+                "ul.tags li a",
+            ]:
+                try:
+                    els = page.locator(selector)
+                    if els.count() > 0:
+                        for i in range(els.count()):
+                            t = els.nth(i).inner_text().strip()
+                            if t and t not in tags:
+                                tags.append(t)
+                        if tags:
+                            logger.info(f"[{self.name}] {len(tags)} tags from '{selector}'")
+                            break
+                except Exception:
+                    continue
+
+            return EntryItem(
+                title=title,
+                url=self.url,
+                description=description,
+                body=body,
+                tags=tags,
+            )

spiders/__init__.py

@@ -12,6 +12,7 @@
 from spiders.gazetaDoPovo import GazetaDoPovoSpider
 from spiders.maisGoias import MaisGoiasSpider
 from spiders.aliadosBrasil import AliadosBrasilSpider
+from spiders.agoraNoVale import AgoraNoValeSpider
 from spiders.jornaldaparaiba import JornalDaParaibaSpider
 
 __all__ = [
@@ -29,5 +30,6 @@
     GazetaDoPovoSpider,
     MaisGoiasSpider,
     AliadosBrasilSpider,
+    AgoraNoValeSpider,
     JornalDaParaibaSpider,
 ]

spiders/agoraNoVale.py

@@ -0,0 +1,104 @@
+import re
+
+import scrapy
+
+from spiders.base import BaseSpider
+from spiders.items import URLItem
+
+
+class AgoraNoValeSpider(BaseSpider):
+    name = "agoranovalespider"
+    allowed_domains = ["agoranovale.com.br", "www.agoranovale.com.br"]
+    start_urls = ["https://agoranovale.com.br/"]
+
+    custom_settings = {
+        **BaseSpider.custom_settings,
+        "COOKIES_ENABLED": True,
+        "DOWNLOAD_DELAY": 3,
+        "RANDOMIZE_DOWNLOAD_DELAY": True,
+        "DEFAULT_REQUEST_HEADERS": {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+            "Accept-Language": "pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7",
+            "Connection": "keep-alive",
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": (
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                "AppleWebKit/537.36 (KHTML, like Gecko) "
+                "Chrome/114.0 Safari/537.36"
+            ),
+            "Cache-Control": "max-age=0",
+        },
+        "CONCURRENT_REQUESTS": 4,
+    }
+
+    BLACKLISTED_SECTIONS = {
+        "/agoranacozinha",
+
+    }
+
+
+    def _normalize(self, url: str) -> str:
+        return url.split("#", 1)[0].split("?", 1)[0]
+
+    def allow_url(self, url: str) -> bool:
+        if not url:
+            return False
+
+        url = self._normalize(url)
+
+
+        path = re.sub(r"^https?://[^/]+", "", url)
+        segments = [s for s in path.strip("/").split("/") if s]
+        if len(segments) < 2:
+            return False  
+
+        # blacklist por prefixo 
+        bl = {p.lstrip("/").lower() for p in self.BLACKLISTED_SECTIONS}
+        for seg in (s.lower() for s in segments):
+            if any(seg.startswith(pref) for pref in bl):
+                return False
+
+        slug = segments[-1].lower()
+
+
+        # heurística simples de notícia: slug com ao menos 3 hífens ou nome longo
+        return slug.count("-") >= 3 or len(slug) >= 20
+
+    def start_requests(self):
+        for url in self.start_urls:
+            yield scrapy.Request(
+                url,
+                callback=self.parse,
+                dont_filter=True,
+                meta={"dont_redirect": True, "handle_httpstatus_list": [403]},
+            )
+
+    def parse(self, response):
+        seen = set()
+
+        selectors = [
+            "a[href*='agoranovale.com.br']",
+            "a[href^='/']",
+            "article a[href]",
+            "main a[href]",
+            "h1 a[href], h2 a[href], h3 a[href], .entry-title a[href], .post a[href]",
+            "link[rel='canonical'][href]",
+        ]
+
+        for sel in selectors:
+            for node in response.css(sel):
+                href = node.attrib.get("href")
+                if not href:
+                    continue
+
+                url = self._normalize(response.urljoin(href))
+                if url in seen:
+                    continue
+
+                if self.allow_url(url):
+                    seen.add(url)
+                    yield URLItem(url=url)
+
+        self.logger.info(
+            f"[AgoraNoVale] Coletadas {len(seen)} URLs únicas de notícia em {response.url}"
+        )