From 3425e2e479d046081a349f677b320bd80e56864e Mon Sep 17 00:00:00 2001 From: Alana Gabriele Amorim Silva Date: Wed, 8 Oct 2025 12:03:40 -0300 Subject: [PATCH 1/6] feat: Spider agora no vale --- spiders/__init__.py | 2 + spiders/agoraNoVale.py | 94 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 spiders/agoraNoVale.py diff --git a/spiders/__init__.py b/spiders/__init__.py index ecb5e1f..f84e7ec 100644 --- a/spiders/__init__.py +++ b/spiders/__init__.py @@ -12,6 +12,7 @@ from spiders.gazetaDoPovo import GazetaDoPovoSpider from spiders.maisGoias import MaisGoiasSpider from spiders.aliadosBrasil import AliadosBrasilSpider +from spiders.agoraNoVale import AgoraNoValeSpider __all__ = [ BaseSpider, @@ -28,4 +29,5 @@ GazetaDoPovoSpider, MaisGoiasSpider, AliadosBrasilSpider, + AgoraNoValeSpider, ] diff --git a/spiders/agoraNoVale.py b/spiders/agoraNoVale.py new file mode 100644 index 0000000..31a194a --- /dev/null +++ b/spiders/agoraNoVale.py @@ -0,0 +1,94 @@ +import re + +import scrapy + +from spiders.base import BaseSpider +from spiders.items import URLItem + + +class AgoraNoValeSpider(BaseSpider): + name = "agoranovalespider" + start_urls = ["https://agoranovale.com.br/"] + allowed_domains = ["agoranovale.com.br", "www.agoranovale.com.br"] + + custom_settings = { + **BaseSpider.custom_settings, + "COOKIES_ENABLED": True, + "DOWNLOAD_DELAY": 2, + "RANDOMIZE_DOWNLOAD_DELAY": True, + "DEFAULT_REQUEST_HEADERS": { + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "Accept-Language": "pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7", + "Accept-Encoding": "gzip, deflate, br", + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0 Safari/537.36", + }, + "CONCURRENT_REQUESTS": 4, + } + + BLACKLISTED_SECTIONS = { + + } + + def allow_url(self, url: str) -> bool: + if not url: + return False + + # Normaliza URL, remove fragmentos e query + url = url.split('#', 1)[0].split('?', 1)[0] + + # Considera apenas links do domínio agoranovale.com.br + if not re.match(r"https?://(www\.)?agoranovale\.com\.br/", url): + return False + + path = re.sub(r"https?://[^/]+", "", url) + segments_norm = [s.lower() for s in path.strip("/").split("/") if s] + + # Blacklist por prefixo em qualquer segmento + blacklist_prefixes = {p.lstrip('/').lower() for p in self.BLACKLISTED_SECTIONS} + for seg in segments_norm: + if any(seg.startswith(pref) for pref in blacklist_prefixes): + return False + + # Exige ao menos dois segmentos (seção + slug) + segments = [s for s in path.split('/') if s] + if len(segments) < 2: + return False + + slug = segments[-1] + + # Aceita slugs de notícia com ao menos 3 hífens + return slug.count('-') >= 3 + + def start_requests(self): + for url in self.start_urls: + yield scrapy.Request( + url, + callback=self.parse, + dont_filter=True, + meta={"dont_redirect": True, "handle_httpstatus_list": [403]}, + ) + + def parse(self, response): + seen = set() + selectors = [ + "a[href*='agoranovale.com.br/']", + "a[href^='/']", + ] + + for sel in selectors: + for entry in response.css(sel): + href = entry.attrib.get("href") + if not href: + continue + + url = response.urljoin(href).split('?', 1)[0].split('#', 1)[0] + if url in seen: + continue + + if self.allow_url(url): + seen.add(url) + yield URLItem(url=url) + + self.logger.info(f"Collected {len(seen)} unique news URLs from Agora no Vale") From 56ace042127416f62fd6e17b71d29fbcaa9024fa Mon Sep 17 00:00:00 2001 From: Alana Gabriele Amorim Silva Date: Wed, 8 Oct 2025 13:34:08 -0300 Subject: [PATCH 2/6] fix: Spider agora no vale --- spiders/agoraNoVale.py | 70 ++++++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 30 deletions(-) diff --git a/spiders/agoraNoVale.py b/spiders/agoraNoVale.py index 31a194a..7a17427 100644 --- a/spiders/agoraNoVale.py +++ b/spiders/agoraNoVale.py @@ -8,58 +8,61 @@ class AgoraNoValeSpider(BaseSpider): name = "agoranovalespider" - start_urls = ["https://agoranovale.com.br/"] allowed_domains = ["agoranovale.com.br", "www.agoranovale.com.br"] + start_urls = ["https://agoranovale.com.br/"] custom_settings = { **BaseSpider.custom_settings, "COOKIES_ENABLED": True, - "DOWNLOAD_DELAY": 2, + "DOWNLOAD_DELAY": 3, "RANDOMIZE_DOWNLOAD_DELAY": True, "DEFAULT_REQUEST_HEADERS": { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", "Accept-Language": "pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7", - "Accept-Encoding": "gzip, deflate, br", "Connection": "keep-alive", "Upgrade-Insecure-Requests": "1", - "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0 Safari/537.36", + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/114.0 Safari/537.36" + ), + "Cache-Control": "max-age=0", }, "CONCURRENT_REQUESTS": 4, } BLACKLISTED_SECTIONS = { - + "/agoranacozinha", + } + + def _normalize(self, url: str) -> str: + return url.split("#", 1)[0].split("?", 1)[0] + def allow_url(self, url: str) -> bool: if not url: return False - # Normaliza URL, remove fragmentos e query - url = url.split('#', 1)[0].split('?', 1)[0] - - # Considera apenas links do domínio agoranovale.com.br - if not re.match(r"https?://(www\.)?agoranovale\.com\.br/", url): - return False - - path = re.sub(r"https?://[^/]+", "", url) - segments_norm = [s.lower() for s in path.strip("/").split("/") if s] + url = self._normalize(url) - # Blacklist por prefixo em qualquer segmento - blacklist_prefixes = {p.lstrip('/').lower() for p in self.BLACKLISTED_SECTIONS} - for seg in segments_norm: - if any(seg.startswith(pref) for pref in blacklist_prefixes): - return False - # Exige ao menos dois segmentos (seção + slug) - segments = [s for s in path.split('/') if s] + path = re.sub(r"^https?://[^/]+", "", url) + segments = [s for s in path.strip("/").split("/") if s] if len(segments) < 2: - return False + return False + + # blacklist por prefixo + bl = {p.lstrip("/").lower() for p in self.BLACKLISTED_SECTIONS} + for seg in (s.lower() for s in segments): + if any(seg.startswith(pref) for pref in bl): + return False - slug = segments[-1] + slug = segments[-1].lower() - # Aceita slugs de notícia com ao menos 3 hífens - return slug.count('-') >= 3 + + # heurística simples de notícia: slug com ao menos 3 hífens ou nome longo + return slug.count("-") >= 3 or len(slug) >= 20 def start_requests(self): for url in self.start_urls: @@ -72,18 +75,23 @@ def start_requests(self): def parse(self, response): seen = set() + selectors = [ - "a[href*='agoranovale.com.br/']", + "a[href*='agoranovale.com.br']", "a[href^='/']", + "article a[href]", + "main a[href]", + "h1 a[href], h2 a[href], h3 a[href], .entry-title a[href], .post a[href]", + "link[rel='canonical'][href]", ] for sel in selectors: - for entry in response.css(sel): - href = entry.attrib.get("href") + for node in response.css(sel): + href = node.attrib.get("href") if not href: continue - url = response.urljoin(href).split('?', 1)[0].split('#', 1)[0] + url = self._normalize(response.urljoin(href)) if url in seen: continue @@ -91,4 +99,6 @@ def parse(self, response): seen.add(url) yield URLItem(url=url) - self.logger.info(f"Collected {len(seen)} unique news URLs from Agora no Vale") + self.logger.info( + f"[AgoraNoVale] Coletadas {len(seen)} URLs únicas de notícia em {response.url}" + ) From 9ef81457b471460542792c64290a8e280596c686 Mon Sep 17 00:00:00 2001 From: Alana Gabriele Amorim Silva Date: Wed, 8 Oct 2025 13:46:55 -0300 Subject: [PATCH 3/6] feat: Scrape agora no vale --- plays/agoraNoVale.py | 158 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 plays/agoraNoVale.py diff --git a/plays/agoraNoVale.py b/plays/agoraNoVale.py new file mode 100644 index 0000000..aa69c58 --- /dev/null +++ b/plays/agoraNoVale.py @@ -0,0 +1,158 @@ +from playwright.sync_api import sync_playwright + +from plays.base import BasePlay +from plays.items import EntryItem +from plog import logger + + +class AgoraNoValePlay(BasePlay): + name = "agoranovale" + + @classmethod + def match(cls, url): + return "agoranovale.com.br" in url + + def pre_run(self): + pass + + def run(self) -> EntryItem: + with sync_playwright() as p: + browser = self.launch_browser(p, viewport={"width": 1600, "height": 1000}) + page = browser.new_page() + + # Bloquear recursos pesados/ads para acelerar + blocked_domains = ( + "doubleclick.net", + "googlesyndication.com", + "google-analytics.com", + "analytics.google.com", + "clarity.ms", + "tailtarget.com", + "flashtalking.com", + "googletagmanager.com", + ) + + def block_ads(route, request): + try: + if request.resource_type in {"image", "media", "font", "stylesheet"}: + return route.abort() + if any(d in request.url for d in blocked_domains): + return route.abort() + except Exception: + pass + return route.continue_() + + page.route("**/*", block_ads) + page.set_default_navigation_timeout(60_000) + page.set_default_timeout(15_000) + + logger.info(f"[{self.name}] Opening URL '{self.url}'...") + page.goto(self.url, timeout=60_000, wait_until="domcontentloaded") + + # Sincroniza com o conteúdo principal sem esperar recursos de terceiros + try: + page.wait_for_selector("h1", timeout=15_000) + except Exception: + pass + + # Título + title = "" + for selector in [ + "h1.entry-title", + ".post-title h1", + "article header h1", + "h1", + ]: + try: + el = page.locator(selector) + if el.count() > 0: + title = el.first.inner_text().strip() + if title: + logger.info(f"[{self.name}] Title from '{selector}'") + break + except Exception: + continue + if not title: + try: + title = page.title().strip() + except Exception: + title = "" + + # Se a página retornou um placeholder "Ops!", a notícia não existe mais + if title.strip().lower() == "ops!": + logger.warning(f"[{self.name}] Página 'Ops!' detectada. Ignorando URL {self.url}") + raise Exception("Notícia indisponível ('Ops!')") + + # Descrição + description = "" + for selector in [ + ".entry-subtitle", + "header .subtitle", + + ]: + try: + el = page.locator(selector) + if el.count() > 0: + description = el.first.inner_text().strip() + if description: + logger.info(f"[{self.name}] Description from '{selector}'") + break + except Exception: + continue + + + # Corpo da notícia + body = "" + for selector in [ + "article .entry-content", + "article .content p", + "article p", + "div[itemprop='articleBody']", + ]: + try: + els = page.locator(selector) + count = els.count() + if count > 0: + if count > 1: + parts = [] + for i in range(count): + t = els.nth(i).inner_text().strip() + if t: + parts.append(t) + body = "\n\n".join(parts).strip() + else: + body = els.first.inner_text().strip() + if body and len(body) > 100: + logger.info(f"[{self.name}] Body from '{selector}' (len={len(body)})") + break + except Exception: + continue + + # Tags + tags = [] + for selector in [ + ".post-tags a", + ".tags a", + ".entry-tags a", + "ul.tags li a", + ]: + try: + els = page.locator(selector) + if els.count() > 0: + for i in range(els.count()): + t = els.nth(i).inner_text().strip() + if t and t not in tags: + tags.append(t) + if tags: + logger.info(f"[{self.name}] {len(tags)} tags from '{selector}'") + break + except Exception: + continue + + return EntryItem( + title=title, + url=self.url, + description=description, + body=body, + tags=tags, + ) From d3f2175e0d17a071beb9192fabcb82efedc39dd2 Mon Sep 17 00:00:00 2001 From: Alana Gabriele Amorim Silva Date: Wed, 8 Oct 2025 13:49:45 -0300 Subject: [PATCH 4/6] feat: Adicionando comandos no makefile do agora no vale --- Makefile | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 12663c4..1118b8d 100644 --- a/Makefile +++ b/Makefile @@ -82,6 +82,9 @@ scrape_uol: scrape_folha: docker compose exec scraper python scrape_no_openai.py --platform folha.uol.com.br + +scrape_agoranovale: + docker compose exec scraper python scrape_no_openai.py --platform agoranovale.com.br # Crawler para todos os portais ou específicos crawl: @@ -112,6 +115,9 @@ crawl_ig: crawl_folha: docker compose run scraper python crawl.py folhaspider +crawl_agoranovale: + docker compose run scraper python crawl.py agoranovalespider + # Workflow completo de coleta de URLs crawl_all_working: @echo "Executando crawl de todos os portais funcionais..." @@ -182,6 +188,7 @@ help: @echo " make crawl_aliadosBrasil - Coleta URLs do portal AliadosBrasil" @echo " make crawl_ig - Coleta URLs do portal IG" @echo " make crawl_folha - Coleta URLs do portal Folha" + @echo " make crawl_agoranovale - Coleta URLs do portal Agora no Vale" @echo "" @echo "=== COMANDOS DE SCRAPING (Extração de Anúncios) ===" @echo " make scrape_all_working - Executa scraping de todos os portais funcionais" @@ -193,6 +200,7 @@ help: @echo " make scrape_r7 - Scraping do portal R7" @echo " make scrape_uol - Scraping do portal UOL" @echo " make scrape_folha - Scraping do portal Folha" + @echo " make scrape_agoranovale - Scraping do portal Agora no Vale" @echo "" @echo "=== WORKFLOWS COMPLETOS ===" @echo " make pipeline_complete - Executa crawl + scraping de todos os portais" @@ -240,4 +248,3 @@ all: # Parar todos os serviços down: docker-compose down - From c83516196f9d6a86c54c64a828247e5ce8ff53f2 Mon Sep 17 00:00:00 2001 From: Alana Gabriele Amorim Silva Date: Wed, 8 Oct 2025 13:50:31 -0300 Subject: [PATCH 5/6] feat: adicionando portal agora no vale em create_db --- create_db.py | 10 ++-------- plays/__init__.py | 2 ++ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/create_db.py b/create_db.py index 617b85b..1792f70 100644 --- a/create_db.py +++ b/create_db.py @@ -2,14 +2,7 @@ from sqlalchemy import create_engine from sqlalchemy.orm import Session -from models import ( - Advertisement, - Portal, - Entry, - QueueStatus, - URLQueue, -) - +from models import Advertisement, Entry, Portal, QueueStatus, URLQueue if __name__ == "__main__": engine = create_engine(config("DATABASE_URL")) @@ -42,6 +35,7 @@ ("Gazeta do Povo", "https://www.gazetadopovo.com.br/", "gazetadopovo"), ("Mais Goiás", "https://www.maisgoias.com.br/", "maisgoias"), ("Aliados Brasil", "https://www.aliadosbrasiloficial.com.br/", "aliadosbrasil"), + ("Agora no Vale", "https://www.agoranovale.com.br/", "agoranovale"), ] portals_to_add = [] for portal in portals: diff --git a/plays/__init__.py b/plays/__init__.py index 338fe28..ae99645 100644 --- a/plays/__init__.py +++ b/plays/__init__.py @@ -11,6 +11,7 @@ from plays.gazetaDoPovo import GazetaDoPovoPlay from plays.maisGoias import MaisGoias from plays.aliadosBrasil import AliadosBrasilPlay +from plays.agoraNoVale import AgoraNoValePlay __all__ = [ ClicRBSPlay, @@ -26,4 +27,5 @@ GazetaDoPovoPlay, MaisGoias, AliadosBrasilPlay, + AgoraNoValePlay, ] From c1dafa98e8f6c3def30c730e53ca247a7ddb6f9f Mon Sep 17 00:00:00 2001 From: Alana Gabriele Amorim Silva Date: Wed, 8 Oct 2025 14:04:19 -0300 Subject: [PATCH 6/6] fix: Scrape agora no vale --- plays/agoraNoVale.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plays/agoraNoVale.py b/plays/agoraNoVale.py index aa69c58..82ff632 100644 --- a/plays/agoraNoVale.py +++ b/plays/agoraNoVale.py @@ -108,6 +108,9 @@ def block_ads(route, request): "article .content p", "article p", "div[itemprop='articleBody']", + "article .post-content", + "article .post-content p .post-content h1, .post-content h2, .post-content h3, .post-content ol, .post-content ul", + ]: try: els = page.locator(selector)