diff --git a/Makefile b/Makefile index a7f7ce2..69e5bb9 100644 --- a/Makefile +++ b/Makefile @@ -88,6 +88,9 @@ scrape_uol: scrape_folha: docker compose exec scraper python scrape_no_openai.py --platform folha.uol.com.br +scrape_agoranovale: + docker compose exec scraper python scrape_no_openai.py --platform agoranovale.com.br + scrape_jornaldaparaiba: docker compose exec scraper python scrape_no_openai.py --platform jornaldaparaiba.com.br @@ -121,6 +124,9 @@ crawl_ig: crawl_folha: docker compose run scraper python crawl.py folhaspider +crawl_agoranovale: + docker compose run scraper python crawl.py agoranovalespider + crawl_jornaldaparaiba: docker compose run scraper python crawl.py jornaldaparaiba @@ -197,6 +203,7 @@ help: @echo " make crawl_aliadosBrasil - Coleta URLs do portal AliadosBrasil" @echo " make crawl_ig - Coleta URLs do portal IG" @echo " make crawl_folha - Coleta URLs do portal Folha" + @echo " make crawl_agoranovale - Coleta URLs do portal Agora no Vale" @echo "" @echo "=== COMANDOS DE SCRAPING (Extração de Anúncios) ===" @echo " make scrape_all_working - Executa scraping de todos os portais funcionais" @@ -208,6 +215,7 @@ help: @echo " make scrape_r7 - Scraping do portal R7" @echo " make scrape_uol - Scraping do portal UOL" @echo " make scrape_folha - Scraping do portal Folha" + @echo " make scrape_agoranovale - Scraping do portal Agora no Vale" @echo "" @echo "=== WORKFLOWS COMPLETOS ===" @echo " make pipeline_complete - Executa crawl + scraping de todos os portais" @@ -255,4 +263,4 @@ all: # Parar todos os serviços down: - docker compose down \ No newline at end of file + docker-compose down \ No newline at end of file diff --git a/create_db.py b/create_db.py index 7ce5701..b5dc343 100644 --- a/create_db.py +++ b/create_db.py @@ -2,14 +2,7 @@ from sqlalchemy import create_engine from sqlalchemy.orm import Session -from models import ( - Advertisement, - Portal, - Entry, - QueueStatus, - URLQueue, -) - +from models import Advertisement, Entry, Portal, QueueStatus, URLQueue if __name__ == "__main__": engine = create_engine(config("DATABASE_URL")) @@ -42,6 +35,7 @@ ("Gazeta do Povo", "https://www.gazetadopovo.com.br/", "gazetadopovo"), ("Mais Goiás", "https://www.maisgoias.com.br/", "maisgoias"), ("Aliados Brasil", "https://www.aliadosbrasiloficial.com.br/", "aliadosbrasil"), + ("Agora no Vale", "https://www.agoranovale.com.br/", "agoranovale"), ("Jornal da Paraíba", "https://jornaldaparaiba.com.br/", "jornaldaparaiba"), ] portals_to_add = [] diff --git a/plays/__init__.py b/plays/__init__.py index 8189816..8a702e9 100644 --- a/plays/__init__.py +++ b/plays/__init__.py @@ -11,6 +11,7 @@ from plays.gazetaDoPovo import GazetaDoPovoPlay from plays.maisGoias import MaisGoias from plays.aliadosBrasil import AliadosBrasilPlay +from plays.agoraNoVale import AgoraNoValePlay from plays.jornaldaparaiba import JornalDaParaibaPlay __all__ = [ @@ -27,5 +28,6 @@ GazetaDoPovoPlay, MaisGoias, AliadosBrasilPlay, + AgoraNoValePlay, JornalDaParaibaPlay, ] diff --git a/plays/agoraNoVale.py b/plays/agoraNoVale.py new file mode 100644 index 0000000..82ff632 --- /dev/null +++ b/plays/agoraNoVale.py @@ -0,0 +1,161 @@ +from playwright.sync_api import sync_playwright + +from plays.base import BasePlay +from plays.items import EntryItem +from plog import logger + + +class AgoraNoValePlay(BasePlay): + name = "agoranovale" + + @classmethod + def match(cls, url): + return "agoranovale.com.br" in url + + def pre_run(self): + pass + + def run(self) -> EntryItem: + with sync_playwright() as p: + browser = self.launch_browser(p, viewport={"width": 1600, "height": 1000}) + page = browser.new_page() + + # Bloquear recursos pesados/ads para acelerar + blocked_domains = ( + "doubleclick.net", + "googlesyndication.com", + "google-analytics.com", + "analytics.google.com", + "clarity.ms", + "tailtarget.com", + "flashtalking.com", + "googletagmanager.com", + ) + + def block_ads(route, request): + try: + if request.resource_type in {"image", "media", "font", "stylesheet"}: + return route.abort() + if any(d in request.url for d in blocked_domains): + return route.abort() + except Exception: + pass + return route.continue_() + + page.route("**/*", block_ads) + page.set_default_navigation_timeout(60_000) + page.set_default_timeout(15_000) + + logger.info(f"[{self.name}] Opening URL '{self.url}'...") + page.goto(self.url, timeout=60_000, wait_until="domcontentloaded") + + # Sincroniza com o conteúdo principal sem esperar recursos de terceiros + try: + page.wait_for_selector("h1", timeout=15_000) + except Exception: + pass + + # Título + title = "" + for selector in [ + "h1.entry-title", + ".post-title h1", + "article header h1", + "h1", + ]: + try: + el = page.locator(selector) + if el.count() > 0: + title = el.first.inner_text().strip() + if title: + logger.info(f"[{self.name}] Title from '{selector}'") + break + except Exception: + continue + if not title: + try: + title = page.title().strip() + except Exception: + title = "" + + # Se a página retornou um placeholder "Ops!", a notícia não existe mais + if title.strip().lower() == "ops!": + logger.warning(f"[{self.name}] Página 'Ops!' detectada. Ignorando URL {self.url}") + raise Exception("Notícia indisponível ('Ops!')") + + # Descrição + description = "" + for selector in [ + ".entry-subtitle", + "header .subtitle", + + ]: + try: + el = page.locator(selector) + if el.count() > 0: + description = el.first.inner_text().strip() + if description: + logger.info(f"[{self.name}] Description from '{selector}'") + break + except Exception: + continue + + + # Corpo da notícia + body = "" + for selector in [ + "article .entry-content", + "article .content p", + "article p", + "div[itemprop='articleBody']", + "article .post-content", + "article .post-content p .post-content h1, .post-content h2, .post-content h3, .post-content ol, .post-content ul", + + ]: + try: + els = page.locator(selector) + count = els.count() + if count > 0: + if count > 1: + parts = [] + for i in range(count): + t = els.nth(i).inner_text().strip() + if t: + parts.append(t) + body = "\n\n".join(parts).strip() + else: + body = els.first.inner_text().strip() + if body and len(body) > 100: + logger.info(f"[{self.name}] Body from '{selector}' (len={len(body)})") + break + except Exception: + continue + + # Tags + tags = [] + for selector in [ + ".post-tags a", + ".tags a", + ".entry-tags a", + "ul.tags li a", + ]: + try: + els = page.locator(selector) + if els.count() > 0: + for i in range(els.count()): + t = els.nth(i).inner_text().strip() + if t and t not in tags: + tags.append(t) + if tags: + logger.info(f"[{self.name}] {len(tags)} tags from '{selector}'") + break + except Exception: + continue + + return EntryItem( + title=title, + url=self.url, + description=description, + body=body, + tags=tags, + ) diff --git a/spiders/__init__.py b/spiders/__init__.py index d2bb555..693a09e 100644 --- a/spiders/__init__.py +++ b/spiders/__init__.py @@ -12,6 +12,7 @@ from spiders.gazetaDoPovo import GazetaDoPovoSpider from spiders.maisGoias import MaisGoiasSpider from spiders.aliadosBrasil import AliadosBrasilSpider +from spiders.agoraNoVale import AgoraNoValeSpider from spiders.jornaldaparaiba import JornalDaParaibaSpider __all__ = [ @@ -29,5 +30,6 @@ GazetaDoPovoSpider, MaisGoiasSpider, AliadosBrasilSpider, + AgoraNoValeSpider, JornalDaParaibaSpider, ] diff --git a/spiders/agoraNoVale.py b/spiders/agoraNoVale.py new file mode 100644 index 0000000..7a17427 --- /dev/null +++ b/spiders/agoraNoVale.py @@ -0,0 +1,104 @@ +import re + +import scrapy + +from spiders.base import BaseSpider +from spiders.items import URLItem + + +class AgoraNoValeSpider(BaseSpider): + name = "agoranovalespider" + allowed_domains = ["agoranovale.com.br", "www.agoranovale.com.br"] + start_urls = ["https://agoranovale.com.br/"] + + custom_settings = { + **BaseSpider.custom_settings, + "COOKIES_ENABLED": True, + "DOWNLOAD_DELAY": 3, + "RANDOMIZE_DOWNLOAD_DELAY": True, + "DEFAULT_REQUEST_HEADERS": { + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "Accept-Language": "pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7", + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/114.0 Safari/537.36" + ), + "Cache-Control": "max-age=0", + }, + "CONCURRENT_REQUESTS": 4, + } + + BLACKLISTED_SECTIONS = { + "/agoranacozinha", + + } + + + def _normalize(self, url: str) -> str: + return url.split("#", 1)[0].split("?", 1)[0] + + def allow_url(self, url: str) -> bool: + if not url: + return False + + url = self._normalize(url) + + + path = re.sub(r"^https?://[^/]+", "", url) + segments = [s for s in path.strip("/").split("/") if s] + if len(segments) < 2: + return False + + # blacklist por prefixo + bl = {p.lstrip("/").lower() for p in self.BLACKLISTED_SECTIONS} + for seg in (s.lower() for s in segments): + if any(seg.startswith(pref) for pref in bl): + return False + + slug = segments[-1].lower() + + + # heurística simples de notícia: slug com ao menos 3 hífens ou nome longo + return slug.count("-") >= 3 or len(slug) >= 20 + + def start_requests(self): + for url in self.start_urls: + yield scrapy.Request( + url, + callback=self.parse, + dont_filter=True, + meta={"dont_redirect": True, "handle_httpstatus_list": [403]}, + ) + + def parse(self, response): + seen = set() + + selectors = [ + "a[href*='agoranovale.com.br']", + "a[href^='/']", + "article a[href]", + "main a[href]", + "h1 a[href], h2 a[href], h3 a[href], .entry-title a[href], .post a[href]", + "link[rel='canonical'][href]", + ] + + for sel in selectors: + for node in response.css(sel): + href = node.attrib.get("href") + if not href: + continue + + url = self._normalize(response.urljoin(href)) + if url in seen: + continue + + if self.allow_url(url): + seen.add(url) + yield URLItem(url=url) + + self.logger.info( + f"[AgoraNoVale] Coletadas {len(seen)} URLs únicas de notícia em {response.url}" + )