diff --git a/Makefile b/Makefile index a7f7ce2..c4fb883 100644 --- a/Makefile +++ b/Makefile @@ -88,6 +88,9 @@ scrape_uol: scrape_folha: docker compose exec scraper python scrape_no_openai.py --platform folha.uol.com.br +scrape_globo: + docker compose exec scraper python scrape_no_openai.py --platform globo.com + scrape_jornaldaparaiba: docker compose exec scraper python scrape_no_openai.py --platform jornaldaparaiba.com.br @@ -121,6 +124,9 @@ crawl_ig: crawl_folha: docker compose run scraper python crawl.py folhaspider +crawl_globo: + docker compose run scraper python crawl.py globospider + crawl_jornaldaparaiba: docker compose run scraper python crawl.py jornaldaparaiba @@ -135,6 +141,7 @@ crawl_all_working: @make crawl_aliadosBrasil @make crawl_ig @make crawl_folha + @make crawl_globo @make crawl_jornaldaparaiba @echo "Crawl de todos os portais concluído!" @@ -149,6 +156,7 @@ scrape_all_working: @make scrape_r7 @make scrape_uol @make scrape_folha + @make scrape_globo @make scrape_jornaldaparaiba @echo "Scraping de todos os portais concluído!" @@ -197,6 +205,7 @@ help: @echo " make crawl_aliadosBrasil - Coleta URLs do portal AliadosBrasil" @echo " make crawl_ig - Coleta URLs do portal IG" @echo " make crawl_folha - Coleta URLs do portal Folha" + @echo " make crawl_globo - Coleta URLs do portal Globo" @echo "" @echo "=== COMANDOS DE SCRAPING (Extração de Anúncios) ===" @echo " make scrape_all_working - Executa scraping de todos os portais funcionais" @@ -208,6 +217,7 @@ help: @echo " make scrape_r7 - Scraping do portal R7" @echo " make scrape_uol - Scraping do portal UOL" @echo " make scrape_folha - Scraping do portal Folha" + @echo " make scrape_globo - Scraping do portal Globo" @echo "" @echo "=== WORKFLOWS COMPLETOS ===" @echo " make pipeline_complete - Executa crawl + scraping de todos os portais" diff --git a/plays/globo.py b/plays/globo.py index 4c74eb8..7ff445d 100644 --- a/plays/globo.py +++ b/plays/globo.py @@ -1,89 +1,149 @@ -import time - -from decouple import config -from playwright.sync_api import TimeoutError as PlayWrightTimeoutError, sync_playwright +from playwright.sync_api import sync_playwright from plays.base import BasePlay -from plays.items import AdItem, EntryItem -from plays.utils import get_or_none +from plays.items import EntryItem from plog import logger class GloboPlay(BasePlay): name = "globo" - n_expected_ads = 50 - - def login(self): - with sync_playwright() as p: - browser = self.launch_browser(p) - page = browser.new_page() - login_url = "https://login.globo.com/login/" - logger.info(f"Opening URL {login_url}...") - page.goto(login_url) - time.sleep(self.wait_time) - page.locator("#barra-item-login").click() - page.locator("#login").fill(config("GLOBO_USERNAME")) - page.locator("#password").fill(config("GLOBO_PASSWORD")) - page.locator("#login-button").click() @classmethod def match(cls, url): - return "oglobo.globo.com/" in url - - @classmethod - def extra_kwargs(cls): - return {"allow_remove_session": False} - - def find_items(self, html_content) -> AdItem: - return AdItem( - title=get_or_none(r'title="(.*?)"', html_content), - url=get_or_none(r'href="(.*?)"', html_content), - thumbnail_url=get_or_none(r'url\("(.*?)"\)', html_content), - tag=get_or_none(r'(.*?)<\/span>', html_content), - excerpt=get_or_none(r'slot="description" title="(.*?)"', html_content), - ) + return "oglobo.globo.com" in url def pre_run(self): pass - def is_logged_in(self, page): - try: - page.locator(".login-profile__menu").first.inner_html() - except PlayWrightTimeoutError: - raise Exception(f"[{self.name}] Not logged in!") - def run(self) -> EntryItem: with sync_playwright() as p: - browser = self.launch_browser(p) + browser = self.launch_browser(p, viewport={"width": 1600, "height": 1000}) page = browser.new_page() - logger.info(f"[{self.name}] Opening URL '{self.url}'...") - page.goto(self.url, timeout=180_000) - logger.info(f"[{self.name}] Searching for ads...") - self.is_logged_in(page) + # Bloquear recursos pesados/ads para acelerar + blocked_domains = ( + "doubleclick.net", + "googlesyndication.com", + "google-analytics.com", + "analytics.google.com", + "clarity.ms", + "tailtarget.com", + "flashtalking.com", + "googletagmanager.com", + ) + + def block_ads(route, request): + try: + if request.resource_type in {"image", "media", "font", "stylesheet"}: + return route.abort() + if any(d in request.url for d in blocked_domains): + return route.abort() + except Exception: + pass + return route.continue_() - page.locator(".tbl-feed-header-text").scroll_into_view_if_needed() - self.scroll_down(page, 40, 400, wait_time=1) - page.locator("#boxComentarios").scroll_into_view_if_needed() + page.route("**/*", block_ads) + page.set_default_navigation_timeout(60_000) + page.set_default_timeout(15_000) - entry_screenshot_path = self.take_screenshot(page, self.url, goto=False, timeout=60_000) - entry_title = page.locator("title").inner_text() + logger.info(f"[{self.name}] Opening URL '{self.url}'...") + page.goto(self.url, timeout=60_000, wait_until="domcontentloaded") + # Sincroniza com o conteúdo principal sem esperar recursos de terceiros + try: + page.wait_for_selector("h1", timeout=15_000) + except Exception: + pass + + # Título + title = "" + for selector in [ + ".content-head__title", + ".title h1", + "h1", + ]: + try: + el = page.locator(selector) + if el.count() > 0: + title = el.first.inner_text().strip() + if title: + logger.info(f"[{self.name}] Title from '{selector}'") + break + except Exception: + continue + if not title: + try: + title = page.title().strip() + except Exception: + title = "" + + # Descrição + description = "" + for selector in [ + "content-head__subtitle", + ".subtitle h2", + "h2" + ]: + try: + el = page.locator(selector) + if el.count() > 0: + description = el.first.inner_text().strip() + if description: + logger.info(f"[{self.name}] Description from '{selector}'") + break + except Exception: + continue + + if not description: + description = "-" + + # Corpo da notícia + body = "" + for selector in [ + "article .content-text__container", + "article .content-text p", + ]: + try: + els = page.locator(selector) + count = els.count() + if count > 0: + if count > 1: + parts = [] + for i in range(count): + t = els.nth(i).inner_text().strip() + if t: + parts.append(t) + body = "\n\n".join(parts).strip() + else: + body = els.first.inner_text().strip() + if body and len(body) > 100: + logger.info(f"[{self.name}] Body from '{selector}' (len={len(body)})") + break + except Exception: + continue - elements = page.locator(".videoCube") - ad_items = [] - visible_elements = [] - for i in range(elements.count()): - element = elements.nth(i) - if not element.is_visible(): + # Tags + tags = [] + for selector in [ + ".entities__list-item a", + ".entities__list li a" + ]: + try: + els = page.locator(selector) + if els.count() > 0: + for i in range(els.count()): + t = els.nth(i).inner_text().strip() + if t and t not in tags: + tags.append(t) + if tags: + logger.info(f"[{self.name}] {len(tags)} tags from '{selector}'") + break + except Exception: continue - visible_elements.append(element) - content = element.inner_html() - ad_item = self.find_items(content) - ad_items.append(ad_item) return EntryItem( - title=entry_title, - ads=ad_items, + title=title, url=self.url, - screenshot_path=entry_screenshot_path, + description=description, + body=body, + tags=tags, ) diff --git a/spiders/globo.py b/spiders/globo.py index 31584de..8054791 100644 --- a/spiders/globo.py +++ b/spiders/globo.py @@ -1,3 +1,5 @@ +import re + import scrapy from spiders.base import BaseSpider @@ -7,16 +9,86 @@ class GloboSpider(BaseSpider): name = "globospider" start_urls = ["https://oglobo.globo.com/"] - allowed_domains = ["oglobo.globo.com/"] + allowed_domains = ["oglobo.globo.com", "www.oglobo.globo.com"] + + custom_settings = { + **BaseSpider.custom_settings, + "COOKIES_ENABLED": True, + "DOWNLOAD_DELAY": 2, + "RANDOMIZE_DOWNLOAD_DELAY": True, + "DEFAULT_REQUEST_HEADERS": { + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "Accept-Language": "pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7", + "Accept-Encoding": "gzip, deflate, br", + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0 Safari/537.36", + }, + "CONCURRENT_REQUESTS": 4, + } + + BLACKLISTED_SECTIONS = { + "videos", "podcast", "podcasts", "blogs", "colunas", "assinante", "clube-o-globo", "guia" + } + + def allow_url(self, url: str) -> bool: + if not url: + return False + + # Normaliza URL, remove fragmentos e query + url = url.split('#', 1)[0].split('?', 1)[0] + + # Considera apenas links do domínio oglobo.globo.com + if not re.match(r"https?://(www\.)?oglobo\.globo\.com/", url): + return False + + path = re.sub(r"https?://[^/]+", "", url) + segments_norm = [s.lower() for s in path.strip("/").split("/") if s] + + # Blacklist por prefixo em qualquer segmento + blacklist_prefixes = {p.lstrip('/').lower() for p in self.blacklist} + for seg in segments_norm: + if any(seg.startswith(pref) for pref in blacklist_prefixes): + return False - def allow_url(self, entry_url): - return len(entry_url) > 100 and entry_url.startswith("https://oglobo.globo.com") + # Exige ao menos dois segmentos (seção + slug) + segments = [s for s in path.split('/') if s] + if len(segments) < 2: + return False + + slug = segments[-1] + + # Slug deve terminar com .ghtml e ter ao menos 3 hífens + return slug.endswith('.ghtml') and (slug.count('-') >= 3) + + def start_requests(self): + for url in self.start_urls: + yield scrapy.Request( + url, + callback=self.parse, + dont_filter=True, + meta={"dont_redirect": True, "handle_httpstatus_list": [403]}, + ) def parse(self, response): - url_item = URLItem() - for entry in response.css("a"): - url = entry.attrib.get("href") - if url and self.allow_url(url): - url_item["url"] = url - yield url_item - yield scrapy.Request(url=url, callback=self.parse) + seen = set() + selectors = [ + "a[href*='oglobo.globo.com/']", + "a[href^='/']", + ] + + for sel in selectors: + for entry in response.css(sel): + href = entry.attrib.get("href") + if not href: + continue + + url = response.urljoin(href).split('?', 1)[0].split('#', 1)[0] + if url in seen: + continue + + if self.allow_url(url): + seen.add(url) + yield URLItem(url=url) + + self.logger.info(f"Collected {len(seen)} unique news URLs from O Globo")