feat(Globo): Adaptação do scraper existente do portal para extrair conteúdo de notícias em vez de anúncios

Makefile

@@ -88,6 +88,9 @@ scrape_uol:
 scrape_folha:
 	docker compose exec scraper python scrape_no_openai.py --platform folha.uol.com.br
 
+scrape_globo:
+	docker compose exec scraper python scrape_no_openai.py --platform globo.com
+
 scrape_jornaldaparaiba:
 	docker compose exec scraper python scrape_no_openai.py --platform jornaldaparaiba.com.br
 
@@ -121,6 +124,9 @@ crawl_ig:
 crawl_folha:
 	docker compose run scraper python crawl.py folhaspider
 
+crawl_globo:
+	docker compose run scraper python crawl.py globospider
+
 crawl_jornaldaparaiba:
 	docker compose run scraper python crawl.py jornaldaparaiba
 
@@ -135,6 +141,7 @@ crawl_all_working:
 	@make crawl_aliadosBrasil
 	@make crawl_ig
 	@make crawl_folha
+	@make crawl_globo
 	@make crawl_jornaldaparaiba
 	@echo "Crawl de todos os portais concluído!"
 
@@ -149,6 +156,7 @@ scrape_all_working:
 	@make scrape_r7
 	@make scrape_uol
 	@make scrape_folha
+	@make scrape_globo
 	@make scrape_jornaldaparaiba
 	@echo "Scraping de todos os portais concluído!"
 
@@ -197,6 +205,7 @@ help:
 	@echo "  make crawl_aliadosBrasil - Coleta URLs do portal AliadosBrasil"
 	@echo "  make crawl_ig          - Coleta URLs do portal IG"
 	@echo "  make crawl_folha       - Coleta URLs do portal Folha"
+	@echo "  make crawl_globo       - Coleta URLs do portal Globo"
 	@echo ""
 	@echo "=== COMANDOS DE SCRAPING (Extração de Anúncios) ==="
 	@echo "  make scrape_all_working - Executa scraping de todos os portais funcionais"
@@ -208,6 +217,7 @@ help:
 	@echo "  make scrape_r7          - Scraping do portal R7"
 	@echo "  make scrape_uol         - Scraping do portal UOL"
 	@echo "  make scrape_folha       - Scraping do portal Folha"
+	@echo "  make scrape_globo       - Scraping do portal Globo"
 	@echo ""
 	@echo "=== WORKFLOWS COMPLETOS ==="
 	@echo "  make pipeline_complete  - Executa crawl + scraping de todos os portais"

plays/globo.py

@@ -1,89 +1,149 @@
-import time
-
-from decouple import config
-from playwright.sync_api import TimeoutError as PlayWrightTimeoutError, sync_playwright
+from playwright.sync_api import sync_playwright
 
 from plays.base import BasePlay
-from plays.items import AdItem, EntryItem
-from plays.utils import get_or_none
+from plays.items import EntryItem
 from plog import logger
 
 
 class GloboPlay(BasePlay):
     name = "globo"
-    n_expected_ads = 50
-
-    def login(self):
-        with sync_playwright() as p:
-            browser = self.launch_browser(p)
-            page = browser.new_page()
-            login_url = "https://login.globo.com/login/"
-            logger.info(f"Opening URL {login_url}...")
-            page.goto(login_url)
-            time.sleep(self.wait_time)
-            page.locator("#barra-item-login").click()
-            page.locator("#login").fill(config("GLOBO_USERNAME"))
-            page.locator("#password").fill(config("GLOBO_PASSWORD"))
-            page.locator("#login-button").click()
 
     @classmethod
     def match(cls, url):
-        return "oglobo.globo.com/" in url
-
-    @classmethod
-    def extra_kwargs(cls):
-        return {"allow_remove_session": False}
-
-    def find_items(self, html_content) -> AdItem:
-        return AdItem(
-            title=get_or_none(r'title="(.*?)"', html_content),
-            url=get_or_none(r'href="(.*?)"', html_content),
-            thumbnail_url=get_or_none(r'url\("(.*?)"\)', html_content),
-            tag=get_or_none(r'<span class="branding-inner".*?>(.*?)<\/span>', html_content),
-            excerpt=get_or_none(r'slot="description" title="(.*?)"', html_content),
-        )
+        return "oglobo.globo.com" in url
 
     def pre_run(self):
         pass
 
-    def is_logged_in(self, page):
-        try:
-            page.locator(".login-profile__menu").first.inner_html()
-        except PlayWrightTimeoutError:
-            raise Exception(f"[{self.name}] Not logged in!")
-
     def run(self) -> EntryItem:
         with sync_playwright() as p:
-            browser = self.launch_browser(p)
+            browser = self.launch_browser(p, viewport={"width": 1600, "height": 1000})
             page = browser.new_page()
-            logger.info(f"[{self.name}] Opening URL '{self.url}'...")
-            page.goto(self.url, timeout=180_000)
-            logger.info(f"[{self.name}] Searching for ads...")
 
-            self.is_logged_in(page)
+            # Bloquear recursos pesados/ads para acelerar
+            blocked_domains = (
+                "doubleclick.net",
+                "googlesyndication.com",
+                "google-analytics.com",
+                "analytics.google.com",
+                "clarity.ms",
+                "tailtarget.com",
+                "flashtalking.com",
+                "googletagmanager.com",
+            )
+
+            def block_ads(route, request):
+                try:
+                    if request.resource_type in {"image", "media", "font", "stylesheet"}:
+                        return route.abort()
+                    if any(d in request.url for d in blocked_domains):
+                        return route.abort()
+                except Exception:
+                    pass
+                return route.continue_()
 
-            page.locator(".tbl-feed-header-text").scroll_into_view_if_needed()
-            self.scroll_down(page, 40, 400, wait_time=1)
-            page.locator("#boxComentarios").scroll_into_view_if_needed()
+            page.route("**/*", block_ads)
+            page.set_default_navigation_timeout(60_000)
+            page.set_default_timeout(15_000)
 
-            entry_screenshot_path = self.take_screenshot(page, self.url, goto=False, timeout=60_000)
-            entry_title = page.locator("title").inner_text()
+            logger.info(f"[{self.name}] Opening URL '{self.url}'...")
+            page.goto(self.url, timeout=60_000, wait_until="domcontentloaded")
+            # Sincroniza com o conteúdo principal sem esperar recursos de terceiros
+            try:
+                page.wait_for_selector("h1", timeout=15_000)
+            except Exception:
+                pass
+
+            # Título
+            title = ""
+            for selector in [
+                ".content-head__title",
+                ".title h1",
+                "h1",
+            ]:
+                try:
+                    el = page.locator(selector)
+                    if el.count() > 0:
+                        title = el.first.inner_text().strip()
+                        if title:
+                            logger.info(f"[{self.name}] Title from '{selector}'")
+                            break
+                except Exception:
+                    continue
+            if not title:
+                try:
+                    title = page.title().strip()
+                except Exception:
+                    title = ""
+
+            # Descrição
+            description = ""
+            for selector in [
+                "content-head__subtitle",
+                ".subtitle h2",
+                "h2"
+            ]:
+                try:
+                    el = page.locator(selector)
+                    if el.count() > 0:
+                        description = el.first.inner_text().strip()
+                        if description:
+                            logger.info(f"[{self.name}] Description from '{selector}'")
+                            break
+                except Exception:
+                    continue
+
+            if not description:
+                description = "-"
+
+            # Corpo da notícia
+            body = ""
+            for selector in [
+                "article .content-text__container",
+                "article .content-text p",
+            ]:
+                try:
+                    els = page.locator(selector)
+                    count = els.count()
+                    if count > 0:
+                        if count > 1:
+                            parts = []
+                            for i in range(count):
+                                t = els.nth(i).inner_text().strip()
+                                if t:
+                                    parts.append(t)
+                            body = "\n\n".join(parts).strip()
+                        else:
+                            body = els.first.inner_text().strip()
+                        if body and len(body) > 100:
+                            logger.info(f"[{self.name}] Body from '{selector}' (len={len(body)})")
+                            break
+                except Exception:
+                    continue
 
-            elements = page.locator(".videoCube")
-            ad_items = []
-            visible_elements = []
-            for i in range(elements.count()):
-                element = elements.nth(i)
-                if not element.is_visible():
+            # Tags
+            tags = []
+            for selector in [
+                ".entities__list-item a",
+                ".entities__list li a"
+            ]:
+                try:
+                    els = page.locator(selector)
+                    if els.count() > 0:
+                        for i in range(els.count()):
+                            t = els.nth(i).inner_text().strip()
+                            if t and t not in tags:
+                                tags.append(t)
+                        if tags:
+                            logger.info(f"[{self.name}] {len(tags)} tags from '{selector}'")
+                            break
+                except Exception:
                     continue
-                visible_elements.append(element)
-                content = element.inner_html()
-                ad_item = self.find_items(content)
-                ad_items.append(ad_item)
 
             return EntryItem(
-                title=entry_title,
-                ads=ad_items,
+                title=title,
                 url=self.url,
-                screenshot_path=entry_screenshot_path,
+                description=description,
+                body=body,
+                tags=tags,
             )

spiders/globo.py

@@ -1,3 +1,5 @@
+import re
+
 import scrapy
 
 from spiders.base import BaseSpider
@@ -7,16 +9,86 @@
 class GloboSpider(BaseSpider):
     name = "globospider"
     start_urls = ["https://oglobo.globo.com/"]
-    allowed_domains = ["oglobo.globo.com/"]
+    allowed_domains = ["oglobo.globo.com", "www.oglobo.globo.com"]
+
+    custom_settings = {
+        **BaseSpider.custom_settings,
+        "COOKIES_ENABLED": True,
+        "DOWNLOAD_DELAY": 2,
+        "RANDOMIZE_DOWNLOAD_DELAY": True,
+        "DEFAULT_REQUEST_HEADERS": {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+            "Accept-Language": "pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7",
+            "Accept-Encoding": "gzip, deflate, br",
+            "Connection": "keep-alive",
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0 Safari/537.36",
+        },
+        "CONCURRENT_REQUESTS": 4,
+    }
+
+    BLACKLISTED_SECTIONS = {
+        "videos", "podcast", "podcasts", "blogs", "colunas", "assinante", "clube-o-globo", "guia"
+    }
+
+    def allow_url(self, url: str) -> bool:
+        if not url:
+            return False
+
+        # Normaliza URL, remove fragmentos e query
+        url = url.split('#', 1)[0].split('?', 1)[0]
+
+        # Considera apenas links do domínio oglobo.globo.com
+        if not re.match(r"https?://(www\.)?oglobo\.globo\.com/", url):
+            return False
+
+        path = re.sub(r"https?://[^/]+", "", url)
+        segments_norm = [s.lower() for s in path.strip("/").split("/") if s]
+
+        # Blacklist por prefixo em qualquer segmento
+        blacklist_prefixes = {p.lstrip('/').lower() for p in self.blacklist}
+        for seg in segments_norm:
+            if any(seg.startswith(pref) for pref in blacklist_prefixes):
+                return False
 
-    def allow_url(self, entry_url):
-        return len(entry_url) > 100 and entry_url.startswith("https://oglobo.globo.com")
+        # Exige ao menos dois segmentos (seção + slug)
+        segments = [s for s in path.split('/') if s]
+        if len(segments) < 2:
+            return False
+
+        slug = segments[-1]
+
+        # Slug deve terminar com .ghtml e ter ao menos 3 hífens
+        return slug.endswith('.ghtml') and (slug.count('-') >= 3)
+
+    def start_requests(self):
+        for url in self.start_urls:
+            yield scrapy.Request(
+                url,
+                callback=self.parse,
+                dont_filter=True,
+                meta={"dont_redirect": True, "handle_httpstatus_list": [403]},
+            )
 
     def parse(self, response):
-        url_item = URLItem()
-        for entry in response.css("a"):
-            url = entry.attrib.get("href")
-            if url and self.allow_url(url):
-                url_item["url"] = url
-                yield url_item
-                yield scrapy.Request(url=url, callback=self.parse)
+        seen = set()
+        selectors = [
+            "a[href*='oglobo.globo.com/']",
+            "a[href^='/']",
+        ]
+
+        for sel in selectors:
+            for entry in response.css(sel):
+                href = entry.attrib.get("href")
+                if not href:
+                    continue
+
+                url = response.urljoin(href).split('?', 1)[0].split('#', 1)[0]
+                if url in seen:
+                    continue
+
+                if self.allow_url(url):
+                    seen.add(url)
+                    yield URLItem(url=url)
+
+        self.logger.info(f"Collected {len(seen)} unique news URLs from O Globo")