feat(Terra): Adaptação do scraper existente do portal para extrair conteúdo de notícias em vez de anúncios

Makefile

@@ -87,7 +87,10 @@ scrape_uol:
 
 scrape_folha:
 	docker compose exec scraper python scrape_no_openai.py --platform folha.uol.com.br
-
+
+scrape_terra:
+	docker compose exec scraper python scrape_no_openai.py --platform terra.com.br
+
 scrape_jornaldaparaiba:
 	docker compose exec scraper python scrape_no_openai.py --platform jornaldaparaiba.com.br
 
@@ -120,7 +123,10 @@ crawl_ig:
 
 crawl_folha:
 	docker compose run scraper python crawl.py folhaspider
-
+
+crawl_terra:
+	docker compose run --rm scraper python crawl.py terraspider
+
 crawl_jornaldaparaiba:
 	docker compose run scraper python crawl.py jornaldaparaiba
 
@@ -135,6 +141,7 @@ crawl_all_working:
 	@make crawl_aliadosBrasil
 	@make crawl_ig
 	@make crawl_folha
+	@make crawl_terra
 	@make crawl_jornaldaparaiba
 	@echo "Crawl de todos os portais concluído!"
 
@@ -149,6 +156,7 @@ scrape_all_working:
 	@make scrape_r7
 	@make scrape_uol
 	@make scrape_folha
+	@make scrape_terra
 	@make scrape_jornaldaparaiba
 	@echo "Scraping de todos os portais concluído!"
 
@@ -197,6 +205,7 @@ help:
 	@echo "  make crawl_aliadosBrasil - Coleta URLs do portal AliadosBrasil"
 	@echo "  make crawl_ig          - Coleta URLs do portal IG"
 	@echo "  make crawl_folha       - Coleta URLs do portal Folha"
+	@echo "  make crawl_terra       - Coleta URLs do portal Terra"
 	@echo ""
 	@echo "=== COMANDOS DE SCRAPING (Extração de Anúncios) ==="
 	@echo "  make scrape_all_working - Executa scraping de todos os portais funcionais"
@@ -208,6 +217,7 @@ help:
 	@echo "  make scrape_r7          - Scraping do portal R7"
 	@echo "  make scrape_uol         - Scraping do portal UOL"
 	@echo "  make scrape_folha       - Scraping do portal Folha"
+	@echo "  make scrape_terra       - Scraping do portal Terra"
 	@echo ""
 	@echo "=== WORKFLOWS COMPLETOS ==="
 	@echo "  make pipeline_complete  - Executa crawl + scraping de todos os portais"
@@ -255,4 +265,4 @@ all:
 
 # Parar todos os serviços
 down:
-	docker compose down
+	docker-compose down

plays/terra.py

@@ -1,60 +1,153 @@
-import time
-
 from playwright.sync_api import sync_playwright
 
 from plays.base import BasePlay
-from plays.items import AdItem, EntryItem
-from plays.utils import get_or_none
+from plays.items import EntryItem
 from plog import logger
 
 
 class TerraPlay(BasePlay):
     name = "terra"
-    n_expected_ads = 10
 
     @classmethod
     def match(cls, url):
         return "terra.com.br" in url
 
-    def find_items(self, html_content) -> AdItem:
-        return AdItem(
-            title=get_or_none(r'title="(.*?)"', html_content),
-            url=get_or_none(r'href="(.*?)"', html_content),
-            thumbnail_url=get_or_none(r'url\("(.*?)"\)', html_content),
-            tag=get_or_none(r'<span class="branding-inner".*?>(.*?)<\/span>', html_content),
-        )
-
     def pre_run(self):
         pass
 
     def run(self) -> EntryItem:
         with sync_playwright() as p:
-            browser = self.launch_browser(p)
+            browser = self.launch_browser(p, viewport={"width": 1600, "height": 1000})
             page = browser.new_page()
+
+            # Bloquear recursos pesados/ads para acelerar
+            blocked_domains = (
+                "doubleclick.net",
+                "googlesyndication.com",
+                "google-analytics.com",
+                "analytics.google.com",
+                "clarity.ms",
+                "tailtarget.com",
+                "flashtalking.com",
+                "googletagmanager.com",
+            )
+
+            def block_ads(route, request):
+                try:
+                    if request.resource_type in {"image", "media", "font", "stylesheet"}:
+                        return route.abort()
+                    if any(d in request.url for d in blocked_domains):
+                        return route.abort()
+                except Exception:
+                    pass
+                return route.continue_()
+
+            page.route("**/*", block_ads)
+            page.set_default_navigation_timeout(60_000)
+            page.set_default_timeout(15_000)
+
             logger.info(f"[{self.name}] Opening URL '{self.url}'...")
-            page.goto(self.url, timeout=180_000)
-            logger.info(f"[{self.name}] Searching for ads...")
-            page.locator("#taboola-below-article-thumbnails").scroll_into_view_if_needed()
-
-            entry_screenshot_path = self.take_screenshot(page, self.url, goto=False)
-            entry_title = page.locator("title").inner_text()
-            time.sleep(self.wait_time * 2)
-
-            elements = page.locator(".videoCube")
-            ad_items = []
-            visible_elements = []
-            for i in range(elements.count()):
-                element = elements.nth(i)
-                if not element.is_visible():
+            page.goto(self.url, timeout=60_000, wait_until="domcontentloaded")
+            # Sincroniza com o conteúdo principal sem esperar recursos de terceiros
+            try:
+                page.wait_for_selector("h1", timeout=15_000)
+            except Exception:
+                pass
+
+            # Título
+            title = ""
+            for selector in [
+                ".article__header__headline h1",
+                ".article__header__headline",
+                ".special-article__header--headline h1",
+                ".special-article__header--headline",
+                "h1"
+            ]:
+                try:
+                    el = page.locator(selector)
+                    if el.count() > 0:
+                        title = el.first.inner_text().strip()
+                        if title:
+                            logger.info(f"[{self.name}] Title from '{selector}'")
+                            break
+                except Exception:
+                    continue
+            if not title:
+                try:
+                    title = page.title().strip()
+                except Exception:
+                    title = ""
+
+            # Descrição
+            description = ""
+            for selector in [
+                ".article__header__subtitle h2",
+                ".article__header__subtitle",
+                ".special-article__header--subtitle h2",
+                ".special-article__header--subtitle",    
+            ]:
+                try:
+                    el = page.locator(selector)
+                    if el.count() > 0:
+                        description = el.first.inner_text().strip()
+                        if description:
+                            logger.info(f"[{self.name}] Description from '{selector}'")
+                            break
+                except Exception:
+                    continue
+
+            # Corpo da notícia
+            body = ""
+            for selector in [
+                ".article__content--body p, .article__content--body h2",
+                ".article__content--body",       
+                ".article__content",
+                ".special-article__content--body p, .special-article__content--body h2",
+                ".special-article__content--body",
+                ".special-article__content",
+            ]:
+                try:
+                    els = page.locator(selector)
+                    count = els.count()
+                    if count > 0:
+                        if count > 1:
+                            parts = []
+                            for i in range(count):
+                                t = els.nth(i).inner_text().strip()
+                                if t:
+                                    parts.append(t)
+                            body = "\n\n".join(parts).strip()
+                        else:
+                            body = els.first.inner_text().strip()
+                        if body and len(body) > 100:
+                            logger.info(f"[{self.name}] Body from '{selector}' (len={len(body)})")
+                            break
+                except Exception:
+                    continue
+
+            # Tags
+            tags = []
+            for selector in [
+                ".t360-tags__list a",
+                ".t360-tags__list"
+            ]:
+                try:
+                    els = page.locator(selector)
+                    if els.count() > 0:
+                        for i in range(els.count()):
+                            t = els.nth(i).inner_text().strip()
+                            if t and t not in tags:
+                                tags.append(t)
+                        if tags:
+                            logger.info(f"[{self.name}] {len(tags)} tags from '{selector}'")
+                            break
+                except Exception:
                     continue
-                visible_elements.append(element)
-                content = element.inner_html()
-                ad_item = self.find_items(content)
-                ad_items.append(ad_item)
 
             return EntryItem(
-                title=entry_title,
-                ads=ad_items,
+                title=title,
                 url=self.url,
-                screenshot_path=entry_screenshot_path,
+                description=description,
+                body=body,
+                tags=tags,
             )