diff --git a/Makefile b/Makefile index a7f7ce2..3fc530a 100644 --- a/Makefile +++ b/Makefile @@ -87,7 +87,10 @@ scrape_uol: scrape_folha: docker compose exec scraper python scrape_no_openai.py --platform folha.uol.com.br - + +scrape_terra: + docker compose exec scraper python scrape_no_openai.py --platform terra.com.br + scrape_jornaldaparaiba: docker compose exec scraper python scrape_no_openai.py --platform jornaldaparaiba.com.br @@ -120,7 +123,10 @@ crawl_ig: crawl_folha: docker compose run scraper python crawl.py folhaspider - + +crawl_terra: + docker compose run --rm scraper python crawl.py terraspider + crawl_jornaldaparaiba: docker compose run scraper python crawl.py jornaldaparaiba @@ -135,6 +141,7 @@ crawl_all_working: @make crawl_aliadosBrasil @make crawl_ig @make crawl_folha + @make crawl_terra @make crawl_jornaldaparaiba @echo "Crawl de todos os portais concluído!" @@ -149,6 +156,7 @@ scrape_all_working: @make scrape_r7 @make scrape_uol @make scrape_folha + @make scrape_terra @make scrape_jornaldaparaiba @echo "Scraping de todos os portais concluído!" @@ -197,6 +205,7 @@ help: @echo " make crawl_aliadosBrasil - Coleta URLs do portal AliadosBrasil" @echo " make crawl_ig - Coleta URLs do portal IG" @echo " make crawl_folha - Coleta URLs do portal Folha" + @echo " make crawl_terra - Coleta URLs do portal Terra" @echo "" @echo "=== COMANDOS DE SCRAPING (Extração de Anúncios) ===" @echo " make scrape_all_working - Executa scraping de todos os portais funcionais" @@ -208,6 +217,7 @@ help: @echo " make scrape_r7 - Scraping do portal R7" @echo " make scrape_uol - Scraping do portal UOL" @echo " make scrape_folha - Scraping do portal Folha" + @echo " make scrape_terra - Scraping do portal Terra" @echo "" @echo "=== WORKFLOWS COMPLETOS ===" @echo " make pipeline_complete - Executa crawl + scraping de todos os portais" @@ -255,4 +265,4 @@ all: # Parar todos os serviços down: - docker compose down \ No newline at end of file + docker-compose down \ No newline at end of file diff --git a/plays/terra.py b/plays/terra.py index 4ac9032..9b524a7 100644 --- a/plays/terra.py +++ b/plays/terra.py @@ -1,60 +1,153 @@ -import time - from playwright.sync_api import sync_playwright from plays.base import BasePlay -from plays.items import AdItem, EntryItem -from plays.utils import get_or_none +from plays.items import EntryItem from plog import logger class TerraPlay(BasePlay): name = "terra" - n_expected_ads = 10 @classmethod def match(cls, url): return "terra.com.br" in url - def find_items(self, html_content) -> AdItem: - return AdItem( - title=get_or_none(r'title="(.*?)"', html_content), - url=get_or_none(r'href="(.*?)"', html_content), - thumbnail_url=get_or_none(r'url\("(.*?)"\)', html_content), - tag=get_or_none(r'(.*?)<\/span>', html_content), - ) - def pre_run(self): pass def run(self) -> EntryItem: with sync_playwright() as p: - browser = self.launch_browser(p) + browser = self.launch_browser(p, viewport={"width": 1600, "height": 1000}) page = browser.new_page() + + # Bloquear recursos pesados/ads para acelerar + blocked_domains = ( + "doubleclick.net", + "googlesyndication.com", + "google-analytics.com", + "analytics.google.com", + "clarity.ms", + "tailtarget.com", + "flashtalking.com", + "googletagmanager.com", + ) + + def block_ads(route, request): + try: + if request.resource_type in {"image", "media", "font", "stylesheet"}: + return route.abort() + if any(d in request.url for d in blocked_domains): + return route.abort() + except Exception: + pass + return route.continue_() + + page.route("**/*", block_ads) + page.set_default_navigation_timeout(60_000) + page.set_default_timeout(15_000) + logger.info(f"[{self.name}] Opening URL '{self.url}'...") - page.goto(self.url, timeout=180_000) - logger.info(f"[{self.name}] Searching for ads...") - page.locator("#taboola-below-article-thumbnails").scroll_into_view_if_needed() - - entry_screenshot_path = self.take_screenshot(page, self.url, goto=False) - entry_title = page.locator("title").inner_text() - time.sleep(self.wait_time * 2) - - elements = page.locator(".videoCube") - ad_items = [] - visible_elements = [] - for i in range(elements.count()): - element = elements.nth(i) - if not element.is_visible(): + page.goto(self.url, timeout=60_000, wait_until="domcontentloaded") + # Sincroniza com o conteúdo principal sem esperar recursos de terceiros + try: + page.wait_for_selector("h1", timeout=15_000) + except Exception: + pass + + # Título + title = "" + for selector in [ + ".article__header__headline h1", + ".article__header__headline", + ".special-article__header--headline h1", + ".special-article__header--headline", + "h1" + ]: + try: + el = page.locator(selector) + if el.count() > 0: + title = el.first.inner_text().strip() + if title: + logger.info(f"[{self.name}] Title from '{selector}'") + break + except Exception: + continue + if not title: + try: + title = page.title().strip() + except Exception: + title = "" + + # Descrição + description = "" + for selector in [ + ".article__header__subtitle h2", + ".article__header__subtitle", + ".special-article__header--subtitle h2", + ".special-article__header--subtitle", + ]: + try: + el = page.locator(selector) + if el.count() > 0: + description = el.first.inner_text().strip() + if description: + logger.info(f"[{self.name}] Description from '{selector}'") + break + except Exception: + continue + + # Corpo da notícia + body = "" + for selector in [ + ".article__content--body p, .article__content--body h2", + ".article__content--body", + ".article__content", + ".special-article__content--body p, .special-article__content--body h2", + ".special-article__content--body", + ".special-article__content", + ]: + try: + els = page.locator(selector) + count = els.count() + if count > 0: + if count > 1: + parts = [] + for i in range(count): + t = els.nth(i).inner_text().strip() + if t: + parts.append(t) + body = "\n\n".join(parts).strip() + else: + body = els.first.inner_text().strip() + if body and len(body) > 100: + logger.info(f"[{self.name}] Body from '{selector}' (len={len(body)})") + break + except Exception: + continue + + # Tags + tags = [] + for selector in [ + ".t360-tags__list a", + ".t360-tags__list" + ]: + try: + els = page.locator(selector) + if els.count() > 0: + for i in range(els.count()): + t = els.nth(i).inner_text().strip() + if t and t not in tags: + tags.append(t) + if tags: + logger.info(f"[{self.name}] {len(tags)} tags from '{selector}'") + break + except Exception: continue - visible_elements.append(element) - content = element.inner_html() - ad_item = self.find_items(content) - ad_items.append(ad_item) return EntryItem( - title=entry_title, - ads=ad_items, + title=title, url=self.url, - screenshot_path=entry_screenshot_path, + description=description, + body=body, + tags=tags, ) diff --git a/spiders/terra.py b/spiders/terra.py index cceb2cb..4ac4680 100644 --- a/spiders/terra.py +++ b/spiders/terra.py @@ -1,4 +1,7 @@ +import re + import scrapy +from scrapy_playwright.page import PageMethod from spiders.base import BaseSpider from spiders.items import URLItem @@ -7,16 +10,143 @@ class TerraSpider(BaseSpider): name = "terraspider" start_urls = ["https://www.terra.com.br/"] - allowed_domains = ["terra.com.br"] + allowed_domains = ["terra.com.br", "www.terra.com.br"] + + custom_settings = { + **BaseSpider.custom_settings, + "COOKIES_ENABLED": True, + "DOWNLOAD_DELAY": 2, + "RANDOMIZE_DOWNLOAD_DELAY": True, + "DEFAULT_REQUEST_HEADERS": { + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "Accept-Language": "pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7", + "Accept-Encoding": "gzip, deflate, br", + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0 Safari/537.36", + }, + # Ativa o Playwright para scrollar e carregar conteúdo dinâmico + "DOWNLOAD_HANDLERS": { + "http": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler", + "https": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler", + }, + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "PLAYWRIGHT_BROWSER_TYPE": "firefox", + "PLAYWRIGHT_LAUNCH_OPTIONS": {"headless": True}, + "PLAYWRIGHT_DEFAULT_NAVIGATION_TIMEOUT": 60 * 1000, + "PLAYWRIGHT_ABORT_RESOURCE_TYPES": ["image", "media", "font", "stylesheet"], + "CONCURRENT_REQUESTS": 4, + } + + # Seções que não são notícias + BLACKLISTED_SECTIONS = { + "videos", "podcast", "podcasts", "blogs", "colunas", "especiais", + "ao-vivo", "story", "amp", "apostas", "parceiros", "avisolegal", "horoscopo", "degusta", + } + + def allow_url(self, url: str) -> bool: + """ + Filtragem melhorada para URLs de notícias do Terra. + """ + if not url: + return False + url = url.split('#', 1)[0] + + # Normaliza e filtra padrões comuns de matéria do Terra, ex.: + # https://www.terra.com.br/noticias/brasil/... + if not re.match(r"https?://(www\.)?terra\.com\.br/", url): + return False + + # Evita áreas que não são notícias + path = re.sub(r"https?://[^/]+", "", url) + segments_norm = [s.lower() for s in path.strip("/").split("/") if s] + + # Ignora se QUALQUER segmento começar com algum prefixo da blacklist + blacklist_prefixes = {p.lstrip('/').lower() for p in self.BLACKLISTED_SECTIONS} + for seg in segments_norm: + if any(seg.startswith(pref) for pref in blacklist_prefixes): + return False + + # Require pelo menos dois segmentos (seção + slug da notícia) + segments = [s for s in path.split('/') if s] + if len(segments) < 2: + return False + + slug = segments[-1] + + # Aceita somente se terminar com .html e tiver slug descritivo + return slug.endswith('.html') and (slug.count('-') >= 3) - def allow_url(self, entry_url): - return True + def _block_ads(self, route, request): + # Bloqueia recursos desnecessários e domínios de anúncios para carregamento mais rápido + try: + blocked_domains = ( + "doubleclick.net", + "googlesyndication.com", + "google-analytics.com", + "analytics.google.com", + "clarity.ms", + "tailtarget.com", + "flashtalking.com", + "googletagmanager.com", + ) + if request.resource_type in {"image", "media", "font", "stylesheet"}: + return route.abort() + if any(d in request.url for d in blocked_domains): + return route.abort() + return route.continue_() + except Exception: + return route.continue_() + + def start_requests(self): + for url in self.start_urls: + methods = [ + PageMethod("route", "**/*", self._block_ads), + PageMethod("wait_for_load_state", "domcontentloaded"), + ] + for _ in range(12): + methods.append(PageMethod("evaluate", "window.scrollBy(0, document.body.scrollHeight)")) + methods.append(PageMethod("wait_for_timeout", 700)) + + yield scrapy.Request( + url, + callback=self.parse, + dont_filter=True, + meta={ + "playwright": True, + "playwright_page_goto_kwargs": {"wait_until": "domcontentloaded", "timeout": 60_000}, + "playwright_page_methods": methods, + }, + ) def parse(self, response): - url_item = URLItem() - for entry in response.css(".main-url"): - url = entry.attrib.get("href") - if url and self.allow_url(url): - url_item["url"] = url - yield url_item - yield scrapy.Request(url=url, callback=self.parse) + """ + Extrai apenas URLs da página inicial, sem crawling recursivo + """ + seen = set() + selectors = [ + "a[href*='terra.com.br/noticias']", + "a[href*='terra.com.br/economia']", + "a[href*='terra.com.br/esportes']", + "a[href*='terra.com.br/vida-e-estilo']", + "a[href*='terra.com.br/mobilidade']", + "a[href*='terra.com.br/diversao']", + "a[href*='terra.com.br/']", + ] + + for sel in selectors: + for entry in response.css(sel): + href = entry.attrib.get("href") + + if not href: + continue + url = response.urljoin(href).split('?', 1)[0].split('#', 1)[0] + + if url in seen: + continue + + if self.allow_url(url): + seen.add(url) + yield URLItem(url=url) + + self.logger.info(f"Collected {len(seen)} unique news URLs from Terra")