From a63b190fec6774e3a4b4acb0affc773f636cc79a Mon Sep 17 00:00:00 2001 From: Alana Gabriele Amorim Silva Date: Wed, 24 Sep 2025 00:58:29 -0300 Subject: [PATCH 1/9] feat: Ajustando spider para extrair noticias do Terra --- spiders/terra.py | 100 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 90 insertions(+), 10 deletions(-) diff --git a/spiders/terra.py b/spiders/terra.py index cceb2cb..334d005 100644 --- a/spiders/terra.py +++ b/spiders/terra.py @@ -1,3 +1,5 @@ +import re + import scrapy from spiders.base import BaseSpider @@ -7,16 +9,94 @@ class TerraSpider(BaseSpider): name = "terraspider" start_urls = ["https://www.terra.com.br/"] - allowed_domains = ["terra.com.br"] + allowed_domains = ["terra.com.br", "www.terra.com.br"] + + custom_settings = { + **BaseSpider.custom_settings, + "COOKIES_ENABLED": True, + "DOWNLOAD_DELAY": 2, + "RANDOMIZE_DOWNLOAD_DELAY": True, + "DEFAULT_REQUEST_HEADERS": { + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "Accept-Language": "pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7", + "Accept-Encoding": "gzip, deflate, br", + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0 Safari/537.36", + }, + } + + # Seções que não são notícias + BLACKLISTED_SECTIONS = { + "videos", "podcast", "podcasts", "blogs", "colunas", "especiais", "ao-vivo", "story", "amp" + } + + def allow_url(self, url: str) -> bool: + """ + Filtragem melhorada para URLs de notícias do Terra. + """ + if not url: + return False + url = url.split('#', 1)[0] + + # Normaliza e filtra padrões comuns de matéria do Terra, ex.: + # https://www.terra.com.br/noticias/brasil/... + if not re.match(r"https?://(www\.)?terra\.com\.br/", url): + return False + + # Evita áreas que não são notícias + path = re.sub(r"https?://[^/]+", "", url) + first = (path.strip("/").split("/") or [""])[0] + if first in self.BLACKLISTED_SECTIONS: + return False + + # Require pelo menos dois segmentos (seção + slug da notícia) + segments = [s for s in path.split('/') if s] + if len(segments) < 2: + return False + + slug = segments[-1] - def allow_url(self, entry_url): - return True + # Heurística de slug de notícia + if slug.count('-') >= 3 or len(slug) > 30 or slug.endswith('.html'): + return True + return False + + def start_requests(self): + for url in self.start_urls: + yield scrapy.Request( + url, + callback=self.parse, + dont_filter=True, + meta={"dont_redirect": True, "handle_httpstatus_list": [403]}, + ) def parse(self, response): - url_item = URLItem() - for entry in response.css(".main-url"): - url = entry.attrib.get("href") - if url and self.allow_url(url): - url_item["url"] = url - yield url_item - yield scrapy.Request(url=url, callback=self.parse) + """ + Extrai apenas URLs da página inicial, sem crawling recursivo + """ + seen = set() + selectors = [ + "a[href*='terra.com.br/noticias']", + "a[href*='terra.com.br/economia']", + "a[href*='terra.com.br/esportes']", + "a[href*='terra.com.br/vida-e-estilo']", + "a[href*='terra.com.br/']", + ] + + for sel in selectors: + for entry in response.css(sel): + href = entry.attrib.get("href") + + if not href: + continue + url = response.urljoin(href).split('?', 1)[0].split('#', 1)[0] + + if url in seen: + continue + + if self.allow_url(url): + seen.add(url) + yield URLItem(url=url) + + self.logger.info(f"Collected {len(seen)} unique news URLs from Terra") From 8f6cc35ffc756f064017cc52b854618c10a0032c Mon Sep 17 00:00:00 2001 From: Alana Gabriele Amorim Silva Date: Wed, 24 Sep 2025 01:30:42 -0300 Subject: [PATCH 2/9] =?UTF-8?q?fix:=20Adi=C3=A7=C3=A3o=20do=20playwright?= =?UTF-8?q?=20no=20spider=20do=20Terra?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- spiders/terra.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/spiders/terra.py b/spiders/terra.py index 334d005..89f169b 100644 --- a/spiders/terra.py +++ b/spiders/terra.py @@ -1,6 +1,7 @@ import re import scrapy +from scrapy_playwright.page import PageMethod from spiders.base import BaseSpider from spiders.items import URLItem @@ -24,6 +25,17 @@ class TerraSpider(BaseSpider): "Upgrade-Insecure-Requests": "1", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0 Safari/537.36", }, + # Ativa o Playwright para scrollar e carregar conteúdo dinâmico + "DOWNLOAD_HANDLERS": { + "http": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler", + "https": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler", + }, + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "PLAYWRIGHT_BROWSER_TYPE": "firefox", + "PLAYWRIGHT_LAUNCH_OPTIONS": {"headless": True}, + "PLAYWRIGHT_DEFAULT_NAVIGATION_TIMEOUT": 60 * 1000, + "PLAYWRIGHT_ABORT_RESOURCE_TYPES": ["image", "media", "font", "stylesheet"], + "CONCURRENT_REQUESTS": 4, } # Seções que não são notícias @@ -62,15 +74,47 @@ def allow_url(self, url: str) -> bool: return True return False + def _block_ads(self, route, request): + try: + + blocked_domains = ( + "doubleclick.net", + "googlesyndication.com", + "google-analytics.com", + "analytics.google.com", + "clarity.ms", + "tailtarget.com", + "flashtalking.com", + "googletagmanager.com", + ) + if request.resource_type in {"image", "media", "font", "stylesheet"}: + return route.abort() + if any(d in request.url for d in blocked_domains): + return route.abort() + return route.continue_() + except Exception: + return route.continue_() + def start_requests(self): for url in self.start_urls: + methods = [ + PageMethod("route", "**/*", self._block_ads), + PageMethod("wait_for_load_state", "domcontentloaded"), + ] + for _ in range(12): + methods.append(PageMethod("evaluate", "window.scrollBy(0, document.body.scrollHeight)")) + methods.append(PageMethod("wait_for_timeout", 700)) + yield scrapy.Request( url, callback=self.parse, dont_filter=True, - meta={"dont_redirect": True, "handle_httpstatus_list": [403]}, + meta={ + "playwright": True, + "playwright_page_goto_kwargs": {"wait_until": "domcontentloaded", "timeout": 60_000}, + "playwright_page_methods": methods, + }, ) - def parse(self, response): """ Extrai apenas URLs da página inicial, sem crawling recursivo From 333b2b100911d8605dc56e4c77793cd74da8a106 Mon Sep 17 00:00:00 2001 From: Alana Gabriele Amorim Silva Date: Wed, 24 Sep 2025 01:39:51 -0300 Subject: [PATCH 3/9] feat: Ajustando play Terra para extrair os dados das noticias --- plays/terra.py | 121 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 87 insertions(+), 34 deletions(-) diff --git a/plays/terra.py b/plays/terra.py index 4ac9032..50fb5e6 100644 --- a/plays/terra.py +++ b/plays/terra.py @@ -1,60 +1,113 @@ -import time - from playwright.sync_api import sync_playwright from plays.base import BasePlay -from plays.items import AdItem, EntryItem -from plays.utils import get_or_none +from plays.items import EntryItem from plog import logger class TerraPlay(BasePlay): name = "terra" - n_expected_ads = 10 @classmethod def match(cls, url): return "terra.com.br" in url - def find_items(self, html_content) -> AdItem: - return AdItem( - title=get_or_none(r'title="(.*?)"', html_content), - url=get_or_none(r'href="(.*?)"', html_content), - thumbnail_url=get_or_none(r'url\("(.*?)"\)', html_content), - tag=get_or_none(r'(.*?)<\/span>', html_content), - ) - def pre_run(self): pass def run(self) -> EntryItem: with sync_playwright() as p: - browser = self.launch_browser(p) + browser = self.launch_browser(p, viewport={"width": 1600, "height": 1000}) page = browser.new_page() logger.info(f"[{self.name}] Opening URL '{self.url}'...") page.goto(self.url, timeout=180_000) - logger.info(f"[{self.name}] Searching for ads...") - page.locator("#taboola-below-article-thumbnails").scroll_into_view_if_needed() - - entry_screenshot_path = self.take_screenshot(page, self.url, goto=False) - entry_title = page.locator("title").inner_text() - time.sleep(self.wait_time * 2) - - elements = page.locator(".videoCube") - ad_items = [] - visible_elements = [] - for i in range(elements.count()): - element = elements.nth(i) - if not element.is_visible(): + + # Título + title = "" + for selector in [ + "h1", + ".article-title", + ".post-title", + ".c-news__title", + ]: + try: + el = page.locator(selector) + if el.count() > 0: + title = el.first.inner_text().strip() + if title: + logger.info(f"[{self.name}] Title from '{selector}'") + break + except Exception: + continue + + # Descrição + description = "" + for selector in [ + "h2", + ".article-subtitle", + ".c-news__subtitle", + ]: + try: + el = page.locator(selector) + if el.count() > 0: + description = el.first.inner_text().strip() + if description: + logger.info(f"[{self.name}] Description from '{selector}'") + break + except Exception: + continue + + # Corpo da notícia + body = "" + for selector in [ + ".article__content--body p, .article__content--body h2", + ".article__content--body", + ".article__content", + ".content", + ".c-news__body", + ]: + try: + els = page.locator(selector) + count = els.count() + if count > 0: + if count > 1: + parts = [] + for i in range(count): + t = els.nth(i).inner_text().strip() + if t: + parts.append(t) + body = "\n\n".join(parts).strip() + else: + body = els.first.inner_text().strip() + if body and len(body) > 100: + logger.info(f"[{self.name}] Body from '{selector}' (len={len(body)})") + break + except Exception: + continue + + # Tags + tags = [] + for selector in [ + ".t360-tags__list a", + ".t360-tags__list" + ]: + try: + els = page.locator(selector) + if els.count() > 0: + for i in range(els.count()): + t = els.nth(i).inner_text().strip() + if t and t not in tags: + tags.append(t) + if tags: + logger.info(f"[{self.name}] {len(tags)} tags from '{selector}'") + break + except Exception: continue - visible_elements.append(element) - content = element.inner_html() - ad_item = self.find_items(content) - ad_items.append(ad_item) return EntryItem( - title=entry_title, - ads=ad_items, + title=title, url=self.url, - screenshot_path=entry_screenshot_path, + description=description, + body=body, + tags=tags, ) From f9b76dcb3b296c65af6d93524199845a01a4443f Mon Sep 17 00:00:00 2001 From: Alana Gabriele Amorim Silva Date: Wed, 24 Sep 2025 01:48:46 -0300 Subject: [PATCH 4/9] fix: Adicionando novas classes para mapear o body da noticia do Terra --- plays/terra.py | 42 +++++++++++++++++++++++++++++++++++++++++- spiders/terra.py | 2 +- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/plays/terra.py b/plays/terra.py index 50fb5e6..81b2eff 100644 --- a/plays/terra.py +++ b/plays/terra.py @@ -19,8 +19,40 @@ def run(self) -> EntryItem: with sync_playwright() as p: browser = self.launch_browser(p, viewport={"width": 1600, "height": 1000}) page = browser.new_page() + + # Bloquear recursos pesados/ads para acelerar + blocked_domains = ( + "doubleclick.net", + "googlesyndication.com", + "google-analytics.com", + "analytics.google.com", + "clarity.ms", + "tailtarget.com", + "flashtalking.com", + "googletagmanager.com", + ) + + def block_ads(route, request): + try: + if request.resource_type in {"image", "media", "font", "stylesheet"}: + return route.abort() + if any(d in request.url for d in blocked_domains): + return route.abort() + except Exception: + pass + return route.continue_() + + page.route("**/*", block_ads) + page.set_default_navigation_timeout(60_000) + page.set_default_timeout(15_000) + logger.info(f"[{self.name}] Opening URL '{self.url}'...") - page.goto(self.url, timeout=180_000) + page.goto(self.url, timeout=60_000, wait_until="domcontentloaded") + # Sincroniza com o conteúdo principal sem esperar recursos de terceiros + try: + page.wait_for_selector("h1", timeout=15_000) + except Exception: + pass # Título title = "" @@ -39,6 +71,11 @@ def run(self) -> EntryItem: break except Exception: continue + if not title: + try: + title = page.title().strip() + except Exception: + title = "" # Descrição description = "" @@ -63,6 +100,9 @@ def run(self) -> EntryItem: ".article__content--body p, .article__content--body h2", ".article__content--body", ".article__content", + ".special-article__content--body p, .special-article__content--body h2", + ".special-article__content--body", + ".special-article__content", ".content", ".c-news__body", ]: diff --git a/spiders/terra.py b/spiders/terra.py index 89f169b..bfb49ce 100644 --- a/spiders/terra.py +++ b/spiders/terra.py @@ -40,7 +40,7 @@ class TerraSpider(BaseSpider): # Seções que não são notícias BLACKLISTED_SECTIONS = { - "videos", "podcast", "podcasts", "blogs", "colunas", "especiais", "ao-vivo", "story", "amp" + "videos", "podcast", "podcasts", "blogs", "colunas", "especiais", "ao-vivo", "story", "amp", "apostas" } def allow_url(self, url: str) -> bool: From dc6476d0fb954ed597cb7f18fc662632405dbba9 Mon Sep 17 00:00:00 2001 From: Alana Gabriele Amorim Silva Date: Wed, 24 Sep 2025 02:15:03 -0300 Subject: [PATCH 5/9] =?UTF-8?q?fix:=20Adicionando=20novas=20se=C3=A7=C3=B5?= =?UTF-8?q?es=20na=20blacklist=20do=20Terra?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- spiders/terra.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/spiders/terra.py b/spiders/terra.py index bfb49ce..a92ed0a 100644 --- a/spiders/terra.py +++ b/spiders/terra.py @@ -40,7 +40,8 @@ class TerraSpider(BaseSpider): # Seções que não são notícias BLACKLISTED_SECTIONS = { - "videos", "podcast", "podcasts", "blogs", "colunas", "especiais", "ao-vivo", "story", "amp", "apostas" + "videos", "podcast", "podcasts", "blogs", "colunas", "especiais", + "ao-vivo", "story", "amp", "apostas", "parceiros", "avisolegal" } def allow_url(self, url: str) -> bool: @@ -69,10 +70,9 @@ def allow_url(self, url: str) -> bool: slug = segments[-1] - # Heurística de slug de notícia - if slug.count('-') >= 3 or len(slug) > 30 or slug.endswith('.html'): - return True - return False + # Aceita somente se terminar com .html e tiver slug descritivo + return slug.endswith('.html') and (slug.count('-') >= 3) + def _block_ads(self, route, request): try: From 7272f6ea207df5acf848e42f42e6160cc4a937e1 Mon Sep 17 00:00:00 2001 From: Alana Gabriele Amorim Silva Date: Wed, 24 Sep 2025 02:40:48 -0300 Subject: [PATCH 6/9] fix: adicionando nossas classes e lidando melhor com a black list no Terra --- plays/terra.py | 21 ++++++++++++--------- spiders/terra.py | 13 ++++++++----- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/plays/terra.py b/plays/terra.py index 81b2eff..274bfb8 100644 --- a/plays/terra.py +++ b/plays/terra.py @@ -57,10 +57,11 @@ def block_ads(route, request): # Título title = "" for selector in [ - "h1", - ".article-title", - ".post-title", - ".c-news__title", + ".article__header__headline h1", + ".article__header__headline", + ".special-article__header--headline h1", + ".special-article__header--headline", + "h1" ]: try: el = page.locator(selector) @@ -80,9 +81,10 @@ def block_ads(route, request): # Descrição description = "" for selector in [ - "h2", - ".article-subtitle", - ".c-news__subtitle", + ".article__header__subtitle h2", + ".article__header__subtitle", + ".special-article__header--subtitle h2", + ".special-article__header--subtitle", ]: try: el = page.locator(selector) @@ -94,6 +96,9 @@ def block_ads(route, request): except Exception: continue + if not description: + description = "-" + # Corpo da notícia body = "" for selector in [ @@ -103,8 +108,6 @@ def block_ads(route, request): ".special-article__content--body p, .special-article__content--body h2", ".special-article__content--body", ".special-article__content", - ".content", - ".c-news__body", ]: try: els = page.locator(selector) diff --git a/spiders/terra.py b/spiders/terra.py index a92ed0a..ef10904 100644 --- a/spiders/terra.py +++ b/spiders/terra.py @@ -41,7 +41,7 @@ class TerraSpider(BaseSpider): # Seções que não são notícias BLACKLISTED_SECTIONS = { "videos", "podcast", "podcasts", "blogs", "colunas", "especiais", - "ao-vivo", "story", "amp", "apostas", "parceiros", "avisolegal" + "ao-vivo", "story", "amp", "apostas", "parceiros", "avisolegal", "horoscopo", "degusta", } def allow_url(self, url: str) -> bool: @@ -59,9 +59,13 @@ def allow_url(self, url: str) -> bool: # Evita áreas que não são notícias path = re.sub(r"https?://[^/]+", "", url) - first = (path.strip("/").split("/") or [""])[0] - if first in self.BLACKLISTED_SECTIONS: - return False + segments_norm = [s.lower() for s in path.strip("/").split("/") if s] + + # Ignora se QUALQUER segmento começar com algum prefixo da blacklist + blacklist_prefixes = {p.lstrip('/').lower() for p in self.BLACKLISTED_SECTIONS} + for seg in segments_norm: + if any(seg.startswith(pref) for pref in blacklist_prefixes): + return False # Require pelo menos dois segmentos (seção + slug da notícia) segments = [s for s in path.split('/') if s] @@ -76,7 +80,6 @@ def allow_url(self, url: str) -> bool: def _block_ads(self, route, request): try: - blocked_domains = ( "doubleclick.net", "googlesyndication.com", From 439682ebfe3ec264e351e0e28af9d6659d7b9f45 Mon Sep 17 00:00:00 2001 From: Alana Gabriele Amorim Silva Date: Wed, 24 Sep 2025 02:42:14 -0300 Subject: [PATCH 7/9] feat: Adicionando comandos de scraping e crawling para o portal Terra --- Makefile | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 12663c4..d2ad511 100644 --- a/Makefile +++ b/Makefile @@ -82,6 +82,9 @@ scrape_uol: scrape_folha: docker compose exec scraper python scrape_no_openai.py --platform folha.uol.com.br + +scrape_terra: + docker compose exec scraper python scrape_no_openai.py --platform terra.com.br # Crawler para todos os portais ou específicos crawl: @@ -112,6 +115,9 @@ crawl_ig: crawl_folha: docker compose run scraper python crawl.py folhaspider +crawl_terra: + docker compose run --rm scraper python crawl.py terraspider + # Workflow completo de coleta de URLs crawl_all_working: @echo "Executando crawl de todos os portais funcionais..." @@ -123,6 +129,7 @@ crawl_all_working: @make crawl_aliadosBrasil @make crawl_ig @make crawl_folha + @make crawl_terra @echo "Crawl de todos os portais concluído!" # Workflow completo de scraping @@ -136,6 +143,7 @@ scrape_all_working: @make scrape_r7 @make scrape_uol @make scrape_folha + @make scrape_terra @echo "Scraping de todos os portais concluído!" # Pipeline completo: crawl + scrape @@ -182,6 +190,7 @@ help: @echo " make crawl_aliadosBrasil - Coleta URLs do portal AliadosBrasil" @echo " make crawl_ig - Coleta URLs do portal IG" @echo " make crawl_folha - Coleta URLs do portal Folha" + @echo " make crawl_terra - Coleta URLs do portal Terra" @echo "" @echo "=== COMANDOS DE SCRAPING (Extração de Anúncios) ===" @echo " make scrape_all_working - Executa scraping de todos os portais funcionais" @@ -193,6 +202,7 @@ help: @echo " make scrape_r7 - Scraping do portal R7" @echo " make scrape_uol - Scraping do portal UOL" @echo " make scrape_folha - Scraping do portal Folha" + @echo " make scrape_terra - Scraping do portal Terra" @echo "" @echo "=== WORKFLOWS COMPLETOS ===" @echo " make pipeline_complete - Executa crawl + scraping de todos os portais" @@ -240,4 +250,3 @@ all: # Parar todos os serviços down: docker-compose down - From 4ef6e4ba69a8643125854df1c90261572cc79bfa Mon Sep 17 00:00:00 2001 From: Alana Gabriele Amorim Silva Date: Wed, 24 Sep 2025 10:52:06 -0300 Subject: [PATCH 8/9] =?UTF-8?q?fix:=20Bloquear=20recursos=20desnecess?= =?UTF-8?q?=C3=A1rios=20e=20dom=C3=ADnios=20de=20an=C3=BAncios=20para=20me?= =?UTF-8?q?lhorar=20o=20carregamento?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- spiders/terra.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/spiders/terra.py b/spiders/terra.py index ef10904..4ac4680 100644 --- a/spiders/terra.py +++ b/spiders/terra.py @@ -79,6 +79,7 @@ def allow_url(self, url: str) -> bool: def _block_ads(self, route, request): + # Bloqueia recursos desnecessários e domínios de anúncios para carregamento mais rápido try: blocked_domains = ( "doubleclick.net", @@ -128,6 +129,8 @@ def parse(self, response): "a[href*='terra.com.br/economia']", "a[href*='terra.com.br/esportes']", "a[href*='terra.com.br/vida-e-estilo']", + "a[href*='terra.com.br/mobilidade']", + "a[href*='terra.com.br/diversao']", "a[href*='terra.com.br/']", ] From 59a755e66fa25c3fdc1018c23566807b973a414b Mon Sep 17 00:00:00 2001 From: Alana Gabriele Amorim Silva Date: Wed, 24 Sep 2025 10:59:13 -0300 Subject: [PATCH 9/9] =?UTF-8?q?fix:=20Ajuste=20de=20descri=C3=A7=C3=A3o=20?= =?UTF-8?q?no=20Terra?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- plays/terra.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/plays/terra.py b/plays/terra.py index 274bfb8..9b524a7 100644 --- a/plays/terra.py +++ b/plays/terra.py @@ -96,9 +96,6 @@ def block_ads(route, request): except Exception: continue - if not description: - description = "-" - # Corpo da notícia body = "" for selector in [