From 44e17b06877b41d161e2117051cad2a4a1c30339 Mon Sep 17 00:00:00 2001 From: Eduardo Ferreira Date: Wed, 24 Sep 2025 11:22:06 -0300 Subject: [PATCH 1/5] =?UTF-8?q?(feat):=20ajuste=20no=20spider=20para=20ext?= =?UTF-8?q?rair=20not=C3=ADcias=20do=20Jornal=20de=20Bras=C3=ADlia?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- spiders/jornalDeBrasilia.py | 92 +++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 spiders/jornalDeBrasilia.py diff --git a/spiders/jornalDeBrasilia.py b/spiders/jornalDeBrasilia.py new file mode 100644 index 0000000..2cd90c4 --- /dev/null +++ b/spiders/jornalDeBrasilia.py @@ -0,0 +1,92 @@ +import scrapy +from urllib.parse import urlparse +from spiders.base import BaseSpider +from spiders.items import URLItem + +class JornalDeBrasiliaSpider(BaseSpider): + name = "jornaldebrasiliaspider" + start_urls = ["https://jornaldebrasilia.com.br/"] + allowed_domains = ["jornaldebrasilia.com.br"] + + custom_settings = { + **BaseSpider.custom_settings, + "COOKIES_ENABLED": True, + "DOWNLOAD_DELAY": 2, + "DEFAULT_REQUEST_HEADERS": { + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "Accept-Language": "pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7", + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36", + } + } + + def allow_url(self, url: str) -> bool: + """ + Filtra apenas URLs de notícias válidas + """ + p = urlparse(url) + path = p.path.rstrip('/') + + # Blacklist de seções que não são notícias + blacklist = { + "/videos", "/podcasts", "/newsletter", "/contato", + "/sobre", "/anuncie", "/assine", "/busca" + } + + if path in blacklist: + return False + + # Requer pelo menos 2 segmentos (seção + slug) + segments = [seg for seg in path.split('/') if seg] + if len(segments) < 2: + return False + + # Verificar se é uma notícia (slug longo com hífens) + slug = segments[-1] + if slug.count('-') >= 2 or len(slug) > 20: + return True + + return False + + def start_requests(self): + for url in self.start_urls: + yield scrapy.Request( + url, + callback=self.parse, + dont_filter=True, + meta={"dont_redirect": True, "handle_httpstatus_list": [403]}, + ) + + def parse(self, response): + """ + Extrai URLs de notícias da página inicial + """ + # Seletores específicos do Jornal de Brasília + selectors = [ + "article a[href]", # Links dentro de artigos + ".news-item a[href]", # Links em itens de notícia + ".headline a[href]", # Links em manchetes + "h2 a[href], h3 a[href]", # Links em títulos + ] + + seen_urls = set() + + for selector in selectors: + links = response.css(selector) + + for link in links: + url = link.attrib.get("href") + if not url: + continue + + # Construir URL absoluta + full_url = response.urljoin(url) + full_url = full_url.split('#', 1)[0].split('?', 1)[0] + + # Evitar duplicatas + if full_url in seen_urls: + continue + seen_urls.add(full_url) + + # Filtrar URLs válidos + if self.allow_url(full_url): + yield URLItem(url=full_url) From 6597a033bde46293ccedf0e046e611f11b127746 Mon Sep 17 00:00:00 2001 From: Eduardo Ferreira Date: Wed, 24 Sep 2025 12:04:48 -0300 Subject: [PATCH 2/5] =?UTF-8?q?(fix):=20Adi=C3=A7=C3=A3o=20crawl=20Makefil?= =?UTF-8?q?e=20e=20Inicializa=C3=A7=C3=A3o=20Spider?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 4 + plays/jornalDeBrasilia.py | 161 ++++++++++++++++++++++++++++++++++++++ spiders/__init__.py | 14 ++-- 3 files changed, 173 insertions(+), 6 deletions(-) create mode 100644 plays/jornalDeBrasilia.py diff --git a/Makefile b/Makefile index 12663c4..df8afb1 100644 --- a/Makefile +++ b/Makefile @@ -112,6 +112,9 @@ crawl_ig: crawl_folha: docker compose run scraper python crawl.py folhaspider +crawl_jornalDeBrasilia: + docker compose run scraper python crawl.py jornaldebrasiliaspider + # Workflow completo de coleta de URLs crawl_all_working: @echo "Executando crawl de todos os portais funcionais..." @@ -123,6 +126,7 @@ crawl_all_working: @make crawl_aliadosBrasil @make crawl_ig @make crawl_folha + @make crawl_jornalDeBrasilia @echo "Crawl de todos os portais concluído!" # Workflow completo de scraping diff --git a/plays/jornalDeBrasilia.py b/plays/jornalDeBrasilia.py new file mode 100644 index 0000000..d96c6ca --- /dev/null +++ b/plays/jornalDeBrasilia.py @@ -0,0 +1,161 @@ +import time +from playwright.sync_api import sync_playwright +from plays.base import BasePlay +from plays.items import EntryItem +from plog import logger + +class CorreioPlay(BasePlay): + name = "correio" + + @classmethod + def match(cls, url): + """Identifica se a URL é do Correio Braziliense""" + return "correiobraziliense.com.br" in url + + def pre_run(self): + """Configurações antes da execução""" + pass + + def run(self) -> EntryItem: + """Extrai conteúdo da notícia""" + with sync_playwright() as p: + browser = self.launch_browser(p, viewport={"width": 1920, "height": 1080}) + page = browser.new_page() + + logger.info(f"[{self.name}] Opening URL '{self.url}'...") + page.goto(self.url, timeout=180_000) + + # Aguardar carregamento + page.wait_for_selector("h1", timeout=30000) + + # 1. EXTRAIR TÍTULO + title = self._extract_title(page) + + # 2. EXTRAIR DESCRIÇÃO + description = self._extract_description(page) + + # 3. EXTRAIR CORPO + body = self._extract_body(page) + + # 4. EXTRAIR TAGS + tags = self._extract_tags(page) + + return EntryItem( + title=title, + url=self.url, + description=description, + body=body, + tags=tags, + ) + + def _extract_title(self, page): + """Extrai o título da notícia""" + selectors = [ + "h1.title", + "h1.headline", + "h1", + ".article-title h1", + ".post-title h1" + ] + + for selector in selectors: + try: + element = page.locator(selector) + if element.count() > 0: + title = element.first.inner_text().strip() + if title: + logger.info(f"[{self.name}] Title extracted: {title[:50]}...") + return title + except Exception as e: + logger.warning(f"[{self.name}] Failed to extract title with {selector}: {str(e)}") + continue + + logger.error(f"[{self.name}] Failed to extract title") + return "" + + def _extract_description(self, page): + """Extrai a descrição/subtítulo da notícia""" + selectors = [ + ".subtitle", + ".lead", + ".summary", + ".article-subtitle", + "h2.subtitle", + ".excerpt" + ] + + for selector in selectors: + try: + element = page.locator(selector) + if element.count() > 0: + description = element.first.inner_text().strip() + if description and len(description) > 10: + logger.info(f"[{self.name}] Description extracted: {description[:50]}...") + return description + except Exception as e: + logger.warning(f"[{self.name}] Failed to extract description with {selector}: {str(e)}") + continue + + logger.info(f"[{self.name}] No description found") + return "" + + def _extract_body(self, page): + """Extrai o corpo da notícia""" + selectors = [ + ".article-content", + ".post-content", + ".content", + ".article-body", + ".text", + "article .content", + ".entry-content" + ] + + for selector in selectors: + try: + element = page.locator(selector) + if element.count() > 0: + # Remover elementos indesejados + element.locator("script, style, .ad, .advertisement").evaluate_all("el => el.remove()") + + body = element.inner_text().strip() + if body and len(body) > 100: # Garantir que tem conteúdo substancial + logger.info(f"[{self.name}] Body extracted ({len(body)} chars)") + return body + except Exception as e: + logger.warning(f"[{self.name}] Failed to extract body with {selector}: {str(e)}") + continue + + logger.error(f"[{self.name}] Failed to extract body") + return "" + + def _extract_tags(self, page): + """Extrai as tags da notícia""" + selectors = [ + ".tags a", + ".categories a", + ".post-tags a", + ".article-tags a", + ".keywords a", + "[rel='tag']" + ] + + tags = [] + + for selector in selectors: + try: + elements = page.locator(selector) + if elements.count() > 0: + for i in range(elements.count()): + tag_text = elements.nth(i).inner_text().strip() + if tag_text and tag_text not in tags: + tags.append(tag_text) + + if tags: + logger.info(f"[{self.name}] Tags extracted: {tags}") + break + except Exception as e: + logger.warning(f"[{self.name}] Failed to extract tags with {selector}: {str(e)}") + continue + + return tags \ No newline at end of file diff --git a/spiders/__init__.py b/spiders/__init__.py index ecb5e1f..22a68e1 100644 --- a/spiders/__init__.py +++ b/spiders/__init__.py @@ -1,19 +1,20 @@ +from spiders.aliadosBrasil import AliadosBrasilSpider from spiders.base import BaseSpider from spiders.estadao import EstadaoSpider from spiders.folha import FolhaSpider +from spiders.gazetaDoPovo import GazetaDoPovoSpider from spiders.globo import GloboSpider from spiders.ig import IGSpider +from spiders.jornalDeBrasilia import JornalDeBrasiliaSpider +from spiders.maisGoias import MaisGoiasSpider from spiders.metropoles import MetropolesSpider from spiders.r7 import R7Spider from spiders.rbs import RBSSpider -from spiders.veja import VejaSpider from spiders.terra import TerraSpider from spiders.uol import UOLSpider -from spiders.gazetaDoPovo import GazetaDoPovoSpider -from spiders.maisGoias import MaisGoiasSpider -from spiders.aliadosBrasil import AliadosBrasilSpider +from spiders.veja import VejaSpider -__all__ = [ +_all_ = [ BaseSpider, EstadaoSpider, FolhaSpider, @@ -28,4 +29,5 @@ GazetaDoPovoSpider, MaisGoiasSpider, AliadosBrasilSpider, -] + JornalDeBrasiliaSpider, +] \ No newline at end of file From 0e032c665f35808a689da1ad30cc2d12a6bbc453 Mon Sep 17 00:00:00 2001 From: Eduardo Ferreira Date: Wed, 8 Oct 2025 11:23:58 -0300 Subject: [PATCH 3/5] =?UTF-8?q?(feat):=20Implementa=C3=A7=C3=A3o=20da=20Pl?= =?UTF-8?q?ay=20e=20integra=C3=A7=C3=A3o=20dos=20comandos=20no=20Makefile?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 6 +- plays/__init__.py | 2 + plays/jornalDeBrasilia.py | 271 +++++++++++++++++------------------- spiders/jornalDeBrasilia.py | 9 +- 4 files changed, 142 insertions(+), 146 deletions(-) diff --git a/Makefile b/Makefile index df8afb1..78e70ca 100644 --- a/Makefile +++ b/Makefile @@ -82,7 +82,10 @@ scrape_uol: scrape_folha: docker compose exec scraper python scrape_no_openai.py --platform folha.uol.com.br - + +scrape_jornalDeBrasilia: + docker compose exec scraper python scrape_no_openai.py --platform jornaldebrasilia.com.br + # Crawler para todos os portais ou específicos crawl: docker compose run --rm scraper python crawl.py @@ -140,6 +143,7 @@ scrape_all_working: @make scrape_r7 @make scrape_uol @make scrape_folha + @make scrape_jornalDeBrasilia @echo "Scraping de todos os portais concluído!" # Pipeline completo: crawl + scrape diff --git a/plays/__init__.py b/plays/__init__.py index 338fe28..700eb9e 100644 --- a/plays/__init__.py +++ b/plays/__init__.py @@ -11,6 +11,7 @@ from plays.gazetaDoPovo import GazetaDoPovoPlay from plays.maisGoias import MaisGoias from plays.aliadosBrasil import AliadosBrasilPlay +from plays.jornalDeBrasilia import JornalDeBrasiliaPlay __all__ = [ ClicRBSPlay, @@ -26,4 +27,5 @@ GazetaDoPovoPlay, MaisGoias, AliadosBrasilPlay, + JornalDeBrasiliaPlay, ] diff --git a/plays/jornalDeBrasilia.py b/plays/jornalDeBrasilia.py index d96c6ca..56113ae 100644 --- a/plays/jornalDeBrasilia.py +++ b/plays/jornalDeBrasilia.py @@ -1,45 +1,147 @@ -import time -from playwright.sync_api import sync_playwright +from playwright.sync_api import sync_playwright, TimeoutError as PlaywrightTimeoutError + from plays.base import BasePlay from plays.items import EntryItem from plog import logger -class CorreioPlay(BasePlay): - name = "correio" + +class JornalDeBrasiliaPlay(BasePlay): + name = "jornaldebrasilia" @classmethod - def match(cls, url): - """Identifica se a URL é do Correio Braziliense""" - return "correiobraziliense.com.br" in url + def match(cls, url: str) -> bool: + """Verifica se a URL pertence ao Jornal de Brasília.""" + return "jornaldebrasilia.com.br" in url def pre_run(self): - """Configurações antes da execução""" - pass + """Método executado antes do 'run'.""" + logger.info(f"[{self.name}] Preparando para extrair a URL: {self.url}") def run(self) -> EntryItem: - """Extrai conteúdo da notícia""" + """Executa a extração principal dos dados da página.""" with sync_playwright() as p: - browser = self.launch_browser(p, viewport={"width": 1920, "height": 1080}) - page = browser.new_page() - - logger.info(f"[{self.name}] Opening URL '{self.url}'...") - page.goto(self.url, timeout=180_000) - - # Aguardar carregamento - page.wait_for_selector("h1", timeout=30000) - - # 1. EXTRAIR TÍTULO - title = self._extract_title(page) - - # 2. EXTRAIR DESCRIÇÃO - description = self._extract_description(page) - - # 3. EXTRAIR CORPO - body = self._extract_body(page) + browser = p.chromium.launch() + page = browser.new_page( + user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36" + ) + + page.route( + "**/*", + lambda route: route.abort() + if route.request.resource_type in {"image", "font", "media", "stylesheet"} + else route.continue_(), + ) - # 4. EXTRAIR TAGS - tags = self._extract_tags(page) + page.set_default_navigation_timeout(60_000) + page.set_default_timeout(10_000) + + try: + page.goto(self.url, wait_until="domcontentloaded", timeout=60_000) + page.wait_for_timeout(2000) + except PlaywrightTimeoutError: + logger.error(f"[{self.name}] Timeout ao carregar a página: {self.url}") + browser.close() + return EntryItem(title="", url=self.url, description="", body="", tags=[]) + + + # Título + title = "" + title_selectors = [ + "h1.tdb-title-text", + ".wrap-title > h1", + "h1.title", + "h1.entry-title", + ".post-title", + ".content-head__title", + "h1" + ] + for selector in title_selectors: + try: + el = page.locator(selector).first + if el.is_visible(timeout=1000): + title = el.inner_text().strip() + if title: + logger.info(f"[{self.name}] Título encontrado com o seletor: '{selector}'") + break + except Exception: + continue + if not title: + try: + title = page.title().strip().split("|")[0].strip() + except Exception: + title = "" + + + # Descrição (Subtítulo) + description = "" + description_selectors = [ + ".wrap-title > p:nth-child(2)", + "p.tdb-post-sub-title", + ".subtitle h2", + "p.td-post-sub-title" + ] + for selector in description_selectors: + try: + el = page.locator(selector).first + if el.is_visible(timeout=1000): + description = el.inner_text().strip() + if description: + logger.info(f"[{self.name}] Descrição encontrada com o seletor: '{selector}'") + break + except Exception: + continue + + # Corpo da Notícia + body = "" + body_container_selectors = [ + "div.tdb-block-inner.td-fix-index", + ".the-post-content", + ".td-post-content" + ] + for selector in body_container_selectors: + try: + container = page.locator(selector).first + if container.is_visible(timeout=1500): + logger.info(f"[{self.name}] Contêiner do corpo encontrado com o seletor: '{selector}'") + + # Extrai todos os parágrafos

de dentro do contêiner + paragraphs = container.locator("p") + parts = [] + for i in range(paragraphs.count()): + p_text = paragraphs.nth(i).inner_text().strip() + # Filtro para remover parágrafos indesejados + if p_text and len(p_text) > 40 and not p_text.lower().startswith(("leia mais", ">>", "crédito:", "siga o jornal")): + parts.append(p_text) + + if parts: + body = "\n\n".join(parts) + logger.info(f"[{self.name}] Corpo extraído com sucesso! Caracteres: {len(body)}") + break # Sai do loop principal se o corpo foi extraído + except Exception: + continue + + # Extração de Tags (opcional, pois nem sempre estão presentes) + tags = [] + tag_selectors = [ + ".tdb-tags a", + ".entities__list-item a", + ".entities__list li a" + ] + for selector in tag_selectors: + try: + tag_elements = page.locator(selector) + if tag_elements.count() > 0: + tag_list = [el.inner_text().strip() for el in tag_elements.all()] + tags = [t for t in tag_list if t] # Remove tags vazias + if tags: + logger.info(f"[{self.name}] {len(tags)} tags encontradas com o seletor: '{selector}'") + break + except Exception: + continue + + browser.close() + return EntryItem( title=title, url=self.url, @@ -48,114 +150,3 @@ def run(self) -> EntryItem: tags=tags, ) - def _extract_title(self, page): - """Extrai o título da notícia""" - selectors = [ - "h1.title", - "h1.headline", - "h1", - ".article-title h1", - ".post-title h1" - ] - - for selector in selectors: - try: - element = page.locator(selector) - if element.count() > 0: - title = element.first.inner_text().strip() - if title: - logger.info(f"[{self.name}] Title extracted: {title[:50]}...") - return title - except Exception as e: - logger.warning(f"[{self.name}] Failed to extract title with {selector}: {str(e)}") - continue - - logger.error(f"[{self.name}] Failed to extract title") - return "" - - def _extract_description(self, page): - """Extrai a descrição/subtítulo da notícia""" - selectors = [ - ".subtitle", - ".lead", - ".summary", - ".article-subtitle", - "h2.subtitle", - ".excerpt" - ] - - for selector in selectors: - try: - element = page.locator(selector) - if element.count() > 0: - description = element.first.inner_text().strip() - if description and len(description) > 10: - logger.info(f"[{self.name}] Description extracted: {description[:50]}...") - return description - except Exception as e: - logger.warning(f"[{self.name}] Failed to extract description with {selector}: {str(e)}") - continue - - logger.info(f"[{self.name}] No description found") - return "" - - def _extract_body(self, page): - """Extrai o corpo da notícia""" - selectors = [ - ".article-content", - ".post-content", - ".content", - ".article-body", - ".text", - "article .content", - ".entry-content" - ] - - for selector in selectors: - try: - element = page.locator(selector) - if element.count() > 0: - # Remover elementos indesejados - element.locator("script, style, .ad, .advertisement").evaluate_all("el => el.remove()") - - body = element.inner_text().strip() - if body and len(body) > 100: # Garantir que tem conteúdo substancial - logger.info(f"[{self.name}] Body extracted ({len(body)} chars)") - return body - except Exception as e: - logger.warning(f"[{self.name}] Failed to extract body with {selector}: {str(e)}") - continue - - logger.error(f"[{self.name}] Failed to extract body") - return "" - - def _extract_tags(self, page): - """Extrai as tags da notícia""" - selectors = [ - ".tags a", - ".categories a", - ".post-tags a", - ".article-tags a", - ".keywords a", - "[rel='tag']" - ] - - tags = [] - - for selector in selectors: - try: - elements = page.locator(selector) - if elements.count() > 0: - for i in range(elements.count()): - tag_text = elements.nth(i).inner_text().strip() - if tag_text and tag_text not in tags: - tags.append(tag_text) - - if tags: - logger.info(f"[{self.name}] Tags extracted: {tags}") - break - except Exception as e: - logger.warning(f"[{self.name}] Failed to extract tags with {selector}: {str(e)}") - continue - - return tags \ No newline at end of file diff --git a/spiders/jornalDeBrasilia.py b/spiders/jornalDeBrasilia.py index 2cd90c4..7858e4c 100644 --- a/spiders/jornalDeBrasilia.py +++ b/spiders/jornalDeBrasilia.py @@ -60,12 +60,11 @@ def parse(self, response): """ Extrai URLs de notícias da página inicial """ - # Seletores específicos do Jornal de Brasília selectors = [ - "article a[href]", # Links dentro de artigos - ".news-item a[href]", # Links em itens de notícia - ".headline a[href]", # Links em manchetes - "h2 a[href], h3 a[href]", # Links em títulos + "article a[href]", + ".news-item a[href]", + ".headline a[href]", + "h2 a[href], h3 a[href]", ] seen_urls = set() From cd814a6ae7dc245f838d00c6c3b1e909cac5c1cd Mon Sep 17 00:00:00 2001 From: Eduardo Ferreira Date: Wed, 22 Oct 2025 12:29:45 -0300 Subject: [PATCH 4/5] =?UTF-8?q?fix(playwright):=20substitui=20uso=20incorr?= =?UTF-8?q?eto=20de=20chromium=20ap=C3=B3s=20feedback=20de=20revis=C3=A3o?= =?UTF-8?q?=20de=20PR?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- plays/jornalDeBrasilia.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plays/jornalDeBrasilia.py b/plays/jornalDeBrasilia.py index 56113ae..c22513c 100644 --- a/plays/jornalDeBrasilia.py +++ b/plays/jornalDeBrasilia.py @@ -20,9 +20,8 @@ def pre_run(self): def run(self) -> EntryItem: """Executa a extração principal dos dados da página.""" with sync_playwright() as p: - browser = p.chromium.launch() + browser = self.launch_browser(p, viewport={"width": 1920, "height": 1080}) page = browser.new_page( - user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36" ) page.route( From 9766c9b2f0d8180bdb848706bf8eea26c7ff6fa4 Mon Sep 17 00:00:00 2001 From: Eduardo Ferreira Date: Wed, 22 Oct 2025 12:52:25 -0300 Subject: [PATCH 5/5] =?UTF-8?q?feat(db):=20registrar=20Jornal=20de=20Bras?= =?UTF-8?q?=C3=ADlia=20no=20create=5Fdb=20para=20cria=C3=A7=C3=A3o=20autom?= =?UTF-8?q?=C3=A1tica?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- create_db.py | 1 + 1 file changed, 1 insertion(+) diff --git a/create_db.py b/create_db.py index 617b85b..3232ffe 100644 --- a/create_db.py +++ b/create_db.py @@ -42,6 +42,7 @@ ("Gazeta do Povo", "https://www.gazetadopovo.com.br/", "gazetadopovo"), ("Mais Goiás", "https://www.maisgoias.com.br/", "maisgoias"), ("Aliados Brasil", "https://www.aliadosbrasiloficial.com.br/", "aliadosbrasil"), + ("Jornal de Brasília", "https://www.jornaldebrasilia.com.br/", "jornaldebrasilia"), ] portals_to_add = [] for portal in portals: