From 762469c5b3a75c3ef8e5c006c56e1b9ee5223716 Mon Sep 17 00:00:00 2001 From: mateus9levy Date: Wed, 8 Oct 2025 01:50:09 -0300 Subject: [PATCH 1/2] feat/adiciona portal360 --- Makefile | 21 +++++--- create_db.py | 1 + plays/__init__.py | 2 + plays/poder360.py | 127 ++++++++++++++++++++++++++++++++++++++++++++ spiders/__init__.py | 2 + spiders/poder360.py | 100 ++++++++++++++++++++++++++++++++++ 6 files changed, 246 insertions(+), 7 deletions(-) create mode 100644 plays/poder360.py create mode 100644 spiders/poder360.py diff --git a/Makefile b/Makefile index 12663c4..43cc38b 100644 --- a/Makefile +++ b/Makefile @@ -82,6 +82,9 @@ scrape_uol: scrape_folha: docker compose exec scraper python scrape_no_openai.py --platform folha.uol.com.br + +scrape_poder360: + docker compose exec scraper python scrape_no_openai.py --platform poder360.com.br # Crawler para todos os portais ou específicos crawl: @@ -112,6 +115,9 @@ crawl_ig: crawl_folha: docker compose run scraper python crawl.py folhaspider +crawl_poder360: + docker compose run scraper python crawl.py poder360 + # Workflow completo de coleta de URLs crawl_all_working: @echo "Executando crawl de todos os portais funcionais..." @@ -123,6 +129,7 @@ crawl_all_working: @make crawl_aliadosBrasil @make crawl_ig @make crawl_folha + @make crawl_poder360 @echo "Crawl de todos os portais concluído!" # Workflow completo de scraping @@ -136,6 +143,7 @@ scrape_all_working: @make scrape_r7 @make scrape_uol @make scrape_folha + @make scrape_poder360 @echo "Scraping de todos os portais concluído!" # Pipeline completo: crawl + scrape @@ -219,25 +227,24 @@ run-frontend: # Rodar backend (server-web) com Docker run-backend-docker: - docker-compose up server-web + docker compose up server-web # Rodar frontend (client-web) com Docker run-frontend-docker: - docker-compose up client-web + docker compose up client-web # Construir e rodar backend (server-web) com Docker build-backend-docker: - docker-compose up --build server-web + docker compose up --build server-web # Construir e rodar frontend (client-web) com Docker build-frontend-docker: - docker-compose up --build client-web + docker compose up --build client-web # Rodar todos os serviços (backend, frontend, minio, db, scraper) all: - docker-compose up -d + docker compose up -d # Parar todos os serviços down: - docker-compose down - + docker compose down \ No newline at end of file diff --git a/create_db.py b/create_db.py index 617b85b..3db0d39 100644 --- a/create_db.py +++ b/create_db.py @@ -42,6 +42,7 @@ ("Gazeta do Povo", "https://www.gazetadopovo.com.br/", "gazetadopovo"), ("Mais Goiás", "https://www.maisgoias.com.br/", "maisgoias"), ("Aliados Brasil", "https://www.aliadosbrasiloficial.com.br/", "aliadosbrasil"), + ("Poder360", "https://www.poder360.com.br/", "poder360"), ] portals_to_add = [] for portal in portals: diff --git a/plays/__init__.py b/plays/__init__.py index 338fe28..a73f45f 100644 --- a/plays/__init__.py +++ b/plays/__init__.py @@ -11,6 +11,7 @@ from plays.gazetaDoPovo import GazetaDoPovoPlay from plays.maisGoias import MaisGoias from plays.aliadosBrasil import AliadosBrasilPlay +from plays.poder360 import Poder360Play __all__ = [ ClicRBSPlay, @@ -26,4 +27,5 @@ GazetaDoPovoPlay, MaisGoias, AliadosBrasilPlay, + Poder360Play, ] diff --git a/plays/poder360.py b/plays/poder360.py new file mode 100644 index 0000000..acd369e --- /dev/null +++ b/plays/poder360.py @@ -0,0 +1,127 @@ + +import time +from playwright.sync_api import sync_playwright +from plays.base import BasePlay +from plays.items import EntryItem +from plog import logger + +class Poder360Play(BasePlay): + name = "poder360" + + @classmethod + def match(cls, url): + """Identifica se a URL é do Poder360""" + return "poder360.com.br" in url + + def pre_run(self): + """Configurações antes da execução""" + pass + + def run(self) -> EntryItem: + """Extrai conteúdo da notícia""" + with sync_playwright() as p: + browser = self.launch_browser(p, viewport={"width": 1920, "height": 1080}) + page = browser.new_page() + + logger.info(f"[{self.name}] Opening URL '{self.url}'...") + page.goto(self.url, timeout=180_000) + + # Adicionar um seletor de espera mais robusto + page.wait_for_selector("h1.inner-page-section__title", timeout=30000) + + title = self._extract_title(page) + description = self._extract_description(page) + body = self._extract_body(page) + tags = self._extract_tags(page) + + return EntryItem( + title=title, + url=self.url, + description=description, + body=body, + tags=tags, + ) + + def _extract_title(self, page): + """Extrai o título da notícia""" + selectors = ["h1.inner-page-section__title"] + + for selector in selectors: + try: + element = page.locator(selector) + if element.count() > 0: + title = element.first.inner_text().strip() + if title: + logger.info(f"[{self.name}] Title extracted: {title[:50]}...") + return title + except Exception as e: + logger.warning(f"[{self.name}] Failed to extract title with {selector}: {str(e)}") + continue + + logger.error(f"[{self.name}] Failed to extract title") + return "" + + def _extract_description(self, page): + """Extrai a descrição/subtítulo da notícia""" + selectors = ["h2.inner-page-section__line"] + + for selector in selectors: + try: + element = page.locator(selector) + if element.count() > 0: + description = element.first.inner_text().strip() + if description and len(description) > 10: + logger.info(f"[{self.name}] Description extracted: {description[:50]}...") + return description + except Exception as e: + logger.warning(f"[{self.name}] Failed to extract description with {selector}: {str(e)}") + continue + + logger.info(f"[{self.name}] No description found") + return "" + + def _extract_body(self, page): + """Extrai o corpo da notícia""" + selectors = ["div.inner-page-section__text"] + + for selector in selectors: + try: + element = page.locator(selector) + if element.count() > 0: + # Remover elementos indesejados (anúncios, formulários de inscrição, etc.) + element.locator(".ads-div, .shortcode-newsletter, .form-subscribe, .box-advertising").evaluate_all("el => el.remove()") + + body = element.inner_text().strip() + if body and len(body) > 100: # Garantir que tem conteúdo substancial + logger.info(f"[{self.name}] Body extracted ({len(body)} chars)") + return body + except Exception as e: + logger.warning(f"[{self.name}] Failed to extract body with {selector}: {str(e)}") + continue + + logger.error(f"[{self.name}] Failed to extract body") + return "" + + def _extract_tags(self, page): + """Extrai as tags da notícia""" + selectors = [".footer-post__subject-list a"] + + tags = [] + + for selector in selectors: + try: + elements = page.locator(selector) + if elements.count() > 0: + for i in range(elements.count()): + tag_text = elements.nth(i).inner_text().strip() + if tag_text and tag_text not in tags: + tags.append(tag_text) + + if tags: + logger.info(f"[{self.name}] Tags extracted: {tags}") + break + except Exception as e: + logger.warning(f"[{self.name}] Failed to extract tags with {selector}: {str(e)}") + continue + + return tags diff --git a/spiders/__init__.py b/spiders/__init__.py index ecb5e1f..08aa487 100644 --- a/spiders/__init__.py +++ b/spiders/__init__.py @@ -12,6 +12,7 @@ from spiders.gazetaDoPovo import GazetaDoPovoSpider from spiders.maisGoias import MaisGoiasSpider from spiders.aliadosBrasil import AliadosBrasilSpider +from spiders.poder360 import Poder360Spider __all__ = [ BaseSpider, @@ -28,4 +29,5 @@ GazetaDoPovoSpider, MaisGoiasSpider, AliadosBrasilSpider, + Poder360Spider, ] diff --git a/spiders/poder360.py b/spiders/poder360.py new file mode 100644 index 0000000..d32d6d9 --- /dev/null +++ b/spiders/poder360.py @@ -0,0 +1,100 @@ + +import scrapy +from urllib.parse import urlparse +from spiders.base import BaseSpider +from spiders.items import URLItem + +class Poder360Spider(BaseSpider): + name = "poder360" + start_urls = ["https://www.poder360.com.br/"] + allowed_domains = ["poder360.com.br"] + + custom_settings = { + **BaseSpider.custom_settings, + "COOKIES_ENABLED": True, + "DOWNLOAD_DELAY": 2, + "DEFAULT_REQUEST_HEADERS": { + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "Accept-Language": "pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7", + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36", + } + } + + def allow_url(self, url: str) -> bool: + """ + Filtra apenas URLs de notícias válidas + """ + p = urlparse(url) + path = p.path.rstrip('/') + + # Blacklist de seções que não são notícias + blacklist = { + "/videos", "/podcasts", "/newsletter", "/contato", + "/sobre", "/anuncie", "/assine", "/busca", "/autores", + "/eleicoes", "/pesquisas", "/poderdata", "/drive", + "/podcasts", "/videos", "/author", "/articulistas", + "/conteudo-patrocinado", "/expediente", "/quem-somos", + "/anuncie", "/contato", "/perguntas-frequentes", + "/principios-editoriais", "/politica-de-privacidade", + "/termos-de-uso", + } + + # Checa se o path começa com algum item da blacklist + for section in blacklist: + if path.startswith(section): + return False + + # Requer pelo menos 2 segmentos (seção + slug) + segments = [seg for seg in path.split('/') if seg] + if len(segments) < 2: + return False + + # O slug (último segmento) deve ter pelo menos 2 hífens + slug = segments[-1] + if slug.count('-') < 2: + return False + + return True + + def start_requests(self): + for url in self.start_urls: + yield scrapy.Request( + url, + callback=self.parse, + dont_filter=True, + meta={"dont_redirect": True, "handle_httpstatus_list": [403]}, + ) + + def parse(self, response): + """ + Extrai URLs de notícias da página inicial + """ + selectors = [ + "h2.box-news-list__highlight-subhead a[href]", + "h2.box-news-list__subhead a[href]", + "h3.box-queue__subhead a[href]", + ".box-highlight__list li a[href]", + ] + + seen_urls = set() + + for selector in selectors: + links = response.css(selector) + + for link in links: + url = link.attrib.get("href") + if not url: + continue + + # Construir URL absoluta + full_url = response.urljoin(url) + full_url = full_url.split('#', 1)[0].split('?', 1)[0] + + # Evitar duplicatas + if full_url in seen_urls: + continue + seen_urls.add(full_url) + + # Filtrar URLs válidos + if self.allow_url(full_url): + yield URLItem(url=full_url) From 7930a91fa050e9fca70279bc1e2afed556c8bb7f Mon Sep 17 00:00:00 2001 From: mateus9levy Date: Tue, 21 Oct 2025 22:10:26 -0300 Subject: [PATCH 2/2] =?UTF-8?q?fix:=20Corrige=20um=20erro=20de=20extra?= =?UTF-8?q?=C3=A7=C3=A3o=20no=20corpo=20da=20not=C3=ADcia=20do=20Poder360.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 4 +++- Makefile | 2 ++ plays/poder360.py | 8 +++++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1871764..97d34ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,6 +13,7 @@ RUN /root/.local/bin/pipenv sync --dev # Instala minio e browsers no venv RUN /root/.local/bin/pipenv install minio==7.2.15 RUN /root/.local/bin/pipenv install scrapy-playwright==0.0.43 +ENV PLAYWRIGHT_BROWSERS_PATH=/usr/src/.venv/pw-browsers RUN /root/.local/bin/pipenv run playwright install --with-deps firefox # Confirma que playwright foi instalado corretamente @@ -34,7 +35,8 @@ RUN apt-get update \ ENV PATH=/usr/src/.venv/bin:$PATH \ PYTHONDONTWRITEBYTECODE=1 \ - PYTHONUNBUFFERED=1 + PYTHONUNBUFFERED=1 \ + PLAYWRIGHT_BROWSERS_PATH=/usr/src/.venv/pw-browsers WORKDIR /project diff --git a/Makefile b/Makefile index 43cc38b..4cdd6a1 100644 --- a/Makefile +++ b/Makefile @@ -190,6 +190,7 @@ help: @echo " make crawl_aliadosBrasil - Coleta URLs do portal AliadosBrasil" @echo " make crawl_ig - Coleta URLs do portal IG" @echo " make crawl_folha - Coleta URLs do portal Folha" + @echo " make crawl_poder360 - Coleta URLs do portal Poder360" @echo "" @echo "=== COMANDOS DE SCRAPING (Extração de Anúncios) ===" @echo " make scrape_all_working - Executa scraping de todos os portais funcionais" @@ -201,6 +202,7 @@ help: @echo " make scrape_r7 - Scraping do portal R7" @echo " make scrape_uol - Scraping do portal UOL" @echo " make scrape_folha - Scraping do portal Folha" + @echo " make scrape_poder360 - Scraping do portal Poder360" @echo "" @echo "=== WORKFLOWS COMPLETOS ===" @echo " make pipeline_complete - Executa crawl + scraping de todos os portais" diff --git a/plays/poder360.py b/plays/poder360.py index acd369e..f1c0379 100644 --- a/plays/poder360.py +++ b/plays/poder360.py @@ -89,7 +89,13 @@ def _extract_body(self, page): element = page.locator(selector) if element.count() > 0: # Remover elementos indesejados (anúncios, formulários de inscrição, etc.) - element.locator(".ads-div, .shortcode-newsletter, .form-subscribe, .box-advertising").evaluate_all("el => el.remove()") + element.evaluate(''' + (element) => { + const selectors = ".ads-div, .shortcode-newsletter, .form-subscribe, .box-advertising"; + const toRemove = element.querySelectorAll(selectors); + toRemove.forEach(el => el.remove()); + } + ''') body = element.inner_text().strip() if body and len(body) > 100: # Garantir que tem conteúdo substancial