feat(poder360): Criação do spider e scraper do portal poder360

Dockerfile

@@ -13,6 +13,7 @@ RUN /root/.local/bin/pipenv sync --dev
 # Instala minio e browsers no venv
 RUN /root/.local/bin/pipenv install minio==7.2.15
 RUN /root/.local/bin/pipenv install scrapy-playwright==0.0.43
+ENV PLAYWRIGHT_BROWSERS_PATH=/usr/src/.venv/pw-browsers
 RUN /root/.local/bin/pipenv run playwright install --with-deps firefox
 
 # Confirma que playwright foi instalado corretamente
@@ -34,7 +35,8 @@ RUN apt-get update \
 
 ENV PATH=/usr/src/.venv/bin:$PATH \
     PYTHONDONTWRITEBYTECODE=1 \
-    PYTHONUNBUFFERED=1
+    PYTHONUNBUFFERED=1 \
+    PLAYWRIGHT_BROWSERS_PATH=/usr/src/.venv/pw-browsers
 
 WORKDIR /project
 

Makefile

@@ -82,6 +82,9 @@ scrape_uol:
 
 scrape_folha:
 	docker compose exec scraper python scrape_no_openai.py --platform folha.uol.com.br
+
+scrape_poder360:
+	docker compose exec scraper python scrape_no_openai.py --platform poder360.com.br
 
 # Crawler para todos os portais ou específicos
 crawl:
@@ -112,6 +115,9 @@ crawl_ig:
 crawl_folha:
 	docker compose run scraper python crawl.py folhaspider
 
+crawl_poder360:
+	docker compose run scraper python crawl.py poder360
+
 # Workflow completo de coleta de URLs
 crawl_all_working:
 	@echo "Executando crawl de todos os portais funcionais..."
@@ -123,6 +129,7 @@ crawl_all_working:
 	@make crawl_aliadosBrasil
 	@make crawl_ig
 	@make crawl_folha
+	@make crawl_poder360
 	@echo "Crawl de todos os portais concluído!"
 
 # Workflow completo de scraping
@@ -136,6 +143,7 @@ scrape_all_working:
 	@make scrape_r7
 	@make scrape_uol
 	@make scrape_folha
+	@make scrape_poder360
 	@echo "Scraping de todos os portais concluído!"
 
 # Pipeline completo: crawl + scrape
@@ -182,6 +190,7 @@ help:
 	@echo "  make crawl_aliadosBrasil - Coleta URLs do portal AliadosBrasil"
 	@echo "  make crawl_ig          - Coleta URLs do portal IG"
 	@echo "  make crawl_folha       - Coleta URLs do portal Folha"
+	@echo "  make crawl_poder360    - Coleta URLs do portal Poder360"
 	@echo ""
 	@echo "=== COMANDOS DE SCRAPING (Extração de Anúncios) ==="
 	@echo "  make scrape_all_working - Executa scraping de todos os portais funcionais"
@@ -193,6 +202,7 @@ help:
 	@echo "  make scrape_r7          - Scraping do portal R7"
 	@echo "  make scrape_uol         - Scraping do portal UOL"
 	@echo "  make scrape_folha       - Scraping do portal Folha"
+	@echo "  make scrape_poder360   - Scraping do portal Poder360"
 	@echo ""
 	@echo "=== WORKFLOWS COMPLETOS ==="
 	@echo "  make pipeline_complete  - Executa crawl + scraping de todos os portais"
@@ -219,25 +229,24 @@ run-frontend:
 
 # Rodar backend (server-web) com Docker
 run-backend-docker:
-	docker-compose up server-web
+	docker compose up server-web
 
 # Rodar frontend (client-web) com Docker
 run-frontend-docker:
-	docker-compose up client-web
+	docker compose up client-web
 
 # Construir e rodar backend (server-web) com Docker
 build-backend-docker:
-	docker-compose up --build server-web
+	docker compose up --build server-web
 
 # Construir e rodar frontend (client-web) com Docker
 build-frontend-docker:
-	docker-compose up --build client-web
+	docker compose up --build client-web
 
 # Rodar todos os serviços (backend, frontend, minio, db, scraper)
 all:
-	docker-compose up -d
+	docker compose up -d
 
 # Parar todos os serviços
 down:
-	docker-compose down
-
+	docker compose down

create_db.py

@@ -42,6 +42,7 @@
         ("Gazeta do Povo", "https://www.gazetadopovo.com.br/", "gazetadopovo"),
         ("Mais Goiás", "https://www.maisgoias.com.br/", "maisgoias"),
         ("Aliados Brasil", "https://www.aliadosbrasiloficial.com.br/", "aliadosbrasil"),
+        ("Poder360", "https://www.poder360.com.br/", "poder360"),
     ]
     portals_to_add = []
     for portal in portals:

plays/__init__.py

@@ -11,6 +11,7 @@
 from plays.gazetaDoPovo import GazetaDoPovoPlay
 from plays.maisGoias import MaisGoias
 from plays.aliadosBrasil import AliadosBrasilPlay
+from plays.poder360 import Poder360Play
 
 __all__ = [
     ClicRBSPlay,
@@ -26,4 +27,5 @@
     GazetaDoPovoPlay,
     MaisGoias,
     AliadosBrasilPlay,
+    Poder360Play,
 ]

plays/poder360.py

@@ -0,0 +1,133 @@
+
+import time
+from playwright.sync_api import sync_playwright
+from plays.base import BasePlay
+from plays.items import EntryItem
+from plog import logger
+
+class Poder360Play(BasePlay):
+    name = "poder360"
+
+    @classmethod
+    def match(cls, url):
+        """Identifica se a URL é do Poder360"""
+        return "poder360.com.br" in url
+
+    def pre_run(self):
+        """Configurações antes da execução"""
+        pass
+
+    def run(self) -> EntryItem:
+        """Extrai conteúdo da notícia"""
+        with sync_playwright() as p:
+            browser = self.launch_browser(p, viewport={"width": 1920, "height": 1080})
+            page = browser.new_page()
+
+            logger.info(f"[{self.name}] Opening URL '{self.url}'...")
+            page.goto(self.url, timeout=180_000)
+
+            # Adicionar um seletor de espera mais robusto
+            page.wait_for_selector("h1.inner-page-section__title", timeout=30000)
+
+            title = self._extract_title(page)
+            description = self._extract_description(page)
+            body = self._extract_body(page)
+            tags = self._extract_tags(page)
+
+            return EntryItem(
+                title=title,
+                url=self.url,
+                description=description,
+                body=body,
+                tags=tags,
+            )
+
+    def _extract_title(self, page):
+        """Extrai o título da notícia"""
+        selectors = ["h1.inner-page-section__title"]
+
+        for selector in selectors:
+            try:
+                element = page.locator(selector)
+                if element.count() > 0:
+                    title = element.first.inner_text().strip()
+                    if title:
+                        logger.info(f"[{self.name}] Title extracted: {title[:50]}...")
+                        return title
+            except Exception as e:
+                logger.warning(f"[{self.name}] Failed to extract title with {selector}: {str(e)}")
+                continue
+
+        logger.error(f"[{self.name}] Failed to extract title")
+        return ""
+
+    def _extract_description(self, page):
+        """Extrai a descrição/subtítulo da notícia"""
+        selectors = ["h2.inner-page-section__line"]
+
+        for selector in selectors:
+            try:
+                element = page.locator(selector)
+                if element.count() > 0:
+                    description = element.first.inner_text().strip()
+                    if description and len(description) > 10:
+                        logger.info(f"[{self.name}] Description extracted: {description[:50]}...")
+                        return description
+            except Exception as e:
+                logger.warning(f"[{self.name}] Failed to extract description with {selector}: {str(e)}")
+                continue
+
+        logger.info(f"[{self.name}] No description found")
+        return ""
+
+    def _extract_body(self, page):
+        """Extrai o corpo da notícia"""
+        selectors = ["div.inner-page-section__text"]
+
+        for selector in selectors:
+            try:
+                element = page.locator(selector)
+                if element.count() > 0:
+                    # Remover elementos indesejados (anúncios, formulários de inscrição, etc.)
+                    element.evaluate('''
+                        (element) => {
+                            const selectors = ".ads-div, .shortcode-newsletter, .form-subscribe, .box-advertising";
+                            const toRemove = element.querySelectorAll(selectors);
+                            toRemove.forEach(el => el.remove());
+                        }
+                    ''')
+
+                    body = element.inner_text().strip()
+                    if body and len(body) > 100:  # Garantir que tem conteúdo substancial
+                        logger.info(f"[{self.name}] Body extracted ({len(body)} chars)")
+                        return body
+            except Exception as e:
+                logger.warning(f"[{self.name}] Failed to extract body with {selector}: {str(e)}")
+                continue
+
+        logger.error(f"[{self.name}] Failed to extract body")
+        return ""
+
+    def _extract_tags(self, page):
+        """Extrai as tags da notícia"""
+        selectors = [".footer-post__subject-list a"]
+
+        tags = []
+
+        for selector in selectors:
+            try:
+                elements = page.locator(selector)
+                if elements.count() > 0:
+                    for i in range(elements.count()):
+                        tag_text = elements.nth(i).inner_text().strip()
+                        if tag_text and tag_text not in tags:
+                            tags.append(tag_text)
+
+                    if tags:
+                        logger.info(f"[{self.name}] Tags extracted: {tags}")
+                        break
+            except Exception as e:
+                logger.warning(f"[{self.name}] Failed to extract tags with {selector}: {str(e)}")
+                continue
+
+        return tags

spiders/__init__.py

@@ -12,6 +12,7 @@
 from spiders.gazetaDoPovo import GazetaDoPovoSpider
 from spiders.maisGoias import MaisGoiasSpider
 from spiders.aliadosBrasil import AliadosBrasilSpider
+from spiders.poder360 import Poder360Spider
 
 __all__ = [
     BaseSpider,
@@ -28,4 +29,5 @@
     GazetaDoPovoSpider,
     MaisGoiasSpider,
     AliadosBrasilSpider,
+    Poder360Spider,
 ]

spiders/poder360.py

@@ -0,0 +1,100 @@
+
+import scrapy
+from urllib.parse import urlparse
+from spiders.base import BaseSpider
+from spiders.items import URLItem
+
+class Poder360Spider(BaseSpider):
+    name = "poder360"
+    start_urls = ["https://www.poder360.com.br/"]
+    allowed_domains = ["poder360.com.br"]
+
+    custom_settings = {
+        **BaseSpider.custom_settings,
+        "COOKIES_ENABLED": True,
+        "DOWNLOAD_DELAY": 2,
+        "DEFAULT_REQUEST_HEADERS": {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+            "Accept-Language": "pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+        }
+    }
+
+    def allow_url(self, url: str) -> bool:
+        """
+        Filtra apenas URLs de notícias válidas
+        """
+        p = urlparse(url)
+        path = p.path.rstrip('/')
+
+        # Blacklist de seções que não são notícias
+        blacklist = {
+            "/videos", "/podcasts", "/newsletter", "/contato", 
+            "/sobre", "/anuncie", "/assine", "/busca", "/autores",
+            "/eleicoes", "/pesquisas", "/poderdata", "/drive",
+            "/podcasts", "/videos", "/author", "/articulistas",
+            "/conteudo-patrocinado", "/expediente", "/quem-somos",
+            "/anuncie", "/contato", "/perguntas-frequentes",
+            "/principios-editoriais", "/politica-de-privacidade",
+            "/termos-de-uso",
+        }
+
+        # Checa se o path começa com algum item da blacklist
+        for section in blacklist:
+            if path.startswith(section):
+                return False
+
+        # Requer pelo menos 2 segmentos (seção + slug)
+        segments = [seg for seg in path.split('/') if seg]
+        if len(segments) < 2:
+            return False
+
+        # O slug (último segmento) deve ter pelo menos 2 hífens
+        slug = segments[-1]
+        if slug.count('-') < 2:
+            return False
+
+        return True
+
+    def start_requests(self):
+        for url in self.start_urls:
+            yield scrapy.Request(
+                url,
+                callback=self.parse,
+                dont_filter=True,
+                meta={"dont_redirect": True, "handle_httpstatus_list": [403]},
+            )
+
+    def parse(self, response):
+        """
+        Extrai URLs de notícias da página inicial
+        """
+        selectors = [
+            "h2.box-news-list__highlight-subhead a[href]",
+            "h2.box-news-list__subhead a[href]",
+            "h3.box-queue__subhead a[href]",
+            ".box-highlight__list li a[href]",
+        ]
+
+        seen_urls = set()
+
+        for selector in selectors:
+            links = response.css(selector)
+
+            for link in links:
+                url = link.attrib.get("href")
+                if not url:
+                    continue
+
+                # Construir URL absoluta
+                full_url = response.urljoin(url)
+                full_url = full_url.split('#', 1)[0].split('?', 1)[0]
+
+                # Evitar duplicatas
+                if full_url in seen_urls:
+                    continue
+                seen_urls.add(full_url)
+
+                # Filtrar URLs válidos
+                if self.allow_url(full_url):
+                    yield URLItem(url=full_url)