feat(faxaju): Adiciona Spider e Scraper para o Portal Faxaju

Makefile

@@ -91,6 +91,12 @@ scrape_folha:
 scrape_jornaldaparaiba:
 	docker compose exec scraper python scrape_no_openai.py --platform jornaldaparaiba.com.br
 
+scrape_cidadeverde:
+	docker compose exec scraper python scrape_no_openai.py --platform cidadeverde.com
+
+scrape_faxaju:
+	docker compose exec scraper python scrape_no_openai.py --platform faxaju.com.br
+
 
 # Crawler para todos os portais ou específicos
 crawl:
@@ -124,6 +130,12 @@ crawl_folha:
 crawl_jornaldaparaiba:
 	docker compose run scraper python crawl.py jornaldaparaiba
 
+crawl_cidadeverde:
+	docker compose run scraper python crawl.py cidadeverdespider
+
+crawl_faxaju:
+	docker compose run scraper python crawl.py faxajuspider
+
 # Workflow completo de coleta de URLs
 crawl_all_working:
 	@echo "Executando crawl de todos os portais funcionais..."
@@ -136,6 +148,8 @@ crawl_all_working:
 	@make crawl_ig
 	@make crawl_folha
 	@make crawl_jornaldaparaiba
+	@make crawl_cidadeverde
+	@make crawl_faxaju
 	@echo "Crawl de todos os portais concluído!"
 
 # Workflow completo de scraping
@@ -150,6 +164,8 @@ scrape_all_working:
 	@make scrape_uol
 	@make scrape_folha
 	@make scrape_jornaldaparaiba
+	@make scrape_cidadeverde
+	@make scrape_faxaju
 	@echo "Scraping de todos os portais concluído!"
 
 # Pipeline completo: crawl + scrape
@@ -255,4 +271,4 @@ all:
 
 # Parar todos os serviços
 down:
-	docker compose down
+	docker compose down

create_db.py

@@ -2,14 +2,7 @@
 from sqlalchemy import create_engine
 from sqlalchemy.orm import Session
 
-from models import (
-    Advertisement,
-    Portal,
-    Entry,
-    QueueStatus,
-    URLQueue,
-)
-
+from models import Advertisement, Entry, Portal, QueueStatus, URLQueue
 
 if __name__ == "__main__":
     engine = create_engine(config("DATABASE_URL"))
@@ -43,6 +36,7 @@
         ("Mais Goiás", "https://www.maisgoias.com.br/", "maisgoias"),
         ("Aliados Brasil", "https://www.aliadosbrasiloficial.com.br/", "aliadosbrasil"),
         ("Jornal da Paraíba", "https://jornaldaparaiba.com.br/", "jornaldaparaiba"),
+        ("Faxaju", "https://www.faxaju.com.br/", "faxaju"),
     ]
     portals_to_add = []
     for portal in portals:

plays/__init__.py

@@ -1,17 +1,18 @@
+from plays.aliadosBrasil import AliadosBrasilPlay
 from plays.estadao import EstadaoPlay
+from plays.faxaju import FaxajuPlay
 from plays.folha import FolhaPlay
+from plays.gazetaDoPovo import GazetaDoPovoPlay
 from plays.globo import GloboPlay
 from plays.ig import IGPlay
+from plays.jornaldaparaiba import JornalDaParaibaPlay
+from plays.maisGoias import MaisGoias
 from plays.metropoles import MetropolesPlay
 from plays.r7 import R7Play
 from plays.rbs import ClicRBSPlay
 from plays.terra import TerraPlay
-from plays.veja import VejaPlay
 from plays.uol import UOLPlay
-from plays.gazetaDoPovo import GazetaDoPovoPlay
-from plays.maisGoias import MaisGoias
-from plays.aliadosBrasil import AliadosBrasilPlay
-from plays.jornaldaparaiba import JornalDaParaibaPlay
+from plays.veja import VejaPlay
 
 __all__ = [
     ClicRBSPlay,
@@ -28,4 +29,5 @@
     MaisGoias,
     AliadosBrasilPlay,
     JornalDaParaibaPlay,
+    FaxajuPlay,
 ]

plays/faxaju.py

@@ -0,0 +1,185 @@
+from playwright.sync_api import sync_playwright
+
+from plays.base import BasePlay
+from plays.items import EntryItem
+from plog import logger
+
+
+class FaxajuPlay(BasePlay):
+    name = "faxaju"
+
+    @classmethod
+    def match(cls, url: str) -> bool:
+        return "faxaju.com.br" in url
+
+    def pre_run(self):
+        # Nenhuma preparação especial conhecida (sem login, etc.)
+        pass
+
+    def run(self) -> EntryItem:
+        with sync_playwright() as p:
+            browser = self.launch_browser(p, viewport={"width": 1600, "height": 1000})
+            page = browser.new_page()
+
+            # Bloqueia recursos pesados e domínios de anúncios para acelerar
+            blocked_domains = (
+                "doubleclick.net",
+                "googlesyndication.com",
+                "google-analytics.com",
+                "analytics.google.com",
+                "clarity.ms",
+                "tailtarget.com",
+                "flashtalking.com",
+                "googletagmanager.com",
+            )
+
+            def block_ads(route, request):
+                try:
+                    if request.resource_type in {"image", "media", "font", "stylesheet"}:
+                        return route.abort()
+                    if any(d in request.url for d in blocked_domains):
+                        return route.abort()
+                except Exception:
+                    pass
+                return route.continue_()
+
+            page.route("**/*", block_ads)
+            page.set_default_navigation_timeout(60_000)
+            page.set_default_timeout(20_000)
+
+            logger.info(f"[{self.name}] Opening URL '{self.url}'...")
+            page.goto(self.url, wait_until="domcontentloaded", timeout=60_000)
+
+            # Tenta sincronizar com o título principal
+            try:
+                page.wait_for_selector("h1", timeout=15_000)
+            except Exception:
+                pass
+
+            # Título
+            title = ""
+            for selector in [
+                "section h3"
+                "section"
+            ]:
+                try:
+                    if selector.startswith("meta["):
+                        el = page.locator(selector)
+                        if el.count() > 0:
+                            content = el.first.get_attribute("content") or ""
+                            content = content.strip()
+                            if content:
+                                title = content
+                                logger.info(f"[{self.name}] Title from '{selector}'")
+                                break
+                        continue
+
+                    el = page.locator(selector)
+                    if el.count() > 0:
+                        title = el.first.inner_text().strip()
+                        if title:
+                            logger.info(f"[{self.name}] Title from '{selector}'")
+                            break
+                except Exception:
+                    continue
+
+            if not title:
+                try:
+                    title = page.title().strip()
+                except Exception:
+                    title = ""
+
+            # Descrição
+            description = ""
+            for selector in [
+                "article h2",
+                "header h2",
+            ]:
+                try:
+                    if selector.startswith("meta["):
+                        el = page.locator(selector)
+                        if el.count() > 0:
+                            content = el.first.get_attribute("content") or ""
+                            content = content.strip()
+                            if content:
+                                description = content
+                                logger.info(f"[{self.name}] Description from '{selector}'")
+                                break
+                        continue
+
+                    el = page.locator(selector)
+                    if el.count() > 0:
+                        description = el.first.inner_text().strip()
+                        if description:
+                            logger.info(f"[{self.name}] Description from '{selector}'")
+                            break
+                except Exception:
+                    continue
+
+            # Corpo da notícia
+            body = ""
+            for selector in [
+                ".elementor-widget-container p"
+                "section p",
+                "p",
+            ]:
+                try:
+                    els = page.locator(selector)
+                    count = els.count()
+                    if count == 0:
+                        continue
+
+                    if count > 1:
+                        parts = []
+                        for i in range(count):
+                            t = els.nth(i).inner_text().strip()
+                            # Ignora parágrafos que começam com "Foto: "
+                            if t and not t.startswith("Foto: "):
+                                parts.append(t)
+                        body = "\n\n".join(parts).strip()
+                    else:
+                        first_text = els.first.inner_text().strip()
+                        if first_text.startswith("Foto: "):
+                            first_text = ""
+                        body = first_text
+
+                    if body and len(body) > 100:
+                        logger.info(
+                            f"[{self.name}] Body from '{selector}' (len={len(body)})"
+                        )
+                        break
+                except Exception:
+                    continue
+
+            # Tags
+            tags = []
+            for selector in [
+                ".tags a",
+                ".tags li a",
+                ".tags li",
+            ]:
+                try:
+                    els = page.locator(selector)
+                    n = els.count()
+                    if n == 0:
+                        continue
+                    for i in range(n):
+                        t = els.nth(i).inner_text().strip()
+                        if t and t not in tags:
+                            tags.append(t)
+                    if tags:
+                        logger.info(
+                            f"[{self.name}] {len(tags)} tags from '{selector}'"
+                        )
+                        break
+                except Exception:
+                    continue
+
+            return EntryItem(
+                title=title,
+                url=self.url,
+                description=description,
+                body=body,
+                tags=tags,
+            )
+

spiders/__init__.py

@@ -1,18 +1,19 @@
+from spiders.aliadosBrasil import AliadosBrasilSpider
 from spiders.base import BaseSpider
 from spiders.estadao import EstadaoSpider
+from spiders.faxaju import FaxajuSpider
 from spiders.folha import FolhaSpider
+from spiders.gazetaDoPovo import GazetaDoPovoSpider
 from spiders.globo import GloboSpider
 from spiders.ig import IGSpider
+from spiders.jornaldaparaiba import JornalDaParaibaSpider
+from spiders.maisGoias import MaisGoiasSpider
 from spiders.metropoles import MetropolesSpider
 from spiders.r7 import R7Spider
 from spiders.rbs import RBSSpider
-from spiders.veja import VejaSpider
 from spiders.terra import TerraSpider
 from spiders.uol import UOLSpider
-from spiders.gazetaDoPovo import GazetaDoPovoSpider
-from spiders.maisGoias import MaisGoiasSpider
-from spiders.aliadosBrasil import AliadosBrasilSpider
-from spiders.jornaldaparaiba import JornalDaParaibaSpider
+from spiders.veja import VejaSpider
 
 __all__ = [
     BaseSpider,
@@ -30,4 +31,5 @@
     MaisGoiasSpider,
     AliadosBrasilSpider,
     JornalDaParaibaSpider,
+    FaxajuSpider,
 ]