diff --git a/Makefile b/Makefile index a7f7ce2..6d01d07 100644 --- a/Makefile +++ b/Makefile @@ -91,6 +91,12 @@ scrape_folha: scrape_jornaldaparaiba: docker compose exec scraper python scrape_no_openai.py --platform jornaldaparaiba.com.br +scrape_cidadeverde: + docker compose exec scraper python scrape_no_openai.py --platform cidadeverde.com + +scrape_faxaju: + docker compose exec scraper python scrape_no_openai.py --platform faxaju.com.br + # Crawler para todos os portais ou específicos crawl: @@ -124,6 +130,12 @@ crawl_folha: crawl_jornaldaparaiba: docker compose run scraper python crawl.py jornaldaparaiba +crawl_cidadeverde: + docker compose run scraper python crawl.py cidadeverdespider + +crawl_faxaju: + docker compose run scraper python crawl.py faxajuspider + # Workflow completo de coleta de URLs crawl_all_working: @echo "Executando crawl de todos os portais funcionais..." @@ -136,6 +148,8 @@ crawl_all_working: @make crawl_ig @make crawl_folha @make crawl_jornaldaparaiba + @make crawl_cidadeverde + @make crawl_faxaju @echo "Crawl de todos os portais concluído!" # Workflow completo de scraping @@ -150,6 +164,8 @@ scrape_all_working: @make scrape_uol @make scrape_folha @make scrape_jornaldaparaiba + @make scrape_cidadeverde + @make scrape_faxaju @echo "Scraping de todos os portais concluído!" # Pipeline completo: crawl + scrape @@ -255,4 +271,4 @@ all: # Parar todos os serviços down: - docker compose down \ No newline at end of file + docker compose down diff --git a/create_db.py b/create_db.py index 7ce5701..52afa85 100644 --- a/create_db.py +++ b/create_db.py @@ -2,14 +2,7 @@ from sqlalchemy import create_engine from sqlalchemy.orm import Session -from models import ( - Advertisement, - Portal, - Entry, - QueueStatus, - URLQueue, -) - +from models import Advertisement, Entry, Portal, QueueStatus, URLQueue if __name__ == "__main__": engine = create_engine(config("DATABASE_URL")) @@ -43,6 +36,7 @@ ("Mais Goiás", "https://www.maisgoias.com.br/", "maisgoias"), ("Aliados Brasil", "https://www.aliadosbrasiloficial.com.br/", "aliadosbrasil"), ("Jornal da Paraíba", "https://jornaldaparaiba.com.br/", "jornaldaparaiba"), + ("Faxaju", "https://www.faxaju.com.br/", "faxaju"), ] portals_to_add = [] for portal in portals: diff --git a/plays/__init__.py b/plays/__init__.py index 8189816..62ac8d1 100644 --- a/plays/__init__.py +++ b/plays/__init__.py @@ -1,17 +1,18 @@ +from plays.aliadosBrasil import AliadosBrasilPlay from plays.estadao import EstadaoPlay +from plays.faxaju import FaxajuPlay from plays.folha import FolhaPlay +from plays.gazetaDoPovo import GazetaDoPovoPlay from plays.globo import GloboPlay from plays.ig import IGPlay +from plays.jornaldaparaiba import JornalDaParaibaPlay +from plays.maisGoias import MaisGoias from plays.metropoles import MetropolesPlay from plays.r7 import R7Play from plays.rbs import ClicRBSPlay from plays.terra import TerraPlay -from plays.veja import VejaPlay from plays.uol import UOLPlay -from plays.gazetaDoPovo import GazetaDoPovoPlay -from plays.maisGoias import MaisGoias -from plays.aliadosBrasil import AliadosBrasilPlay -from plays.jornaldaparaiba import JornalDaParaibaPlay +from plays.veja import VejaPlay __all__ = [ ClicRBSPlay, @@ -28,4 +29,5 @@ MaisGoias, AliadosBrasilPlay, JornalDaParaibaPlay, + FaxajuPlay, ] diff --git a/plays/faxaju.py b/plays/faxaju.py new file mode 100644 index 0000000..52f266c --- /dev/null +++ b/plays/faxaju.py @@ -0,0 +1,185 @@ +from playwright.sync_api import sync_playwright + +from plays.base import BasePlay +from plays.items import EntryItem +from plog import logger + + +class FaxajuPlay(BasePlay): + name = "faxaju" + + @classmethod + def match(cls, url: str) -> bool: + return "faxaju.com.br" in url + + def pre_run(self): + # Nenhuma preparação especial conhecida (sem login, etc.) + pass + + def run(self) -> EntryItem: + with sync_playwright() as p: + browser = self.launch_browser(p, viewport={"width": 1600, "height": 1000}) + page = browser.new_page() + + # Bloqueia recursos pesados e domínios de anúncios para acelerar + blocked_domains = ( + "doubleclick.net", + "googlesyndication.com", + "google-analytics.com", + "analytics.google.com", + "clarity.ms", + "tailtarget.com", + "flashtalking.com", + "googletagmanager.com", + ) + + def block_ads(route, request): + try: + if request.resource_type in {"image", "media", "font", "stylesheet"}: + return route.abort() + if any(d in request.url for d in blocked_domains): + return route.abort() + except Exception: + pass + return route.continue_() + + page.route("**/*", block_ads) + page.set_default_navigation_timeout(60_000) + page.set_default_timeout(20_000) + + logger.info(f"[{self.name}] Opening URL '{self.url}'...") + page.goto(self.url, wait_until="domcontentloaded", timeout=60_000) + + # Tenta sincronizar com o título principal + try: + page.wait_for_selector("h1", timeout=15_000) + except Exception: + pass + + # Título + title = "" + for selector in [ + "section h3" + "section" + ]: + try: + if selector.startswith("meta["): + el = page.locator(selector) + if el.count() > 0: + content = el.first.get_attribute("content") or "" + content = content.strip() + if content: + title = content + logger.info(f"[{self.name}] Title from '{selector}'") + break + continue + + el = page.locator(selector) + if el.count() > 0: + title = el.first.inner_text().strip() + if title: + logger.info(f"[{self.name}] Title from '{selector}'") + break + except Exception: + continue + + if not title: + try: + title = page.title().strip() + except Exception: + title = "" + + # Descrição + description = "" + for selector in [ + "article h2", + "header h2", + ]: + try: + if selector.startswith("meta["): + el = page.locator(selector) + if el.count() > 0: + content = el.first.get_attribute("content") or "" + content = content.strip() + if content: + description = content + logger.info(f"[{self.name}] Description from '{selector}'") + break + continue + + el = page.locator(selector) + if el.count() > 0: + description = el.first.inner_text().strip() + if description: + logger.info(f"[{self.name}] Description from '{selector}'") + break + except Exception: + continue + + # Corpo da notícia + body = "" + for selector in [ + ".elementor-widget-container p" + "section p", + "p", + ]: + try: + els = page.locator(selector) + count = els.count() + if count == 0: + continue + + if count > 1: + parts = [] + for i in range(count): + t = els.nth(i).inner_text().strip() + # Ignora parágrafos que começam com "Foto: " + if t and not t.startswith("Foto: "): + parts.append(t) + body = "\n\n".join(parts).strip() + else: + first_text = els.first.inner_text().strip() + if first_text.startswith("Foto: "): + first_text = "" + body = first_text + + if body and len(body) > 100: + logger.info( + f"[{self.name}] Body from '{selector}' (len={len(body)})" + ) + break + except Exception: + continue + + # Tags + tags = [] + for selector in [ + ".tags a", + ".tags li a", + ".tags li", + ]: + try: + els = page.locator(selector) + n = els.count() + if n == 0: + continue + for i in range(n): + t = els.nth(i).inner_text().strip() + if t and t not in tags: + tags.append(t) + if tags: + logger.info( + f"[{self.name}] {len(tags)} tags from '{selector}'" + ) + break + except Exception: + continue + + return EntryItem( + title=title, + url=self.url, + description=description, + body=body, + tags=tags, + ) + diff --git a/spiders/__init__.py b/spiders/__init__.py index d2bb555..144056d 100644 --- a/spiders/__init__.py +++ b/spiders/__init__.py @@ -1,18 +1,19 @@ +from spiders.aliadosBrasil import AliadosBrasilSpider from spiders.base import BaseSpider from spiders.estadao import EstadaoSpider +from spiders.faxaju import FaxajuSpider from spiders.folha import FolhaSpider +from spiders.gazetaDoPovo import GazetaDoPovoSpider from spiders.globo import GloboSpider from spiders.ig import IGSpider +from spiders.jornaldaparaiba import JornalDaParaibaSpider +from spiders.maisGoias import MaisGoiasSpider from spiders.metropoles import MetropolesSpider from spiders.r7 import R7Spider from spiders.rbs import RBSSpider -from spiders.veja import VejaSpider from spiders.terra import TerraSpider from spiders.uol import UOLSpider -from spiders.gazetaDoPovo import GazetaDoPovoSpider -from spiders.maisGoias import MaisGoiasSpider -from spiders.aliadosBrasil import AliadosBrasilSpider -from spiders.jornaldaparaiba import JornalDaParaibaSpider +from spiders.veja import VejaSpider __all__ = [ BaseSpider, @@ -30,4 +31,5 @@ MaisGoiasSpider, AliadosBrasilSpider, JornalDaParaibaSpider, + FaxajuSpider, ] diff --git a/spiders/faxaju.py b/spiders/faxaju.py new file mode 100644 index 0000000..08ef79b --- /dev/null +++ b/spiders/faxaju.py @@ -0,0 +1,87 @@ +import re + +import scrapy + +from spiders.base import BaseSpider +from spiders.items import URLItem + + +class FaxajuSpider(BaseSpider): + name = "faxajuspider" + start_urls = ["https://www.faxaju.com.br/"] + allowed_domains = ["faxaju.com.br", "www.faxaju.com.br"] + + BLACKLISTED_SECTIONS = { + + } + + def allow_url(self, url: str) -> bool: + """ + Filtra URLs de matérias do Faxaju de forma genérica. + """ + if not url: + return False + + # Remove fragmentos e query string + url = url.split("#", 1)[0].split("?", 1)[0] + + # Garante que é do domínio alvo + if not re.match(r"https?://(www\.)?faxaju\.com\.br/", url): + return False + + path = re.sub(r"https?://[^/]+", "", url) + segments = [s for s in path.strip("/").split("/") if s] + + # Muito curto normalmente significa home ou seção agregadora + if len(segments) < 2: + return False + + # Filtra seções que não são matérias + blacklist_prefixes = {p.lower() for p in self.BLACKLISTED_SECTIONS} + for seg in segments: + seg_low = seg.lower() + if any(seg_low.startswith(pref) for pref in blacklist_prefixes): + return False + + slug = segments[-1] + + # Exige slug minimamente descritivo: + # - termina com .html OU + # - possui hífens suficientes para parecer título de matéria + if slug.endswith(".html"): + base_slug = slug[:-5] + else: + base_slug = slug + + return base_slug.count("-") >= 2 and len(base_slug) > 20 + + def parse(self, response): + """ + Coleta URLs de notícias da página inicial (sem crawling recursivo). + """ + seen = set() + + selectors = [ + "a[href*='faxaju.com.br']", + "a[href^='/']", + ] + + for sel in selectors: + for entry in response.css(sel): + href = entry.attrib.get("href") + if not href: + continue + + url = response.urljoin(href).split("?", 1)[0].split("#", 1)[0] + + if url in seen: + continue + + if self.allow_url(url): + seen.add(url) + yield URLItem(url=url) + + self.logger.info( + f"Collected {len(seen)} unique news URLs from Faxaju" + ) +