Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -31,3 +31,4 @@ __pycache__/
# OS
.DS_Store
Thumbs.db
web/client/package-lock.json
3 changes: 2 additions & 1 deletion plays/items.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
from dataclasses import dataclass
from typing import List
from typing import List, Optional


@dataclass
Expand Down Expand Up @@ -30,6 +30,7 @@ class EntryItem:
description: str | None = None
body: str | None = None
tags: List[str] | None = None
image_url: Optional[str] = None

def __post_init__(self):
if self.ads is None:
Expand Down
54 changes: 46 additions & 8 deletions plays/metropoles.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,9 @@
import time

from playwright.sync_api import sync_playwright

from plays.base import BasePlay
from plays.items import EntryItem
from plog import logger


class MetropolesPlay(BasePlay):
name = "metropoles"

Expand All @@ -25,18 +22,59 @@ def run(self) -> EntryItem:
page.goto(self.url, timeout=180_000)

# Wait for the main content to be visible
page.wait_for_selector("h1", timeout=30000)
try:
page.wait_for_selector("h1", timeout=30000)
except Exception as e:
logger.error(f"[{self.name}] Timeout waiting for h1: {str(e)}")
# Opcional: retornar item vazio ou lançar erro

# Extract article content
entry_title = page.locator("//h1").first.inner_text()

entry_title = ""
try:
entry_title = page.locator("//h1").first.inner_text()
except Exception:
logger.warning(f"[{self.name}] Failed to extract title")

# Extract description from noticiaCabecalho__subtitulo
description = ""
try:
description = page.locator(".noticiaCabecalho__subtitulo").inner_text()
except Exception:
logger.warning(f"[{self.name}] Failed to extract description")

# --- NOVO: EXTRAÇÃO DE IMAGEM (THUMBNAIL) ---
image_url = ""
try:
# Tentativa 1: Meta tag og:image (Padrão Ouro)
image_url = page.locator('meta[property="og:image"]').get_attribute("content")
logger.info(f"[{self.name}] Image found via og:image")
except Exception:
pass

if not image_url:
try:
# Tentativa 2: Tag de link image_src (usada pelo Google)
image_url = page.locator('link[rel="image_src"]').get_attribute("href")
logger.info(f"[{self.name}] Image found via link rel")
except Exception:
pass

if not image_url:
try:
# Tentativa 3: Primeira imagem dentro do article ou figure
# Procura imagens grandes para evitar ícones
imgs = page.locator("article img")
count = imgs.count()
for i in range(count):
src = imgs.nth(i).get_attribute("src")
if src and ("jpg" in src or "png" in src or "webp" in src):
image_url = src
logger.info(f"[{self.name}] Image found via article img fallback")
break
except Exception:
pass
# --- FIM DA EXTRAÇÃO DE IMAGEM ---

# Extract article body with multiple attempts
body = ""
try:
Expand Down Expand Up @@ -76,11 +114,11 @@ def run(self) -> EntryItem:
tags.append(tag_text)
except Exception:
logger.warning(f"[{self.name}] Failed to extract tags")

return EntryItem(
title=entry_title,
url=self.url,
description=description,
body=body,
tags=tags,
)
image_url=image_url
)
3 changes: 2 additions & 1 deletion scrape_no_openai.py
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,7 @@ def main():
"description": getattr(entry_item, "description", None),
"scraped_at": datetime.utcnow().isoformat(),
"entry_id": entry.id,
"image_url": getattr(entry_item, "image_url", None),
"ads": [
{"title": ad.title,
"url": ad.url,
Expand All @@ -171,7 +172,7 @@ def main():
for ad in entry_item.ads if ad.is_valid()
]
}

logger.info(f"JSON PREPARED. Image URL: {article_data.get('image_url')}")
timestamp = datetime.utcnow().strftime("%Y%m%d_%H%M%S")
object_name = f"{PORTAL_FOLDER}/{timestamp}_{entry.id}.json"

Expand Down
Loading
Loading