diff --git a/Dockerfile b/Dockerfile index 1871764..80d456f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,36 +1,45 @@ -# builder +# ========== builder ========== FROM python:3.12.3 AS builder RUN pip install --user pipenv==2023.12.1 ENV PIPENV_VENV_IN_PROJECT=1 +# onde os binários dos browsers ficarão (fora do home) +ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright COPY Pipfile Pipfile.lock /usr/src/ WORKDIR /usr/src -# Instala dependências com pipenv RUN /root/.local/bin/pipenv sync --dev - -# Instala minio e browsers no venv RUN /root/.local/bin/pipenv install minio==7.2.15 RUN /root/.local/bin/pipenv install scrapy-playwright==0.0.43 -RUN /root/.local/bin/pipenv run playwright install --with-deps firefox -# Confirma que playwright foi instalado corretamente -RUN /usr/src/.venv/bin/python -c "import playwright; print(playwright)" +# baixa SOMENTE os browsers (sem --with-deps aqui) +RUN /root/.local/bin/pipenv run playwright install firefox + +# sanity check +RUN /usr/src/.venv/bin/python -c "import playwright; print('Playwright OK')" + -# runtime +# ========== runtime ========== FROM python:3.12.3-slim AS runtime -COPY --from=builder /usr/src/.venv/ /usr/src/.venv/ +# mesmo path de browsers no runtime +ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ +# dependências do Firefox +RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates fonts-liberation libasound2 libatk-bridge2.0-0 libatk1.0-0 \ - libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgbm1 libgcc1 \ - libglib2.0-0 libgtk-3-0 libnspr4 libnss3 libpango-1.0-0 libstdc++6 libx11-6 \ - libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 \ - libxi6 libxrandr2 libxrender1 libxss1 libxtst6 wget xdg-utils \ - && rm -rf /var/lib/apt/lists/* + libc6 libcairo2 libcups2 libdbus-1-3 libdbus-glib-1-2 libexpat1 libfontconfig1 \ + libgbm1 libgcc1 libglib2.0-0 libgtk-3-0 libnspr4 libnss3 libpango-1.0-0 \ + libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 \ + libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxshmfence1 \ + libxss1 libxtst6 wget xdg-utils \ + && rm -rf /var/lib/apt/lists/* + +# venv do app +COPY --from=builder /usr/src/.venv/ /usr/src/.venv/ +# binários dos browsers (copiados do builder) +COPY --from=builder /ms-playwright/ /ms-playwright/ ENV PATH=/usr/src/.venv/bin:$PATH \ PYTHONDONTWRITEBYTECODE=1 \ @@ -40,7 +49,7 @@ WORKDIR /project RUN groupadd -r scraper && \ useradd --no-log-init -r -g scraper -d /project scraper && \ - chown -R scraper:scraper /project /usr/src/.venv + chown -R scraper:scraper /project /usr/src/.venv /ms-playwright COPY --chown=scraper:scraper . /project @@ -49,5 +58,4 @@ USER scraper HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ CMD python -c "import sys; sys.exit(0)" - -CMD ["ipython"] +CMD ["ipython"] \ No newline at end of file diff --git a/Makefile b/Makefile index 12663c4..f501d2f 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: start scrape scrape_no_openai crawl crawl_metropoles init_db migrate_db setup bash env stop wait-for-db prune +.PHONY: start scrape scrape_no_openai crawl crawl_metropoles init_db migrate_db setup bash env stop wait-for-db prune test_playwright # Configuração do ambiente env: @@ -62,6 +62,11 @@ scrape_no_openai: scrape_metropoles: docker compose exec scraper python scrape_no_openai.py --platform metropoles.com +# Comando para verificação de instalação do Playwright +test_playwright: + @echo "Verificando instalação do Playwright..." + docker compose run --rm scraper python -c "from playwright.sync_api import sync_playwright; print('Iniciando teste do Playwright'); p = sync_playwright().start(); print('Playwright iniciado com sucesso'); browser = p.firefox.launch(); print('Navegador lançado com sucesso'); page = browser.new_page(); print('Nova página criada'); page.goto('https://www.example.com'); print('Página carregada'); browser.close(); p.stop(); print('Teste do Playwright concluído com sucesso')" + scrape_maisgoias: docker compose exec scraper python scrape_no_openai.py --platform maisgoias.com.br @@ -161,6 +166,7 @@ collect: crawl_metropoles scrape_no_openai collect_working: crawl_all_working scrape_all_working @echo "Coleta completa de todos os portais funcionais concluída!" + # Exibir ajuda help: @echo "Check-up - Ferramenta de análise de anúncios de saúde" @@ -202,6 +208,7 @@ help: @echo " make init_db - Inicializa as tabelas do banco de dados" @echo " make migrate_db - Executa as migrações do banco de dados" @echo " make prune - Remove containers, redes e imagens não utilizadas" + @echo " make test_playwright - Testa se a instalação do Playwright está funcionando" .PHONY: start scrape scrape_no_openai scrape_fixed scrape_working install run-backend run-frontend docker-run-backend docker-run-frontend docker-build-backend docker-build-frontend all stop down