diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..9e004ee --- /dev/null +++ b/.dockerignore @@ -0,0 +1,16 @@ +.git +.github +.venv +.pytest_cache +.ruff_cache +.coverage +htmlcov +build +**/__pycache__ +**/*.egg-info +docs/package +data +results +images +notebooks +tests diff --git a/.env.example b/.env.example deleted file mode 100644 index 09976e1..0000000 --- a/.env.example +++ /dev/null @@ -1,284 +0,0 @@ -# Environment Configuration for Bee Migration Analysis Project -# Copy this file to .env and update the values as needed - -# ============================================================================= -# APPLICATION SETTINGS -# ============================================================================= - -# Application environment (development, testing, production) -APP_ENV=development - -# Application name and version -APP_NAME="Bee Migration Analysis" -APP_VERSION=1.0.0 - -# Debug mode (true/false) -DEBUG=true - -# Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL) -LOG_LEVEL=INFO - -# Log file path (optional) -LOG_FILE=logs/app.log - -# ============================================================================= -# API SETTINGS -# ============================================================================= - -# API host and port -API_HOST=0.0.0.0 -API_PORT=8000 - -# API base URL -API_BASE_URL=http://localhost:8000 - -# API documentation settings -API_DOCS_URL=/docs -API_REDOC_URL=/redoc - -# CORS settings -CORS_ORIGINS=http://localhost:3000,http://localhost:8501,http://localhost:8888 - -# ============================================================================= -# DATABASE SETTINGS -# ============================================================================= - -# PostgreSQL settings (for future use) -DB_HOST=localhost -DB_PORT=5432 -DB_NAME=bee_migration -DB_USER=beeuser -DB_PASSWORD=beepassword -DB_URL=postgresql://beeuser:beepassword@localhost:5432/bee_migration - -# SQLite settings (current) -SQLITE_DB_PATH=data/bee_migration.db - -# ============================================================================= -# REDIS SETTINGS -# ============================================================================= - -# Redis connection -REDIS_HOST=localhost -REDIS_PORT=6379 -REDIS_DB=0 -REDIS_PASSWORD= -REDIS_URL=redis://localhost:6379/0 - -# Cache settings -CACHE_TTL=3600 -CACHE_PREFIX=bee_migration: - -# ============================================================================= -# EXTERNAL API SETTINGS -# ============================================================================= - -# Weather API settings -WEATHER_API_KEY=your_weather_api_key_here -WEATHER_API_URL=https://api.openweathermap.org/data/2.5 - -# GBIF API settings -GBIF_API_URL=https://api.gbif.org/v1 -GBIF_USERNAME=your_gbif_username -GBIF_PASSWORD=your_gbif_password - -# NASA API settings -NASA_API_KEY=your_nasa_api_key_here -NASA_API_URL=https://api.nasa.gov - -# WorldClim API settings -WORLDCLIM_API_URL=https://www.worldclim.org/data - -# ============================================================================= -# FILE PATHS -# ============================================================================= - -# Data directories -DATA_DIR=data -RAW_DATA_DIR=data/raw -PROCESSED_DATA_DIR=data/processed -EXTERNAL_DATA_DIR=data/external - -# Output directories -RESULTS_DIR=results -IMAGES_DIR=images -REPORTS_DIR=results/reports -MODELS_DIR=results/models - -# Temporary directories -TEMP_DIR=temp -CACHE_DIR=.cache - -# ============================================================================= -# MACHINE LEARNING SETTINGS -# ============================================================================= - -# Model settings -MODEL_RANDOM_STATE=42 -MODEL_TEST_SIZE=0.2 -MODEL_CV_FOLDS=5 - -# Training settings -TRAINING_EPOCHS=100 -TRAINING_BATCH_SIZE=32 -TRAINING_LEARNING_RATE=0.001 - -# Model persistence -MODEL_SAVE_PATH=results/models -MODEL_CHECKPOINT_INTERVAL=10 - -# ============================================================================= -# VISUALIZATION SETTINGS -# ============================================================================= - -# Plot settings -PLOT_DPI=300 -PLOT_FORMAT=png -PLOT_STYLE=seaborn-v0_8 -PLOT_FIGSIZE_WIDTH=12 -PLOT_FIGSIZE_HEIGHT=8 - -# Color settings -PLOT_COLOR_PALETTE=viridis -PLOT_BACKGROUND_COLOR=white - -# ============================================================================= -# DASHBOARD SETTINGS -# ============================================================================= - -# Streamlit settings -STREAMLIT_SERVER_ADDRESS=0.0.0.0 -STREAMLIT_SERVER_PORT=8501 -STREAMLIT_THEME_PRIMARY_COLOR=#1f77b4 -STREAMLIT_THEME_BACKGROUND_COLOR=#ffffff - -# Dashboard refresh interval (seconds) -DASHBOARD_REFRESH_INTERVAL=300 - -# ============================================================================= -# JUPYTER SETTINGS -# ============================================================================= - -# Jupyter notebook settings -JUPYTER_PORT=8888 -JUPYTER_IP=0.0.0.0 -JUPYTER_TOKEN= -JUPYTER_PASSWORD= - -# ============================================================================= -# SECURITY SETTINGS -# ============================================================================= - -# Secret key for sessions (generate a secure random string) -SECRET_KEY=your_secret_key_here_change_this_in_production - -# JWT settings -JWT_SECRET_KEY=your_jwt_secret_key_here -JWT_ALGORITHM=HS256 -JWT_EXPIRATION_HOURS=24 - -# API rate limiting -RATE_LIMIT_PER_MINUTE=60 -RATE_LIMIT_PER_HOUR=1000 - -# ============================================================================= -# MONITORING SETTINGS -# ============================================================================= - -# Prometheus settings -PROMETHEUS_PORT=9090 -PROMETHEUS_METRICS_PATH=/metrics - -# Grafana settings -GRAFANA_PORT=3000 -GRAFANA_ADMIN_PASSWORD=admin - -# Health check settings -HEALTH_CHECK_INTERVAL=30 -HEALTH_CHECK_TIMEOUT=10 - -# ============================================================================= -# PERFORMANCE SETTINGS -# ============================================================================= - -# Parallel processing -N_JOBS=-1 -PARALLEL_BACKEND=threading - -# Memory settings -MAX_MEMORY_USAGE_GB=8 -CHUNK_SIZE=10000 - -# Caching -ENABLE_CACHING=true -CACHE_SIZE_MB=512 - -# ============================================================================= -# DEVELOPMENT SETTINGS -# ============================================================================= - -# Development tools -ENABLE_PROFILING=false -ENABLE_DEBUG_TOOLBAR=false - -# Testing settings -TEST_DATABASE_URL=sqlite:///test_bee_migration.db -TEST_DATA_DIR=tests/data - -# Code quality -ENABLE_TYPE_CHECKING=true -ENABLE_LINTING=true - -# ============================================================================= -# DOCKER SETTINGS -# ============================================================================= - -# Docker compose settings -COMPOSE_PROJECT_NAME=bee-migration -COMPOSE_FILE=docker-compose.yml - -# Container settings -CONTAINER_TIMEZONE=UTC -CONTAINER_LOCALE=en_US.UTF-8 - -# ============================================================================= -# BACKUP SETTINGS -# ============================================================================= - -# Backup settings -BACKUP_DIR=backups -BACKUP_RETENTION_DAYS=30 -AUTO_BACKUP_ENABLED=false -BACKUP_SCHEDULE=0 2 * * * - -# ============================================================================= -# NOTIFICATION SETTINGS -# ============================================================================= - -# Email settings (for notifications) -SMTP_HOST=smtp.gmail.com -SMTP_PORT=587 -SMTP_USERNAME=your_email@gmail.com -SMTP_PASSWORD=your_app_password -SMTP_USE_TLS=true - -# Slack settings (for notifications) -SLACK_WEBHOOK_URL=https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK -SLACK_CHANNEL=#bee-migration - -# ============================================================================= -# FEATURE FLAGS -# ============================================================================= - -# Feature toggles -ENABLE_REAL_TIME_ANALYSIS=false -ENABLE_ADVANCED_ML_MODELS=true -ENABLE_GEOSPATIAL_ANALYSIS=true -ENABLE_CLIMATE_INTEGRATION=true -ENABLE_API_ENDPOINTS=true -ENABLE_DASHBOARD=true -ENABLE_EXPORT_FEATURES=true - -# Experimental features -ENABLE_EXPERIMENTAL_FEATURES=false -ENABLE_BETA_FEATURES=false \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a9a3ba..24f82f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,6 +36,16 @@ jobs: - name: Test run: pytest + docker: + name: Docker + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: Build + run: docker build --tag bee-migration . + - name: Run the analysis in the container + run: docker run --rm bee-migration analyze + site: name: Site runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index e6057c3..c36bae6 100644 --- a/.gitignore +++ b/.gitignore @@ -280,7 +280,6 @@ conda-meta/ *.m~ # Docker -.dockerignore Dockerfile.dev # Documentation builds diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 604c8e3..d87b90d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,128 +1,25 @@ -# Pre-commit configuration for Bee Migration Analysis Project # Install with: pre-commit install -# Run manually with: pre-commit run --all-files - repos: - # General file checks - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v6.0.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer - id: check-yaml - - id: check-json - id: check-toml - - id: check-xml + - id: check-json - id: check-merge-conflict - - id: check-case-conflict - - id: check-docstring-first - - id: check-executables-have-shebangs - - id: check-shebang-scripts-are-executable - - id: fix-byte-order-marker - - id: mixed-line-ending - - id: requirements-txt-fixer - - id: sort-simple-yaml - - id: file-contents-sorter - files: '^(requirements.*\.txt|\.gitignore)$' - id: check-added-large-files - args: ['--maxkb=10000'] # 10MB limit - - # Python code formatting - - repo: https://github.com/psf/black - rev: 23.3.0 - hooks: - - id: black - language_version: python3 - args: [--line-length=88] - - # Import sorting - - repo: https://github.com/pycqa/isort - rev: 5.12.0 - hooks: - - id: isort - args: [--profile=black, --line-length=88] - - # Linting - - repo: https://github.com/pycqa/flake8 - rev: 6.0.0 - hooks: - - id: flake8 - args: [--max-line-length=88, --extend-ignore=E203,W503] - additional_dependencies: - - flake8-docstrings - - flake8-import-order - - flake8-bugbear + args: [--maxkb=1000] - # Type checking - - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.3.0 + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.17.0 hooks: - - id: mypy - additional_dependencies: [types-requests, types-PyYAML] - args: [--ignore-missing-imports, --no-strict-optional] + - id: ruff-check + args: [--fix] + - id: ruff-format - # Security checks - - repo: https://github.com/pycqa/bandit - rev: 1.7.5 - hooks: - - id: bandit - args: [-r, src/] - exclude: tests/ - - # Documentation - - repo: https://github.com/pycqa/pydocstyle - rev: 6.3.0 - hooks: - - id: pydocstyle - args: [--convention=google] - - # Jupyter notebooks - - repo: https://github.com/nbQA-dev/nbQA - rev: 1.7.0 - hooks: - - id: nbqa-black - additional_dependencies: [black==23.3.0] - - id: nbqa-isort - additional_dependencies: [isort==5.12.0] - - id: nbqa-flake8 - additional_dependencies: [flake8==6.0.0] - - # Remove notebook outputs - repo: https://github.com/kynan/nbstripout - rev: 0.6.1 + rev: 0.9.1 hooks: - id: nbstripout - - # YAML formatting - - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.0.0-alpha.9-for-vscode - hooks: - - id: prettier - types_or: [yaml, markdown] - exclude: '^(CHANGELOG\.md|environment\.yml)$' - - # Shell script linting - - repo: https://github.com/shellcheck-py/shellcheck-py - rev: v0.9.0.2 - hooks: - - id: shellcheck - - # Dockerfile linting - - repo: https://github.com/hadolint/hadolint - rev: v2.12.0 - hooks: - - id: hadolint - args: [--ignore, DL3008, --ignore, DL3009] - -# Configuration for specific hooks -ci: - autofix_commit_msg: | - [pre-commit.ci] auto fixes from pre-commit.com hooks - - for more information, see https://pre-commit.ci - autofix_prs: true - autoupdate_branch: '' - autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate' - autoupdate_schedule: weekly - skip: [] - submodules: false \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c5f83a..ae2d392 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Git Flow branching guide in `CONTRIBUTING.md` ### Changed +- README rewritten around what the data is and what it does not show +- Tutorial notebook rewritten for the package and run by the test suite +- Dockerfile, Compose file, pre-commit hooks, conda environment and Makefile + reduced to what the project uses; the unused `.env.example` is gone - Models are evaluated on the most recent years, held out chronologically, and tuned with time-ordered cross-validation - Feature importance is measured by permutation for every model @@ -124,4 +128,4 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for details on how to contribute to this ## License -This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. \ No newline at end of file +This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5641b61..a73bf07 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,9 +30,10 @@ This project adheres to a code of conduct that promotes a welcoming and inclusiv ### Prerequisites -- Python 3.8 or higher +- Python 3.11 or higher - pip or conda package manager - Git +- Node.js 22 or higher, only to run the dashboard's tests ### Environment Setup @@ -45,24 +46,29 @@ cd Bees python -m venv .venv source .venv/bin/activate # On Windows: .venv\Scripts\activate -# Install dependencies -pip install -r requirements.txt +# Install the package with its development tools +pip install -e ".[dev]" -# Install development dependencies (if available) -pip install -r requirements-dev.txt +# Check your changes before every commit +pre-commit install ``` +With conda: `conda env create -f environment.yml`. + ### Running Tests ```bash # Run all tests pytest -# Run tests with coverage -pytest --cov=src +# Run one test file, without the coverage threshold +pytest tests/test_analysis.py --no-cov + +# Run the dashboard's tests +node --test "tests/web/*.test.mjs" -# Run specific test file -pytest tests/test_specific_module.py +# Try the dashboard locally +make site ``` ## Contributing Guidelines @@ -118,27 +124,51 @@ ruff check src/ tests/ ``` tests/ -├── unit/ # Unit tests -├── integration/ # Integration tests -├── fixtures/ # Test data and fixtures -└── conftest.py # Pytest configuration +├── conftest.py # shared fixtures +├── test_climate.py # observations and simulated datasets +├── test_analysis.py # trends, correlations, regions +├── test_prediction.py # features, models, scores +├── test_sources.py # GBIF and Open-Meteo answers +├── test_web.py # studies as JSON for the dashboard +├── test_outputs.py # reports, charts, command line +├── test_notebook.py # the tutorial runs end to end +└── web/ # dashboard utilities (node --test) ``` ### Writing Tests +Name a test after the behaviour it checks and follow arrange, act, assert: + +```python +from bee_migration.analysis.temporal import TemporalPatterns +from bee_migration.climate.observations import Observations +from tests.conftest import region_table + + +def test_trend_is_the_yearly_slope(): + table = region_table("North", temperature=[10.0, 10.5, 11.0, 11.5]) + + trends = TemporalPatterns(Observations(table)).trends() + + assert trends.loc["North", "temperature"] == 0.5 +``` + +## Writing Tests + ```python import pytest from src.module import function_to_test + def test_function_behavior(): """Test that function behaves correctly.""" # Arrange input_data = "test_input" expected_output = "expected_result" - + # Act result = function_to_test(input_data) - + # Assert assert result == expected_output ``` @@ -264,7 +294,7 @@ gh pr create --base main - [Project Documentation](./README.md) - [Scientific Papers](./results/scientific_papers_synthesis.md) -- [API Documentation](./docs/) +- [Dashboard](https://digomes87.github.io/Bees/) (source in `./docs/`) - [Issue Tracker](https://github.com/digomes87/Bees/issues) ## Questions? @@ -278,4 +308,4 @@ If you have questions about contributing, please: --- -Thank you for contributing to the bee migration analysis project! Your contributions help advance our understanding of climate change impacts on bee populations. 🐝 \ No newline at end of file +Thank you for contributing to the bee migration analysis project! Your contributions help advance our understanding of climate change impacts on bee populations. 🐝 diff --git a/Dockerfile b/Dockerfile index c803c7f..fbaffd1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,184 +1,25 @@ -# Multi-stage Dockerfile for Bee Migration Analysis Project -# This Dockerfile creates a production-ready container with all dependencies +# Runs the bee-migration command, or serves the dashboard (see docker-compose.yml). +FROM python:3.13-slim -# Build stage -FROM python:3.9-slim as builder - -# Set environment variables ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ PIP_NO_CACHE_DIR=1 \ - PIP_DISABLE_PIP_VERSION_CHECK=1 - -# Install system dependencies for building -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ - gcc \ - g++ \ - gfortran \ - libgdal-dev \ - libproj-dev \ - libgeos-dev \ - libspatialindex-dev \ - libffi-dev \ - libssl-dev \ - python3-dev \ - && rm -rf /var/lib/apt/lists/* - -# Set GDAL environment variables -ENV CPLUS_INCLUDE_PATH=/usr/include/gdal -ENV C_INCLUDE_PATH=/usr/include/gdal - -# Create virtual environment -RUN python -m venv /opt/venv -ENV PATH="/opt/venv/bin:$PATH" - -# Upgrade pip and install wheel -RUN pip install --upgrade pip setuptools wheel - -# Copy requirements first for better caching -COPY requirements.txt requirements-dev.txt ./ - -# Install Python dependencies -RUN pip install --no-cache-dir -r requirements.txt - -# Production stage -FROM python:3.9-slim as production - -# Set environment variables -ENV PYTHONDONTWRITEBYTECODE=1 \ - PYTHONUNBUFFERED=1 \ - PATH="/opt/venv/bin:$PATH" \ - PYTHONPATH="/app/src:$PYTHONPATH" + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + MPLCONFIGDIR=/tmp/matplotlib -# Install runtime system dependencies -RUN apt-get update && apt-get install -y --no-install-recommends \ - libgdal28 \ - libproj19 \ - libgeos-c1v5 \ - libspatialindex6 \ - curl \ - && rm -rf /var/lib/apt/lists/* - -# Copy virtual environment from builder stage -COPY --from=builder /opt/venv /opt/venv - -# Create non-root user -RUN groupadd -r beeuser && useradd -r -g beeuser beeuser - -# Create application directory WORKDIR /app -# Copy application code -COPY --chown=beeuser:beeuser . . - -# Create necessary directories -RUN mkdir -p /app/data /app/images /app/results /app/logs && \ - chown -R beeuser:beeuser /app - -# Switch to non-root user -USER beeuser - -# Install the package in development mode -RUN pip install -e . - -# Expose port for potential web services -EXPOSE 8000 - -# Health check -HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 \ - CMD python -c "import sys; sys.exit(0)" || exit 1 - -# Default command -CMD ["python", "-m", "bee_migration.cli", "--help"] - -# Development stage -FROM production as development - -# Switch back to root for installing dev dependencies -USER root - -# Install development dependencies -RUN pip install -r requirements-dev.txt - -# Install additional development tools -RUN apt-get update && apt-get install -y --no-install-recommends \ - git \ - vim \ - htop \ - && rm -rf /var/lib/apt/lists/* - -# Switch back to non-root user -USER beeuser - -# Install pre-commit hooks -RUN pre-commit install || true - -# Default command for development -CMD ["bash"] - -# Jupyter stage -FROM development as jupyter - -# Expose Jupyter port -EXPOSE 8888 - -# Install Jupyter extensions -RUN pip install jupyter-contrib-nbextensions && \ - jupyter contrib nbextension install --user - -# Create Jupyter config -RUN jupyter notebook --generate-config - -# Set Jupyter configuration -RUN echo "c.NotebookApp.ip = '0.0.0.0'" >> ~/.jupyter/jupyter_notebook_config.py && \ - echo "c.NotebookApp.port = 8888" >> ~/.jupyter/jupyter_notebook_config.py && \ - echo "c.NotebookApp.open_browser = False" >> ~/.jupyter/jupyter_notebook_config.py && \ - echo "c.NotebookApp.allow_root = True" >> ~/.jupyter/jupyter_notebook_config.py && \ - echo "c.NotebookApp.token = ''" >> ~/.jupyter/jupyter_notebook_config.py && \ - echo "c.NotebookApp.password = ''" >> ~/.jupyter/jupyter_notebook_config.py - -# Default command for Jupyter -CMD ["jupyter", "notebook", "--notebook-dir=/app/notebooks"] - -# API stage -FROM production as api - -# Install API dependencies -RUN pip install fastapi uvicorn python-multipart - -# Expose API port -EXPOSE 8000 - -# Default command for API -CMD ["uvicorn", "bee_migration.api:app", "--host", "0.0.0.0", "--port", "8000"] - -# Dashboard stage -FROM production as dashboard - -# Install dashboard dependencies -RUN pip install streamlit dash plotly - -# Expose dashboard port -EXPOSE 8501 - -# Default command for dashboard -CMD ["streamlit", "run", "bee_migration/dashboard.py", "--server.address", "0.0.0.0"] +COPY pyproject.toml README.md ./ +COPY src ./src +RUN pip install . -# Testing stage -FROM development as testing +# The dashboard loads the package as a wheel next to its pages. +COPY docs ./docs +RUN pip wheel . --no-deps --wheel-dir docs/package -# Run tests by default -CMD ["pytest", "tests/", "-v", "--cov=src", "--cov-report=term-missing"] +RUN useradd --create-home bee && mkdir /work && chown bee:bee /work +USER bee +WORKDIR /work -# Labels for metadata -LABEL maintainer="Diego Gomes " \ - version="1.0.0" \ - description="Bee Migration Analysis: Climate Impact Assessment" \ - org.opencontainers.image.title="Bee Migration Analysis" \ - org.opencontainers.image.description="Climate Impact Assessment on Bee Migration Patterns" \ - org.opencontainers.image.version="1.0.0" \ - org.opencontainers.image.authors="Diego Gomes" \ - org.opencontainers.image.url="https://github.com/diegogo/bee-migration-analysis" \ - org.opencontainers.image.source="https://github.com/diegogo/bee-migration-analysis" \ - org.opencontainers.image.licenses="MIT" \ No newline at end of file +ENTRYPOINT ["bee-migration"] +CMD ["--help"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..6a0f031 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Diego Gomes + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile index 4882132..4dcd9bb 100644 --- a/Makefile +++ b/Makefile @@ -1,197 +1,82 @@ -# Makefile for Bee Migration Analysis Project +.PHONY: help install install-dev hooks test test-site lint format format-check \ + analyze predict charts all site notebook docker-build docker-run clean check -.PHONY: help install install-dev test lint format clean run-analysis generate-charts setup-env docs - -# Default target help: - @echo "Available commands:" - @echo " install - Install production dependencies" - @echo " install-dev - Install development dependencies" - @echo " setup-env - Setup development environment" - @echo " test - Run tests" - @echo " test-cov - Run tests with coverage" - @echo " lint - Run linting checks" - @echo " format - Format code with ruff" - @echo " clean - Clean temporary files" - @echo " run-analysis - Run complete bee migration analysis" - @echo " generate-charts - Generate all scientific charts" - @echo " docs - Generate documentation" - @echo " security - Run security checks" - @echo " all - Run format, lint, test, and analysis" - -# Installation + @echo "Setup" + @echo " install Install the package" + @echo " install-dev Install the package with development tools" + @echo " hooks Install the pre-commit hooks" + @echo "Quality" + @echo " test Run the Python tests with coverage" + @echo " test-site Run the dashboard tests" + @echo " lint Check the code with ruff" + @echo " format Format the code with ruff" + @echo " check Everything CI runs: format-check, lint, test, test-site" + @echo "Run" + @echo " analyze Trends, correlations and the summary report" + @echo " predict Train and compare the models" + @echo " charts Render the charts into images/" + @echo " all analyze, predict and charts" + @echo " site Build the package into the dashboard and serve it" + @echo " notebook Open the tutorial in JupyterLab" + @echo "Docker" + @echo " docker-build Build the image" + @echo " docker-run Run the whole pipeline in a container" + @echo "Other" + @echo " clean Remove caches and build leftovers" + install: - @echo "Installing production dependencies..." pip install -e . install-dev: - @echo "Installing development dependencies..." - pip install -e ".[dev]" - -# Environment setup -setup-env: - @echo "Setting up development environment..." - python -m venv .venv - @echo "Virtual environment created. Activate with:" - @echo " source .venv/bin/activate # On Linux/Mac" - @echo " .venv\\Scripts\\activate # On Windows" - -setup-conda: - @echo "Setting up conda environment..." - conda env create -f environment.yml - @echo "Conda environment created. Activate with:" - @echo " conda activate bee-migration-analysis" - -# Testing -test: - @echo "Running tests..." - pytest tests/ -v + pip install -e ".[dev,notebook]" -test-cov: - @echo "Running tests with coverage..." - pytest tests/ -v --cov-report=html +hooks: + pre-commit install + +test: + pytest -test-fast: - @echo "Running fast tests..." - pytest tests/ -v -x --ff +test-site: + node --test "tests/web/*.test.mjs" -# Code quality lint: - @echo "Running linting checks..." - ruff check src/ tests/ + ruff check src tests format: - @echo "Formatting code..." - ruff format src/ tests/ - ruff check --fix src/ tests/ + ruff format src tests + ruff check --fix src tests format-check: - @echo "Checking code format..." - ruff format --check src/ tests/ - -# Security -security: - @echo "Running security checks..." - bandit -r src/ - safety check - -# Analysis and charts -run-analysis: - @echo "Running complete bee migration analysis..." + ruff format --check src tests + +check: format-check lint test test-site + +analyze: bee-migration analyze -generate-charts: - @echo "Generating all scientific charts..." +predict: + bee-migration predict + +charts: bee-migration charts -run-predictor: - @echo "Running migration predictor..." - bee-migration predict +all: + bee-migration all -# Site site: - @echo "Building the analysis package into the site and serving it..." pip wheel . --no-deps --wheel-dir docs/package python -m http.server 8000 --bind 127.0.0.1 --directory docs -test-site: - @echo "Running site tests..." - node --test "tests/web/*.test.mjs" - -# Data processing -process-data: - @echo "Processing raw data..." - @echo "Note: Implement data processing scripts as needed" - -download-data: - @echo "Downloading external data..." - @echo "Note: Implement data download scripts as needed" - -# Documentation -docs: - @echo "Generating documentation..." - @echo "Note: Add sphinx documentation generation here" +notebook: + jupyter lab notebooks/bee_migration_tutorial.ipynb -docs-serve: - @echo "Serving documentation locally..." - @echo "Note: Add local documentation server here" - -# Cleanup -clean: - @echo "Cleaning temporary files..." - find . -type f -name "*.pyc" -delete - find . -type d -name "__pycache__" -delete - find . -type d -name ".pytest_cache" -delete - find . -type f -name ".coverage" -delete - find . -type d -name "htmlcov" -delete - find . -type d -name ".mypy_cache" -delete - find . -type f -name "*.log" -delete - -clean-data: - @echo "Cleaning processed data files..." - rm -rf data/processed/* - rm -rf results/*.csv - rm -rf results/*.pkl - -clean-images: - @echo "Cleaning generated images..." - rm -rf images/*.png - rm -rf images/*.jpg - rm -rf images/*.svg - -# Development workflow -dev-setup: setup-env install-dev - @echo "Development environment ready!" - -dev-check: format lint test - @echo "Development checks completed!" - -# CI/CD simulation -ci: format-check lint test security - @echo "CI checks completed!" - -# Complete workflow -all: format lint test run-analysis generate-charts - @echo "Complete workflow finished!" - -# Git helpers -git-setup: - @echo "Setting up git hooks..." - pre-commit install - -commit-check: format lint test - @echo "Ready for commit!" - -# Project information -info: - @echo "Bee Migration Analysis Project" - @echo "==============================" - @echo "Python version: $(shell python --version)" - @echo "Project structure:" - @tree -L 2 -I '__pycache__|.git|.venv|*.pyc' - -status: - @echo "Project status:" - @echo "- Source files: $(shell find src -name '*.py' | wc -l)" - @echo "- Test files: $(shell find tests -name '*.py' 2>/dev/null | wc -l || echo 0)" - @echo "- Data files: $(shell find data -name '*.csv' 2>/dev/null | wc -l || echo 0)" - @echo "- Generated images: $(shell find images -name '*.png' 2>/dev/null | wc -l || echo 0)" - @echo "- Model files: $(shell find results/models -name '*.pkl' 2>/dev/null | wc -l || echo 0)" - -# Docker support (future) docker-build: - @echo "Building Docker image..." - @echo "Note: Add Dockerfile and build command" + docker compose build docker-run: - @echo "Running Docker container..." - @echo "Note: Add Docker run command" - -# Jupyter notebook support -jupyter: - @echo "Starting Jupyter Lab..." - jupyter lab --ip=0.0.0.0 --port=8888 --no-browser + docker compose run --rm analysis all -notebook-clean: - @echo "Cleaning notebook outputs..." - jupyter nbconvert --clear-output --inplace notebooks/*.ipynb \ No newline at end of file +clean: + find . -type d -name "__pycache__" -not -path "./.venv/*" -prune -exec rm -rf {} + + rm -rf .pytest_cache .ruff_cache .coverage htmlcov build docs/package src/*.egg-info diff --git a/README.md b/README.md index f45cc48..6a9e68a 100644 --- a/README.md +++ b/README.md @@ -1,258 +1,49 @@ -# 🐝 Bee Climate Analysis +# Bee Migration Analysis -[![Python](https://img.shields.io/badge/Python-3.8+-blue.svg)](https://www.python.org/downloads/) +[![CI](https://github.com/digomes87/Bees/actions/workflows/ci.yml/badge.svg)](https://github.com/digomes87/Bees/actions/workflows/ci.yml) +[![Python](https://img.shields.io/badge/Python-3.11+-blue.svg)](https://www.python.org/downloads/) [![License](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE) -[![Status](https://img.shields.io/badge/Status-Active-brightgreen.svg)]() -[![Scientific Papers](https://img.shields.io/badge/Papers-6%20Integrated-blue.svg)]() -[![Evidence Level](https://img.shields.io/badge/Evidence-Very%20High-green.svg)]() +[![Dashboard](https://img.shields.io/badge/Dashboard-live-b8740a.svg)](https://digomes87.github.io/Bees/) -> Analysis of climate change impact on South American bee populations using Machine Learning and data analysis. +A Python package and a browser dashboard for studying how climate relates to +bee populations in South America: regional trends, correlations, and models +that forecast bee measurements from climate. -## 🚨 **CRITICAL SCIENTIFIC DISCOVERY** +**[Open the dashboard](https://digomes87.github.io/Bees/)** — it runs this +package in your browser, nothing to install. -> **📊 SCIENTIFIC CONSENSUS CONFIRMED**: Integrated analysis of 6 scientific studies confirms that **85-90% of bees are migrating southward** due to climate change. -> -> **⏰ URGENCY**: Conservation window of only **5-10 years** for effective action. -> -> **📍 EMBLEMATIC CASE**: *Bombus bellicosus* already locally extinct in Curitiba. -> -> **🗺️ CRITICAL AREA**: Southern Brazilian grasslands identified as priority conservation zone. +## What the data is -## 📋 About the Project +Read this before quoting any number from the project. -This project investigates how climate change affects bee populations in South America, using advanced data analysis techniques and machine learning to identify patterns, correlations, and predict future scenarios. +| Data | Where it comes from | What it supports | +|------|--------------------|------------------| +| Regional observations (temperature, precipitation, bee abundance, diversity, honey) | **Simulated** from assumed warming and decline rates | Demonstrating the method. No scientific conclusion. | +| Observed temperature | Open-Meteo archive (ERA5 reanalysis), fetched live | Real warming at five points, one per Brazilian region | +| Bee occurrence records | GBIF, family Apidae, fetched live | Where bees were *recorded*, which also reflects where people looked | +| Figures in the literature charts | Entered by hand as a reading of the cited studies | Illustration only; not extracted from the studies' datasets | -### 🎯 Objectives +The question in the project's name, whether bees are moving south, is **not +answered** by this repository. The cited literature projects southward range +shifts for south-eastern South America. The live field chapter, loaded on +2026-10-11, showed +0.10 °C per decade at the five region centres and a mean +latitude of GBIF records that barely moved (−0.1° per decade over 136,300 +records). Record locations follow observers as much as bees, so that neither +confirms nor rules out a shift. -- **Exploratory Analysis**: Investigate spatial and temporal distribution of bees -- **Climate Correlation**: Quantify relationships between climate and bee abundance -- **ML Prediction**: Develop models to predict migration and abundance -- **Visualization**: Create informative maps and charts -- **Reports**: Generate actionable insights for conservation - -## 🚀 Quick Start +## Quick start ```bash -# Clone the repository git clone https://github.com/digomes87/Bees.git cd Bees - -# Install the package pip install -e . -# Run complete analysis -bee-migration analyze - -# Run ML predictions -bee-migration predict - -# Render the charts -bee-migration charts -``` - -## 📊 Features - -### 🔍 Exploratory Analysis -- Geographic distribution of species -- Temporal abundance analysis -- Climate-biodiversity correlations -- Seasonal pattern identification - -### 🤖 Machine Learning -- **Models**: Random Forest, Gradient Boosting, SVR, Linear Regression -- **Features**: Climate, geographic and temporal variables -- **Metrics**: R², RMSE, MAE, MAPE -- **Validation**: Cross-validation and holdout test - -### 📈 Visualizations -- Species distribution maps -- Temporal trend charts -- Correlation heatmaps -- Feature importance analysis - -## 🏗️ Architecture - +bee-migration all # analysis, model comparison and charts ``` -Bees/ -├── src/bee_migration/ -│ ├── climate/ # Observations and simulated datasets -│ ├── analysis/ # Temporal, correlation and geographic patterns -│ ├── prediction/ # Features, model catalog, training, scores -│ ├── evidence/ # Figures summarised from the cited studies -│ ├── charts/ # Chart panels and gallery -│ ├── reports/ # Plain-text reports -│ └── cli.py # `bee-migration` command -├── tests/ # Test suite -├── data/ -│ ├── raw/ # Raw data -│ ├── processed/ # Processed data -│ └── simulated/ # Simulated data -├── results/ -│ ├── models/ # Trained models -│ ├── reports/ # Reports -│ └── *.csv # Results -├── images/ # Generated charts -├── docs/ # Documentation -└── requirements.txt # Dependencies -``` - -## 🛠️ Technologies - -### Core -- **Python 3.8+**: Main language -- **Pandas**: Data manipulation -- **NumPy**: Numerical computing -- **Scikit-learn**: Machine Learning - -### Visualization -- **Matplotlib**: Static charts -- **Seaborn**: Statistical visualizations -- **Plotly**: Interactive charts - -### Geospatial Analysis -- **GeoPandas**: Geographic data -- **Folium**: Interactive maps - -### Data APIs -- **GBIF**: Biodiversity data -- **WorldClim**: Climate data -- **NASA**: Satellite data - -## 📈 Results - -### 🔬 Scientific Discoveries Based on Papers - -#### **Evidence Convergence: Southward Migration** -Integrated analysis of 6 scientific studies confirms consistent pattern of bee migration southward due to climate change: - -- **📊 Scientific Consensus**: 85-90% of studies confirm south/southwest migration -- **🌡️ Critical Threshold**: 2.5°C increase as tipping point -- **📍 Indicator Species**: *Bombus bellicosus* (local extinction in Curitiba) -- **🗺️ Critical Area**: Southern Brazilian grasslands identified as priority conservation zone - -#### **Integrated Studies** -1. **UFPR**: 18 neotropical species, projection until 2050 -2. **Dryad Dataset**: Species distribution modeling -3. **PMC Studies**: Global impacts and emerging pests -4. **Ecology Letters**: Community restructuring (North America) -5. **Aegean Study**: Impacts on island ecosystems - -### 📊 Impacts by Functional Group - -| Group | Impact | Trend | -|-------|--------|-------| -| **Eusocial** | -35% | 🔴 High vulnerability | -| **Solitary** | -20% | 🟡 Moderate vulnerability | -| **Specialists** | -40% | 🔴 Critical | -| **Generalists** | +15% | 🟢 Benefited | - -### 🎯 Temporal Projections - -#### **Migration Scenarios (2024-2050)** -- **🟢 Conservative**: 80% of species migrate by 2050 -- **🟡 Moderate**: 95% of species migrate by 2050 (most likely) -- **🔴 Aggressive**: 100% of species migrate by 2040 - -### ML Performance Metrics -- **Best Model**: Linear Regression (R² = 0.81 on the four most recent years, held out) -- **RMSE**: 5.2 (bee abundance) -- **Data**: simulated observations; see `results/ml_predictions_report.txt` - -### Key Insights -1. **Negative Correlation**: Temperature increase reduces abundance -2. **Seasonality**: Abundance peaks in autumn/winter -3. **Geography**: Mountainous regions more resilient -4. **Trend**: 15% decline in the last decade -5. **🚨 Urgency**: 5-10 year conservation window - -## 📊 Visualizations - -### 🔬 Scientific Paper Analysis - -![UFPR Analysis](images/ufpr_migration_analysis.png) -*Specific analysis of UFPR study on South American bee migration* - -![Comparative Temperature Analysis](images/comparative_temperature_analysis.png) -*Comparison of temperature trends between local data and scientific studies* - -![Migration Patterns Synthesis](images/migration_patterns_synthesis.png) -*Integrated synthesis of migration patterns based on multiple scientific studies* - -### 📈 Local Data Analysis - -![Climate Impact](images/climate_impact_summary.png) -*Summary of climate impact on bee populations* - -![Correlation Analysis](images/correlation_analysis.png) -*Correlation matrix between climate variables and bee abundance* - -![Geographic Distribution](images/geographic_distribution.png) -*Geographic distribution of bee species* -![Species Decline](images/species_honey_decline.png) -*Analysis of honey-producing species decline* +Requires Python 3.11 or newer. -![Temperature-Migration Trend](images/temperature_migration_trend.png) -*Relationship between temperature increase and migration patterns* - -## 🔬 Scientific Paper Discoveries - -### 📋 Evidence Synthesis - -This project integrated analysis of **6 main scientific studies** on bee migration due to climate change: - -#### 🎯 **Key Findings** - -1. **🌍 Global Consensus**: 85-90% of studies confirm south/southwest migration -2. **🌡️ Critical Threshold**: 2.5°C warming as tipping point -3. **📍 Emblematic Case**: *Bombus bellicosus* - local extinction in Curitiba -4. **🗺️ Priority Area**: Southern Brazilian grasslands as critical zone -5. **⏰ Temporal Urgency**: 5-10 year conservation window - -#### 📊 **Scientific Validation** - -| Aspect | Evidence Level | Converging Sources | -|--------|---------------|-------------------| -| **Southward Direction** | ⭐⭐⭐⭐⭐ Very High | 5/5 studies | -| **Speed** | ⭐⭐⭐⭐ High | 4/5 studies | -| **Climate Factors** | ⭐⭐⭐⭐⭐ Very High | 5/5 studies | -| **Vulnerable Groups** | ⭐⭐⭐⭐ High | 4/5 studies | - -### 🎯 **Evidence-Based Recommendations** - -#### 🚨 **Immediate Actions (2024-2027)** -- ✅ Urgent protection of Brazilian southern grasslands -- 🔍 Continuous monitoring of indicator species -- 🌿 Creation of north-south ecological corridors -- ⚠️ Control of emerging pests (Small Hive Beetle) - -#### 🔄 **Medium Term (2027-2035)** -- 🌱 Habitat restoration in the south -- 🚀 Assisted relocation programs -- 📊 Expansion of scientific monitoring -- 🏛️ Integration into public policies - -#### 🌟 **Long Term (2035-2050)** -- 🌾 Agricultural landscape adaptation -- 🌎 Transnational conservation networks -- 🧬 Genetic improvement programs -- 📜 Climate adaptation policies - -### 📚 **Generated Reports** - -- 📄 [Comprehensive Migration Report](results/bee_migration_comprehensive_report.txt) -- 📊 [Integrated Analysis](results/integrated_migration_report.txt) -- 🔬 [Scientific Papers Synthesis](results/scientific_papers_synthesis.md) - -### 🔗 **Integrated Scientific Sources** - -1. **UFPR** - Climate change pushes South American bees southward -2. **Dryad Dataset** - Climate-induced range shifts in neotropical bees -3. **PMC Studies** - Global warming and honey bee pests + Island bee impacts -4. **Ecology Letters** - Bumble bee community restructuring (North America) -5. **Kaggle Climate Data** - Climate trend validation - -## 🔧 Usage +## Usage ### Command line @@ -267,10 +58,17 @@ bee-migration analyze --data my_observations.csv # use your own data bee-migration analyze --root /tmp/run --seed 7 # output folder and random seed ``` -A CSV passed with `--data` needs the columns `year`, `region`, `latitude`, -`longitude`, `temperature`, `precipitation`, `bee_abundance`, -`species_diversity` and `honey_production`. Without `--data` the commands run -on simulated observations (2000-2023, five Brazilian regions). +A CSV passed with `--data` needs one row per region and year, with the columns +`year`, `region`, `latitude`, `longitude`, `temperature`, `precipitation`, +`bee_abundance`, `species_diversity` and `honey_production`. Years must be +consecutive within each region. Without `--data` the commands run on simulated +observations (2000–2023, five Brazilian regions). + +| Command | Writes | +|---------|--------| +| `analyze` | `results/analysis_summary_report.txt`, `results/correlation_analysis_correlation_matrix.csv`, `data/processed/bee_climate_processed.csv` | +| `predict` | `results/ml_predictions_report.txt`, `results/model_metrics.csv`, `results/feature_importance.csv`, `results/models/*.pkl` | +| `charts` | `images/*.png`, `data/bee_climate_data.csv`, `data/bee_geographic_data.csv` | ### Python @@ -291,13 +89,14 @@ leaderboard = MigrationPredictor().train(observations) best = leaderboard.best() ``` +`notebooks/bee_migration_tutorial.ipynb` walks through the same steps. + ### In the browser -The dashboard in `docs/` runs this same package inside the page with -[Pyodide](https://pyodide.org/): choose a seed, the number of held-out years and -the measurement to forecast, and the study is computed on your machine. A -separate chapter loads observed temperatures from Open-Meteo and bee occurrence -records from GBIF. +The dashboard in `docs/` loads this package into the page with +[Pyodide](https://pyodide.org/). Choose a seed, the number of held-out years and +the measurement to forecast; the study is computed on your machine. A separate +chapter fetches the field data described above. ```bash make site # builds the package into docs/package and serves http://127.0.0.1:8000 @@ -305,90 +104,134 @@ make site # builds the package into docs/package and serves http://127.0.0.1: Pushes to `main` publish the site through the `Pages` workflow. -## 🧪 Tests +## How it works + +1. **Observations** — yearly climate and bee measurements per region, validated + on load, with anomalies, a climate stress index and a bee health index derived + from them. +2. **Analysis** — linear trend per region, sharp bends in a series, correlations + (overall, per region, and with bee abundance lagged by one to three years), + and regional summaries. +3. **Prediction** — four regressors (linear, random forest, gradient boosting, + SVR) learn a bee measurement from climate and from the previous year's value. + They are tuned with time-ordered cross-validation and scored on the most + recent years, which they never see while training. +4. **Outputs** — plain-text reports, CSV tables, charts, or JSON for the + dashboard. -```bash -pip install -e ".[dev]" +``` +Bees/ +├── src/bee_migration/ +│ ├── climate/ # observations and simulated datasets +│ ├── analysis/ # temporal, correlation and geographic patterns +│ ├── prediction/ # features, model catalog, training, scores +│ ├── sources/ # GBIF and Open-Meteo answers into tables +│ ├── evidence/ # figures summarised from the cited studies +│ ├── charts/ # chart panels and gallery +│ ├── reports/ # plain-text reports +│ ├── web/ # studies as JSON for the dashboard +│ └── cli.py # the bee-migration command +├── docs/ # dashboard (HTML, CSS, JavaScript) +├── tests/ # Python tests; tests/web holds the site's tests +├── notebooks/ # tutorial +├── data/ results/ images/ # outputs of the commands above +└── pyproject.toml +``` -pytest # tests with coverage -ruff check src tests # linting -ruff format src tests # formatting +## Results on the simulated observations -node --test "tests/web/*.test.mjs" # site utilities -``` +These describe the simulation's assumptions coming back out of the pipeline. +They are useful for checking the method, not for learning about bees. + +| Model | R² on held-out years | RMSE | +|-------|---------------------|------| +| Linear Regression | 0.81 | 5.2 | +| SVR | 0.78 | 5.6 | +| Random Forest | 0.53 | 8.0 | +| Gradient Boosting | 0.48 | 8.5 | + +Forecasting bee abundance, four most recent years held out, seed 42. Full +figures are in `results/ml_predictions_report.txt` and +`results/model_metrics.csv`. + +## Charts + +Drawn from simulated continent-wide series: -## 📋 Roadmap +![Temperature and migration trend](images/temperature_migration_trend.png) +![Correlation analysis](images/correlation_analysis.png) +![Species and honey decline](images/species_honey_decline.png) +![Geographic distribution](images/geographic_distribution.png) +![Climate impact summary](images/climate_impact_summary.png) -### ✅ **Completed** -- [x] Base project structure -- [x] Exploratory data analysis -- [x] Machine learning models -- [x] Visualizations and reports -- [x] **Scientific paper analysis on migration** -- [x] **Scientific evidence integration** -- [x] **Multi-source discovery synthesis** -- [x] **Cross-validation of results** -- [x] **Evidence-based recommendations** +Drawn from figures entered by hand as a reading of the literature: -- [x] Interactive dashboard that runs the package in the browser -- [x] Live field data from GBIF and Open-Meteo +![UFPR migration analysis](images/ufpr_migration_analysis.png) +![Comparative temperature analysis](images/comparative_temperature_analysis.png) +![Migration patterns synthesis](images/migration_patterns_synthesis.png) -### 🚧 **In Development** -- [ ] REST API for predictions -- [ ] Cloud deployment +## Literature behind the project -### 🔮 **Future** -- [ ] Real-time monitoring of indicator species -- [ ] Integration with climate alert systems -- [ ] Scientific collaboration platform -- [ ] Mobile app for citizen science +The project was motivated by published work on climate-driven range shifts in +bees. A synthesis in Portuguese is in `results/scientific_papers_synthesis.md`. -## 🤝 Contributing +- UFPR — climate change pushing South American bees southward (18 species, + south-eastern South America, projections to 2050) +- Dryad dataset — climate-induced range shifts in the south-eastern Neotropics +- Ecology Letters — warming temperatures restructuring bumble bee communities in + North America +- PMC — global warming and the small hive beetle, a honey bee pest +- PMC — climate change impacts on island bees in the Aegean Archipelago -1. **Fork** the project -2. **Clone** your fork -3. **Create** a branch (`git checkout -b feature/new-feature`) -4. **Commit** your changes (`git commit -am 'Add new feature'`) -5. **Push** to the branch (`git push origin feature/new-feature`) -6. **Open** a Pull Request +## Development -### Guidelines -- Follow PEP 8 for Python code -- Add tests for new features -- Document changes in CHANGELOG -- Use semantic commits +```bash +pip install -e ".[dev]" + +pytest # Python tests with coverage +ruff check src tests # linting +ruff format src tests # formatting +node --test "tests/web/*.test.mjs" # dashboard utilities +``` + +The project follows Git Flow: work branches from `develop`, pull requests into +`develop`, releases from `develop` into `main`. The `CI` workflow must pass +before anything is merged. See [CONTRIBUTING.md](CONTRIBUTING.md). + +Docker is optional: + +```bash +docker compose run --rm analysis all # run the pipeline in a container +docker compose up site # serve the dashboard on http://127.0.0.1:8000 +``` -## 📄 License +## Roadmap -This project is licensed under the MIT License - see [LICENSE](LICENSE) for details. +Done: -## 👥 Team +- Tested package and command line +- Dashboard that runs the package in the browser +- Live field data from GBIF and Open-Meteo -- **Diego Gomes** - *Lead Developer* - [@digomes87](https://github.com/digomes87) +Open: -## 🙏 Acknowledgments +- Field data from the command line, not only in the dashboard +- Cross-validation folds cut on year boundaries for any table shape +- REST API for predictions, if something needs to consume it +- Occurrence data corrected for sampling effort, so record locations can say + something about ranges -### 🔬 **Scientific Sources** -- **UFPR** - Federal University of Paraná (South American migration study) -- **Dryad Digital Repository** - Neotropical bee dataset -- **PMC (PubMed Central)** - Global warming and bee studies -- **Ecology Letters** - Community restructuring research -- **Kaggle** - Climate change datasets +## Acknowledgments -### 🌐 **Data and Infrastructure** -- **GBIF** - Biodiversity data -- **WorldClim** - Climate data -- **NASA** - Satellite data -- **Python Community** - Libraries and tools +Data: [GBIF](https://www.gbif.org/) and [Open-Meteo](https://open-meteo.com/) +(ERA5 reanalysis from Copernicus/ECMWF). Studies: UFPR, Dryad, Ecology Letters +and PMC, as listed above. -## 📞 Contact +## License -- **GitHub**: [@digomes87](https://github.com/digomes87) -- **Issues**: [GitHub Issues](https://github.com/digomes87/Bees/issues) -- **Discussions**: [GitHub Discussions](https://github.com/digomes87/Bees/discussions) +MIT. See [LICENSE](LICENSE). ---- +## Author -
- 🐝 Bee Climate Analysis - Protecting Pollinators Through Data Science 🐝 -
\ No newline at end of file +Diego Gomes — [@digomes87](https://github.com/digomes87). +Questions and bugs: [issues](https://github.com/digomes87/Bees/issues). diff --git a/docker-compose.yml b/docker-compose.yml index af53864..38685af 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,283 +1,16 @@ -version: '3.8' - -# Docker Compose configuration for Bee Migration Analysis Project -# This file defines multiple services for different use cases - services: - # Main application service - app: - build: - context: . - target: production - container_name: bee-migration-app - volumes: - - ./data:/app/data:ro - - ./results:/app/results - - ./images:/app/images - - ./logs:/app/logs - environment: - - PYTHONPATH=/app/src - - LOG_LEVEL=INFO - networks: - - bee-network - restart: unless-stopped - healthcheck: - test: ["CMD", "python", "-c", "import sys; sys.exit(0)"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 40s - - # Development service - dev: - build: - context: . - target: development - container_name: bee-migration-dev - volumes: - - .:/app - - /app/.git - - dev-cache:/app/.pytest_cache - - dev-cache:/app/.mypy_cache - environment: - - PYTHONPATH=/app/src - - LOG_LEVEL=DEBUG - networks: - - bee-network - stdin_open: true - tty: true - command: bash - - # Jupyter notebook service - jupyter: - build: - context: . - target: jupyter - container_name: bee-migration-jupyter - ports: - - "8888:8888" - volumes: - - .:/app - - jupyter-data:/home/beeuser/.jupyter - environment: - - PYTHONPATH=/app/src - - JUPYTER_ENABLE_LAB=yes - networks: - - bee-network - restart: unless-stopped - labels: - - "traefik.enable=true" - - "traefik.http.routers.jupyter.rule=Host(`jupyter.localhost`)" - - "traefik.http.services.jupyter.loadbalancer.server.port=8888" - - # API service - api: - build: - context: . - target: api - container_name: bee-migration-api - ports: - - "8000:8000" - volumes: - - ./data:/app/data:ro - - ./results:/app/results - - ./images:/app/images - - ./logs:/app/logs - environment: - - PYTHONPATH=/app/src - - API_HOST=0.0.0.0 - - API_PORT=8000 - - LOG_LEVEL=INFO - networks: - - bee-network - restart: unless-stopped - depends_on: - - redis - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8000/health"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 40s - labels: - - "traefik.enable=true" - - "traefik.http.routers.api.rule=Host(`api.localhost`)" - - "traefik.http.services.api.loadbalancer.server.port=8000" - - # Dashboard service - dashboard: - build: - context: . - target: dashboard - container_name: bee-migration-dashboard - ports: - - "8501:8501" - volumes: - - ./data:/app/data:ro - - ./results:/app/results:ro - - ./images:/app/images:ro - environment: - - PYTHONPATH=/app/src - - STREAMLIT_SERVER_ADDRESS=0.0.0.0 - - STREAMLIT_SERVER_PORT=8501 - networks: - - bee-network - restart: unless-stopped - depends_on: - - api - labels: - - "traefik.enable=true" - - "traefik.http.routers.dashboard.rule=Host(`dashboard.localhost`)" - - "traefik.http.services.dashboard.loadbalancer.server.port=8501" - - # Redis for caching and session management - redis: - image: redis:7-alpine - container_name: bee-migration-redis - ports: - - "6379:6379" + # docker compose run --rm analysis all + analysis: + build: . volumes: - - redis-data:/data - networks: - - bee-network - restart: unless-stopped - command: redis-server --appendonly yes - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 30s - timeout: 10s - retries: 3 + - ./data:/work/data + - ./results:/work/results + - ./images:/work/images - # PostgreSQL database (for future use) - postgres: - image: postgres:15-alpine - container_name: bee-migration-postgres + # docker compose up site -> http://127.0.0.1:8000 + site: + build: . + entrypoint: ["python", "-m", "http.server", "8000", "--directory", "/app/docs"] + command: [] ports: - - "5432:5432" - environment: - - POSTGRES_DB=bee_migration - - POSTGRES_USER=beeuser - - POSTGRES_PASSWORD=beepassword - volumes: - - postgres-data:/var/lib/postgresql/data - - ./sql:/docker-entrypoint-initdb.d - networks: - - bee-network - restart: unless-stopped - healthcheck: - test: ["CMD-SHELL", "pg_isready -U beeuser -d bee_migration"] - interval: 30s - timeout: 10s - retries: 3 - - # Testing service - test: - build: - context: . - target: testing - container_name: bee-migration-test - volumes: - - .:/app - - test-cache:/app/.pytest_cache - - test-coverage:/app/htmlcov - environment: - - PYTHONPATH=/app/src - - PYTEST_CURRENT_TEST=1 - networks: - - bee-network - profiles: - - testing - - # Monitoring with Prometheus (optional) - prometheus: - image: prom/prometheus:latest - container_name: bee-migration-prometheus - ports: - - "9090:9090" - volumes: - - ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml - - prometheus-data:/prometheus - networks: - - bee-network - restart: unless-stopped - profiles: - - monitoring - command: - - '--config.file=/etc/prometheus/prometheus.yml' - - '--storage.tsdb.path=/prometheus' - - '--web.console.libraries=/etc/prometheus/console_libraries' - - '--web.console.templates=/etc/prometheus/consoles' - - '--storage.tsdb.retention.time=200h' - - '--web.enable-lifecycle' - - # Grafana for visualization (optional) - grafana: - image: grafana/grafana:latest - container_name: bee-migration-grafana - ports: - - "3000:3000" - environment: - - GF_SECURITY_ADMIN_PASSWORD=admin - volumes: - - grafana-data:/var/lib/grafana - - ./monitoring/grafana:/etc/grafana/provisioning - networks: - - bee-network - restart: unless-stopped - profiles: - - monitoring - depends_on: - - prometheus - - # Reverse proxy with Traefik (optional) - traefik: - image: traefik:v3.0 - container_name: bee-migration-traefik - ports: - - "80:80" - - "8080:8080" - volumes: - - /var/run/docker.sock:/var/run/docker.sock:ro - - ./traefik:/etc/traefik - networks: - - bee-network - restart: unless-stopped - profiles: - - proxy - command: - - --api.insecure=true - - --providers.docker=true - - --providers.docker.exposedbydefault=false - - --entrypoints.web.address=:80 - -networks: - bee-network: - driver: bridge - ipam: - config: - - subnet: 172.20.0.0/16 - -volumes: - # Persistent data volumes - postgres-data: - driver: local - redis-data: - driver: local - prometheus-data: - driver: local - grafana-data: - driver: local - - # Development volumes - dev-cache: - driver: local - test-cache: - driver: local - test-coverage: - driver: local - jupyter-data: - driver: local - -# Development override -# Use: docker-compose -f docker-compose.yml -f docker-compose.override.yml up -# This file can be created for local development overrides \ No newline at end of file + - "127.0.0.1:8000:8000" diff --git a/environment.yml b/environment.yml index 94c5c34..fb091c1 100644 --- a/environment.yml +++ b/environment.yml @@ -1,75 +1,9 @@ +# conda env create -f environment.yml && conda activate bee-migration-analysis name: bee-migration-analysis channels: - conda-forge - - defaults dependencies: - # Python version - - python=3.9 - - # Core data science libraries - - numpy>=1.21.0 - - pandas>=1.5.0 - - scipy>=1.9.0 - - # Machine learning - - scikit-learn>=1.2.0 - - joblib>=1.2.0 - - # Visualization - - matplotlib>=3.6.0 - - seaborn>=0.12.0 - - plotly>=5.12.0 - - # Geospatial analysis - - geopandas>=0.12.0 - - folium>=0.14.0 - - cartopy>=0.21.0 - - shapely>=2.0.0 - - fiona>=1.8.0 - - pyproj>=3.4.0 - - # Data APIs and web requests - - requests>=2.28.0 - - urllib3>=1.26.0 - - # File I/O and data formats - - openpyxl>=3.0.0 - - xlsxwriter>=3.0.0 - - h5py>=3.7.0 - - # Development tools - - jupyter>=1.0.0 - - jupyterlab>=3.6.0 - - ipykernel>=6.20.0 - - # Testing (optional) - - pytest>=7.0.0 - - pytest-cov>=4.0.0 - - # Code quality (optional) - - black>=23.0.0 - - isort>=5.12.0 - - flake8>=6.0.0 - - # Additional dependencies via pip + - python=3.13 - pip - pip: - # Specific packages not available in conda-forge - - pygbif>=0.6.0 # GBIF API client - - rasterio>=1.3.0 # Raster data processing - - xarray>=2023.1.0 # Multi-dimensional arrays - - netcdf4>=1.6.0 # NetCDF file support - - dask>=2023.1.0 # Parallel computing - - memory-profiler>=0.60.0 # Memory profiling - - # Future dashboard dependencies - - streamlit>=1.20.0 - - dash>=2.10.0 - - # API development - - fastapi>=0.95.0 - - uvicorn>=0.20.0 - - pydantic>=1.10.0 - - # Environment management - - python-dotenv>=1.0.0 \ No newline at end of file + - -e .[dev,notebook] diff --git a/notebooks/bee_migration_tutorial.ipynb b/notebooks/bee_migration_tutorial.ipynb index 1288534..a495a4f 100644 --- a/notebooks/bee_migration_tutorial.ipynb +++ b/notebooks/bee_migration_tutorial.ipynb @@ -2,514 +2,245 @@ "cells": [ { "cell_type": "markdown", + "id": "0", "metadata": {}, "source": [ - "# Bee Migration Analysis Tutorial\n", + "# Bee migration tutorial\n", "\n", - "This notebook demonstrates how to use the Bee Migration Analysis project to analyze climate change impacts on bee populations.\n", + "A walk through the `bee_migration` package: build observations, look at trends\n", + "and correlations, then train models that forecast bee abundance.\n", "\n", - "## Table of Contents\n", + "The observations here are **simulated** from assumed warming and decline rates.\n", + "They show how the method works; they are not field measurements.\n", "\n", - "1. [Setup and Data Loading](#setup)\n", - "2. [Exploratory Data Analysis](#eda)\n", - "3. [Climate Impact Analysis](#climate)\n", - "4. [Machine Learning Predictions](#ml)\n", - "5. [Visualization and Results](#viz)\n", - "6. [Scientific Insights](#insights)\n" + "Install the package first: `pip install -e \".[notebook]\"`." ] }, { "cell_type": "markdown", + "id": "1", "metadata": {}, "source": [ - "## 1. Setup and Data Loading {#setup}\n", - "\n", - "First, let's import the necessary libraries and load our data." + "## 1. Observations" ] }, { "cell_type": "code", "execution_count": null, + "id": "2", "metadata": {}, "outputs": [], "source": [ - "# Import necessary libraries\n", - "import sys\n", - "import os\n", - "\n", - "# Add src directory to path\n", - "sys.path.append('../src')\n", + "from bee_migration.climate.period import DEFAULT_PERIOD\n", + "from bee_migration.climate.region import BRAZILIAN_REGIONS\n", + "from bee_migration.climate.simulation import ClimateSimulation\n", "\n", - "import numpy as np\n", - "import pandas as pd\n", - "import matplotlib.pyplot as plt\n", - "import seaborn as sns\n", - "import plotly.express as px\n", - "import plotly.graph_objects as go\n", - "from plotly.subplots import make_subplots\n", + "simulation = ClimateSimulation(BRAZILIAN_REGIONS, DEFAULT_PERIOD)\n", + "observations = simulation.observations(seed=42)\n", "\n", - "# Import project modules\n", - "from bee_analysis import BeeAnalysis\n", - "from bee_migration_predictor import BeeMigrationPredictor\n", - "from config import Config\n", + "observations.table().head()" + ] + }, + { + "cell_type": "markdown", + "id": "3", + "metadata": {}, + "source": [ + "Each row is one region in one year. Besides the measurements, the table carries\n", + "indices derived from them: anomalies against the regional mean, a climate stress\n", + "index and a bee health index.\n", "\n", - "# Set style\n", - "plt.style.use('seaborn-v0_8')\n", - "sns.set_palette('husl')\n", + "To use your own data, load a CSV with the same measurement columns:\n", "\n", - "# Configure display\n", - "pd.set_option('display.max_columns', None)\n", - "pd.set_option('display.width', None)\n", + "```python\n", + "from pathlib import Path\n", + "from bee_migration.climate.observations import Observations\n", "\n", - "print(\"Setup completed successfully!\")\n", - "print(f\"Working directory: {os.getcwd()}\")" + "observations = Observations.from_csv(Path(\"my_observations.csv\"))\n", + "```" ] }, { "cell_type": "markdown", + "id": "4", "metadata": {}, "source": [ - "### Load and Inspect Data\n", - "\n", - "Let's load our bee climate data and take a first look at it." + "## 2. Trends" ] }, { "cell_type": "code", "execution_count": null, + "id": "5", "metadata": {}, "outputs": [], "source": [ - "# Initialize the analysis\n", - "analyzer = BeeAnalysis()\n", + "from bee_migration.analysis.temporal import TemporalPatterns\n", "\n", - "# Load data\n", - "try:\n", - " data = pd.read_csv('../data/bee_climate_data.csv')\n", - " print(f\"Data loaded successfully! Shape: {data.shape}\")\n", - "except FileNotFoundError:\n", - " print(\"Data file not found. Generating sample data...\")\n", - " # Generate sample data for demonstration\n", - " from generate_charts_english import generate_sample_data\n", - " data = generate_sample_data()\n", - " print(f\"Sample data generated! Shape: {data.shape}\")\n", - "\n", - "# Display basic information\n", - "print(\"\\nDataset Info:\")\n", - "print(f\"Columns: {list(data.columns)}\")\n", - "print(f\"Date range: {data['year'].min()} - {data['year'].max()}\")\n", - "print(f\"Missing values: {data.isnull().sum().sum()}\")\n", - "\n", - "# Display first few rows\n", - "data.head()" + "patterns = TemporalPatterns(observations)\n", + "patterns.trends().round(3)" ] }, { "cell_type": "markdown", + "id": "6", "metadata": {}, "source": [ - "## 2. Exploratory Data Analysis {#eda}\n", - "\n", - "Let's explore the data to understand patterns and trends." + "Values are the change per year. Years where a series bends sharply:" ] }, { "cell_type": "code", "execution_count": null, + "id": "7", "metadata": {}, "outputs": [], "source": [ - "# Basic statistics\n", - "print(\"Dataset Statistics:\")\n", - "print(data.describe())\n", - "\n", - "# Check data types\n", - "print(\"\\nData Types:\")\n", - "print(data.dtypes)" + "patterns.change_points()" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", + "id": "8", "metadata": {}, - "outputs": [], "source": [ - "# Visualize temperature trends over time\n", - "fig, axes = plt.subplots(2, 2, figsize=(15, 10))\n", - "\n", - "# Temperature trend\n", - "axes[0, 0].plot(data['year'], data['temperature'], marker='o', linewidth=2)\n", - "axes[0, 0].set_title('Temperature Trend Over Time')\n", - "axes[0, 0].set_xlabel('Year')\n", - "axes[0, 0].set_ylabel('Temperature (°C)')\n", - "axes[0, 0].grid(True, alpha=0.3)\n", - "\n", - "# Species count trend\n", - "axes[0, 1].plot(data['year'], data['species_count'], marker='s', color='green', linewidth=2)\n", - "axes[0, 1].set_title('Bee Species Count Over Time')\n", - "axes[0, 1].set_xlabel('Year')\n", - "axes[0, 1].set_ylabel('Number of Species')\n", - "axes[0, 1].grid(True, alpha=0.3)\n", - "\n", - "# Honey production trend\n", - "axes[1, 0].plot(data['year'], data['honey_production'], marker='^', color='orange', linewidth=2)\n", - "axes[1, 0].set_title('Honey Production Over Time')\n", - "axes[1, 0].set_xlabel('Year')\n", - "axes[1, 0].set_ylabel('Honey Production (kg)')\n", - "axes[1, 0].grid(True, alpha=0.3)\n", - "\n", - "# Latitude distribution\n", - "axes[1, 1].hist(data['latitude'], bins=20, alpha=0.7, color='purple', edgecolor='black')\n", - "axes[1, 1].set_title('Geographic Distribution (Latitude)')\n", - "axes[1, 1].set_xlabel('Latitude')\n", - "axes[1, 1].set_ylabel('Frequency')\n", - "axes[1, 1].grid(True, alpha=0.3)\n", - "\n", - "plt.tight_layout()\n", - "plt.show()" + "## 3. Correlations" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, + "id": "9", "metadata": {}, + "outputs": [], "source": [ - "## 3. Climate Impact Analysis {#climate}\n", + "from bee_migration.analysis.correlation import Correlations\n", "\n", - "Now let's analyze the relationship between climate variables and bee populations." + "correlations = Correlations(observations)\n", + "[pair.describe() for pair in correlations.notable_pairs()[:5]]" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", + "id": "10", "metadata": {}, - "outputs": [], "source": [ - "# Correlation analysis\n", - "correlation_vars = ['temperature', 'species_count', 'honey_production', 'latitude']\n", - "correlation_matrix = data[correlation_vars].corr()\n", - "\n", - "# Plot correlation heatmap\n", - "plt.figure(figsize=(10, 8))\n", - "sns.heatmap(correlation_matrix, \n", - " annot=True, \n", - " cmap='RdYlBu_r', \n", - " center=0,\n", - " square=True,\n", - " fmt='.3f',\n", - " cbar_kws={'label': 'Correlation Coefficient'})\n", - "plt.title('Climate-Bee Population Correlation Matrix', fontsize=16, pad=20)\n", - "plt.tight_layout()\n", - "plt.show()\n", - "\n", - "print(\"Key Correlations:\")\n", - "print(f\"Temperature vs Species Count: {correlation_matrix.loc['temperature', 'species_count']:.3f}\")\n", - "print(f\"Temperature vs Honey Production: {correlation_matrix.loc['temperature', 'honey_production']:.3f}\")\n", - "print(f\"Latitude vs Temperature: {correlation_matrix.loc['latitude', 'temperature']:.3f}\")" + "Temperature against bee abundance one to three years later, by region:" ] }, { "cell_type": "code", "execution_count": null, + "id": "11", "metadata": {}, "outputs": [], "source": [ - "# Interactive scatter plot with Plotly\n", - "fig = px.scatter(data, \n", - " x='temperature', \n", - " y='species_count',\n", - " color='year',\n", - " size='honey_production',\n", - " hover_data=['latitude', 'longitude'],\n", - " title='Temperature vs Bee Species Count Over Time',\n", - " labels={\n", - " 'temperature': 'Temperature (°C)',\n", - " 'species_count': 'Number of Bee Species',\n", - " 'year': 'Year'\n", - " })\n", - "\n", - "fig.update_layout(\n", - " width=800,\n", - " height=600,\n", - " showlegend=True\n", - ")\n", - "\n", - "fig.show()" + "correlations.lagged().round(2)" ] }, { "cell_type": "markdown", + "id": "12", "metadata": {}, "source": [ - "## 4. Machine Learning Predictions {#ml}\n", - "\n", - "Let's use machine learning to predict bee migration patterns." + "## 4. Regions" ] }, { "cell_type": "code", "execution_count": null, + "id": "13", "metadata": {}, "outputs": [], "source": [ - "# Initialize the predictor\n", - "try:\n", - " predictor = BeeMigrationPredictor()\n", - " \n", - " # Prepare features and target\n", - " features = ['temperature', 'latitude', 'longitude']\n", - " target = 'species_count'\n", - " \n", - " X = data[features]\n", - " y = data[target]\n", - " \n", - " print(f\"Features: {features}\")\n", - " print(f\"Target: {target}\")\n", - " print(f\"Training data shape: {X.shape}\")\n", - " \n", - "except Exception as e:\n", - " print(f\"Error initializing predictor: {e}\")\n", - " print(\"Continuing with basic analysis...\")" + "from bee_migration.analysis.geographic import GeographicPatterns\n", + "\n", + "GeographicPatterns(observations).hotspots().round(2)" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", + "id": "14", "metadata": {}, - "outputs": [], "source": [ - "# Simple linear regression analysis\n", - "from sklearn.linear_model import LinearRegression\n", - "from sklearn.model_selection import train_test_split\n", - "from sklearn.metrics import r2_score, mean_squared_error\n", - "\n", - "# Split data\n", - "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n", - "\n", - "# Train model\n", - "model = LinearRegression()\n", - "model.fit(X_train, y_train)\n", - "\n", - "# Make predictions\n", - "y_pred = model.predict(X_test)\n", - "\n", - "# Calculate metrics\n", - "r2 = r2_score(y_test, y_pred)\n", - "rmse = np.sqrt(mean_squared_error(y_test, y_pred))\n", + "## 5. Forecasting\n", "\n", - "print(f\"Model Performance:\")\n", - "print(f\"R² Score: {r2:.3f}\")\n", - "print(f\"RMSE: {rmse:.3f}\")\n", - "\n", - "# Feature importance\n", - "feature_importance = pd.DataFrame({\n", - " 'feature': features,\n", - " 'coefficient': model.coef_\n", - "})\n", - "\n", - "print(\"\\nFeature Coefficients:\")\n", - "print(feature_importance)" + "Four models learn bee abundance from climate and from the previous year's value.\n", + "The four most recent years are held out: no model sees them while training.\n", + "`QUICK_CANDIDATES` uses fixed hyperparameters; `DEFAULT_CANDIDATES` searches for\n", + "them and takes longer." ] }, { "cell_type": "code", "execution_count": null, + "id": "15", "metadata": {}, "outputs": [], "source": [ - "# Visualize predictions vs actual\n", - "plt.figure(figsize=(10, 6))\n", - "\n", - "plt.subplot(1, 2, 1)\n", - "plt.scatter(y_test, y_pred, alpha=0.7)\n", - "plt.plot([y_test.min(), y_test.max()], [y_test.min(), y_test.max()], 'r--', lw=2)\n", - "plt.xlabel('Actual Species Count')\n", - "plt.ylabel('Predicted Species Count')\n", - "plt.title(f'Predictions vs Actual (R² = {r2:.3f})')\n", - "plt.grid(True, alpha=0.3)\n", - "\n", - "plt.subplot(1, 2, 2)\n", - "residuals = y_test - y_pred\n", - "plt.scatter(y_pred, residuals, alpha=0.7)\n", - "plt.axhline(y=0, color='r', linestyle='--')\n", - "plt.xlabel('Predicted Species Count')\n", - "plt.ylabel('Residuals')\n", - "plt.title('Residual Plot')\n", - "plt.grid(True, alpha=0.3)\n", + "from bee_migration.prediction.catalog import QUICK_CANDIDATES\n", + "from bee_migration.prediction.training import MigrationPredictor\n", "\n", - "plt.tight_layout()\n", - "plt.show()" + "leaderboard = MigrationPredictor(QUICK_CANDIDATES, holdout_years=4).train(observations)\n", + "leaderboard.metrics()[[\"test_r2\", \"test_rmse\", \"train_r2\"]].round(3)" ] }, { "cell_type": "markdown", + "id": "16", "metadata": {}, "source": [ - "## 5. Visualization and Results {#viz}\n", - "\n", - "Let's create comprehensive visualizations of our findings." + "What the best model relies on, measured by shuffling each feature:" ] }, { "cell_type": "code", "execution_count": null, + "id": "17", "metadata": {}, "outputs": [], "source": [ - "# Create a comprehensive dashboard\n", - "fig = make_subplots(\n", - " rows=2, cols=2,\n", - " subplot_titles=('Temperature Trend', 'Species Decline', 'Geographic Distribution', 'Climate Impact'),\n", - " specs=[[{'secondary_y': True}, {'secondary_y': False}],\n", - " [{'type': 'geo'}, {'secondary_y': False}]]\n", - ")\n", - "\n", - "# Temperature and species trend\n", - "fig.add_trace(\n", - " go.Scatter(x=data['year'], y=data['temperature'], name='Temperature', line=dict(color='red')),\n", - " row=1, col=1\n", - ")\n", - "\n", - "fig.add_trace(\n", - " go.Scatter(x=data['year'], y=data['species_count'], name='Species Count', line=dict(color='blue')),\n", - " row=1, col=1, secondary_y=True\n", - ")\n", - "\n", - "# Species decline\n", - "decline_data = data.groupby('year')['species_count'].mean().reset_index()\n", - "fig.add_trace(\n", - " go.Scatter(x=decline_data['year'], y=decline_data['species_count'], \n", - " mode='lines+markers', name='Average Species Count', line=dict(color='green')),\n", - " row=1, col=2\n", - ")\n", - "\n", - "# Geographic distribution\n", - "fig.add_trace(\n", - " go.Scattergeo(\n", - " lat=data['latitude'],\n", - " lon=data['longitude'],\n", - " mode='markers',\n", - " marker=dict(\n", - " size=data['species_count']/5,\n", - " color=data['temperature'],\n", - " colorscale='RdYlBu_r',\n", - " showscale=True,\n", - " colorbar=dict(title='Temperature (°C)')\n", - " ),\n", - " name='Bee Populations'\n", - " ),\n", - " row=2, col=1\n", - ")\n", - "\n", - "# Climate impact summary\n", - "impact_categories = ['High Risk', 'Medium Risk', 'Low Risk']\n", - "impact_values = [30, 45, 25] # Example percentages\n", - "\n", - "fig.add_trace(\n", - " go.Bar(x=impact_categories, y=impact_values, name='Climate Risk', \n", - " marker=dict(color=['red', 'orange', 'green'])),\n", - " row=2, col=2\n", - ")\n", - "\n", - "# Update layout\n", - "fig.update_layout(\n", - " title_text='Bee Migration Analysis Dashboard',\n", - " showlegend=True,\n", - " height=800\n", - ")\n", - "\n", - "# Update geo subplot\n", - "fig.update_geos(\n", - " projection_type='natural earth',\n", - " showland=True,\n", - " landcolor='lightgray',\n", - " showocean=True,\n", - " oceancolor='lightblue'\n", - ")\n", - "\n", - "fig.show()" + "best = leaderboard.best()\n", + "print(best.name)\n", + "best.importance.sort_values(ascending=False).head(8).round(3)" ] }, { "cell_type": "markdown", + "id": "18", "metadata": {}, "source": [ - "## 6. Scientific Insights {#insights}\n", + "## 6. Charts\n", "\n", - "Based on our analysis, let's summarize the key scientific findings." + "The chart functions return matplotlib figures, which Jupyter displays when they\n", + "are the last value of a cell." ] }, { "cell_type": "code", "execution_count": null, + "id": "19", "metadata": {}, "outputs": [], "source": [ - "# Calculate key metrics\n", - "temp_increase = data['temperature'].iloc[-1] - data['temperature'].iloc[0]\n", - "species_decline = ((data['species_count'].iloc[0] - data['species_count'].iloc[-1]) / data['species_count'].iloc[0]) * 100\n", - "honey_decline = ((data['honey_production'].iloc[0] - data['honey_production'].iloc[-1]) / data['honey_production'].iloc[0]) * 100\n", - "\n", - "# Migration pattern analysis\n", - "lat_correlation = correlation_matrix.loc['latitude', 'temperature']\n", - "migration_direction = 'southward' if lat_correlation < 0 else 'northward'\n", - "\n", - "print(\"=== SCIENTIFIC FINDINGS ===")\n", - "print(f\"\\n🌡️ CLIMATE CHANGE IMPACT:\")\n", - "print(f\" • Temperature increase over study period: {temp_increase:.2f}°C\")\n", - "print(f\" • Critical threshold identified: 2.5°C (literature consensus)\")\n", - "\n", - "print(f\"🐝 BEE POPULATION TRENDS:\")\n", - "print(f\" • Species count decline: {species_decline:.1f}%\")\n", - "print(f\" • Honey production decline: {honey_decline:.1f}%\")\n", - "print(f\" • Model prediction accuracy (R²): {r2:.3f}\")\n", + "from bee_migration.charts import observed\n", + "from bee_migration.climate.continental import continental_series\n", "\n", - "print(f\"🗺️ MIGRATION PATTERNS:\")\n", - "print(f\" • Primary migration direction: {migration_direction}\")\n", - "print(f\" • Latitude-temperature correlation: {lat_correlation:.3f}\")\n", - "print(f\" • Geographic resilience varies by region\")\n", - "\n", - "print(f\"⚠️ CONSERVATION URGENCY:\")\n", - "print(f\" • Critical conservation window: 5-10 years\")\n", - "print(f\" • Priority areas: Southern Brazil (literature)\")\n", - "print(f\" • Indicator species: Bombus bellicosus\")\n", - "\n", - "print(f'📊 STATISTICAL CONFIDENCE:')\n", - "print(f' • Literature consensus: 85-90% agreement')\n", - "print(f' • Model validation: Cross-validated')\n", - "print(f' • Data sources: GBIF, WorldClim, NASA')" + "series = continental_series(DEFAULT_PERIOD, seed=42).set_index(\"year\")\n", + "observed.temperature_migration_trend(series)" ] }, { "cell_type": "markdown", + "id": "20", "metadata": {}, "source": [ - "## Conclusions\n", - "\n", - "This analysis demonstrates the significant impact of climate change on bee populations and migration patterns. Key findings include:\n", - "\n", - "### 🔬 **Scientific Evidence**\n", - "- Strong negative correlation between temperature increase and bee abundance\n", - "- Clear southward migration pattern in response to warming\n", - "- Critical temperature threshold of 2.5°C identified\n", - "\n", - "### 📈 **Quantitative Results**\n", - "- 15% decline in bee populations over the last decade\n", - "- Machine learning models achieve >80% prediction accuracy\n", - "- Geographic analysis reveals regional vulnerability patterns\n", - "\n", - "### 🎯 **Conservation Implications**\n", - "- Immediate action required within 5-10 year window\n", - "- Southern Brazil identified as critical conservation area\n", - "- *Bombus bellicosus* serves as key indicator species\n", - "\n", - "### 🔮 **Future Research**\n", - "- Expand analysis to additional species and regions\n", - "- Develop real-time monitoring systems\n", - "- Create predictive models for conservation planning\n", - "\n", - "---\n", + "## Next steps\n", "\n", - "**For more information, see the full project documentation and scientific papers synthesis.**" + "- `bee-migration all` runs everything above from the command line and writes the\n", + " reports, tables and charts.\n", + "- The [dashboard](https://digomes87.github.io/Bees/) runs the same package in the\n", + " browser and adds live field data from GBIF and Open-Meteo." ] } ], @@ -520,18 +251,9 @@ "name": "python3" }, "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.0" + "name": "python" } }, "nbformat": 4, - "nbformat_minor": 4 -} \ No newline at end of file + "nbformat_minor": 5 +} diff --git a/pyproject.toml b/pyproject.toml index 2e93e21..1935c88 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=68"] +requires = ["setuptools>=77"] build-backend = "setuptools.build_meta" [project] @@ -9,7 +9,8 @@ authors = [{ name = "Diego Gomes", email = "diego@example.com" }] description = "Climate impact assessment on bee migration patterns in South America" readme = "README.md" requires-python = ">=3.11" -license = { text = "MIT" } +license = "MIT" +license-files = ["LICENSE"] keywords = ["bees", "migration", "climate", "analysis", "machine-learning"] dependencies = [ "numpy>=1.26", @@ -21,7 +22,8 @@ dependencies = [ ] [project.optional-dependencies] -dev = ["pytest>=8.0", "pytest-cov>=5.0", "ruff>=0.6"] +dev = ["pytest>=8.0", "pytest-cov>=5.0", "ruff>=0.6", "pre-commit>=4.0"] +notebook = ["jupyterlab>=4.0"] [project.urls] Homepage = "https://github.com/digomes87/Bees" diff --git a/requirements-dev.txt b/requirements-dev.txt index a6de4e2..41897b0 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,3 +3,4 @@ pytest>=8.0 pytest-cov>=5.0 ruff>=0.6 +pre-commit>=4.0 diff --git a/results/bee_migration_comprehensive_report.txt b/results/bee_migration_comprehensive_report.txt index 9a5875a..28ed866 100644 --- a/results/bee_migration_comprehensive_report.txt +++ b/results/bee_migration_comprehensive_report.txt @@ -5,8 +5,8 @@ Data de Geração: 22/08/2025 16:11:50 ## RESUMO EXECUTIVO -Este relatório analisa evidências científicas sobre a migração de abelhas para o sul -devido às mudanças climáticas, baseado em estudos recentes da UFPR, dataset Dryad +Este relatório analisa evidências científicas sobre a migração de abelhas para o sul +devido às mudanças climáticas, baseado em estudos recentes da UFPR, dataset Dryad e pesquisas internacionais. ## 1. ESTUDO UFPR - ABELHAS SUL-AMERICANAS diff --git a/results/integrated_migration_report.txt b/results/integrated_migration_report.txt index bfc2952..a756ca9 100644 --- a/results/integrated_migration_report.txt +++ b/results/integrated_migration_report.txt @@ -5,8 +5,8 @@ Data de Geração: 22/08/2025 16:14:30 ## RESUMO EXECUTIVO -Este relatório integra dados locais do projeto com evidências científicas de múltiplos -estudos sobre migração de abelhas devido às mudanças climáticas, fornecendo uma +Este relatório integra dados locais do projeto com evidências científicas de múltiplos +estudos sobre migração de abelhas devido às mudanças climáticas, fornecendo uma visão abrangente e comparativa dos padrões observados. ## 1. INTEGRAÇÃO DE DADOS @@ -34,7 +34,7 @@ visão abrangente e comparativa dos padrões observados. - **Consenso**: 85-90% das espécies migram para o sul #### 2.2 Fatores Climáticos: -- **Aumento de Temperatura**: +- **Aumento de Temperatura**: - UFPR: 2.5°C projetado - Dados Locais: 2.1-3.2°C observado/projetado - Consenso: 2-3°C de aumento crítico @@ -151,7 +151,7 @@ visão abrangente e comparativa dos padrões observados. --- **Metodologia de Integração:** -Este relatório combina análise quantitativa de dados locais com revisão sistemática +Este relatório combina análise quantitativa de dados locais com revisão sistemática de literatura científica, utilizando métodos de síntese de evidências e validação cruzada. **Fontes Integradas:** diff --git a/results/scientific_papers_synthesis.md b/results/scientific_papers_synthesis.md index e52314a..51e0e9a 100644 --- a/results/scientific_papers_synthesis.md +++ b/results/scientific_papers_synthesis.md @@ -2,7 +2,7 @@ ## Análise Integrada dos Papers Científicos Fornecidos -**Data:** 22 de agosto de 2025 +**Data:** 22 de agosto de 2025 **Análise baseada em:** 6 fontes científicas principais + dados locais --- @@ -257,24 +257,24 @@ ## 📋 CONCLUSÕES FINAIS ### **CERTEZAS CIENTÍFICAS:** -✅ **Migração para o sul é real e acelerada** -✅ **Bombus bellicosus é indicador validado** -✅ **Campos sulinos são área crítica de conservação** -✅ **Grupos funcionais respondem diferentemente** -✅ **Corredores ecológicos são essenciais** +✅ **Migração para o sul é real e acelerada** +✅ **Bombus bellicosus é indicador validado** +✅ **Campos sulinos são área crítica de conservação** +✅ **Grupos funcionais respondem diferentemente** +✅ **Corredores ecológicos são essenciais** ### **INCERTEZAS PRINCIPAIS:** -❓ **Velocidade exata da migração** -❓ **Capacidade de adaptação local** -❓ **Interações com outros fatores** -❓ **Eficácia de medidas de conservação** -❓ **Impactos socioeconômicos precisos** +❓ **Velocidade exata da migração** +❓ **Capacidade de adaptação local** +❓ **Interações com outros fatores** +❓ **Eficácia de medidas de conservação** +❓ **Impactos socioeconômicos precisos** ### **URGÊNCIA DA AÇÃO:** -🚨 **CRÍTICA** - Janela de oportunidade de 5-10 anos -🕐 **TEMPO LIMITADO** - Mudanças já em curso -💰 **CUSTO-EFETIVO** - Prevenção vs. remediação -🤝 **COOPERAÇÃO** - Necessária entre países +🚨 **CRÍTICA** - Janela de oportunidade de 5-10 anos +🕐 **TEMPO LIMITADO** - Mudanças já em curso +💰 **CUSTO-EFETIVO** - Prevenção vs. remediação +🤝 **COOPERAÇÃO** - Necessária entre países --- @@ -283,19 +283,19 @@ 1. **UFPR** - Mudanças climáticas empurram abelhas sul-americanas para o sul 2. **Dryad Dataset** - Climate-induced range shifts in neotropical bees 3. **PMC 6856679** - Global warming promotes biological invasion of honey bee pest -4. **Ecology Letters** - Warming temperatures restructuring bumble bee communities +4. **Ecology Letters** - Warming temperatures restructuring bumble bee communities 5. **PMC 9030098** - Climate change impacts on island bees (Aegean) 6. **Kaggle** - Climate change datasets (validação) --- -**Análise realizada por:** Sistema de Análise Comparativa -**Metodologia:** Síntese de evidências + validação cruzada -**Confiabilidade:** Alta (convergência de múltiplas fontes) +**Análise realizada por:** Sistema de Análise Comparativa +**Metodologia:** Síntese de evidências + validação cruzada +**Confiabilidade:** Alta (convergência de múltiplas fontes) **Próxima atualização:** Conforme novos dados disponíveis --- *"A convergência de evidências científicas de múltiplas fontes e regiões confirma inequivocamente que as abelhas estão migrando para o sul devido às mudanças climáticas. A janela de oportunidade para ação conservacionista é limitada e crítica."* -**- Síntese Científica Integrada, Agosto 2025** \ No newline at end of file +**- Síntese Científica Integrada, Agosto 2025** diff --git a/tests/test_notebook.py b/tests/test_notebook.py new file mode 100644 index 0000000..891ad9d --- /dev/null +++ b/tests/test_notebook.py @@ -0,0 +1,28 @@ +import json +from pathlib import Path + +import pytest + +NOTEBOOK = Path(__file__).parent.parent / "notebooks" / "bee_migration_tutorial.ipynb" + + +@pytest.fixture(scope="module") +def cells() -> list[dict]: + return json.loads(NOTEBOOK.read_text(encoding="utf-8"))["cells"] + + +def test_tutorial_runs_from_top_to_bottom(cells): + namespace: dict = {} + sources = ["".join(cell["source"]) for cell in cells if cell["cell_type"] == "code"] + + for source in sources: + exec(compile(source, NOTEBOOK.name, "exec"), namespace) # noqa: S102 + + assert namespace["best"].name in namespace["leaderboard"].metrics().index + assert len(namespace["observations"]) == 120 + + +def test_tutorial_is_committed_without_outputs(cells): + outputs = [cell["outputs"] for cell in cells if cell["cell_type"] == "code"] + + assert all(output == [] for output in outputs)