From 11b26282102194763521bd4b443515ff9c339563 Mon Sep 17 00:00:00 2001 From: Diego Go Date: Sun, 11 Oct 2026 09:27:21 +0200 Subject: [PATCH 1/4] docs: rewrite the README and the tutorial notebook State what each dataset is and what it supports, replace the claims the data does not back with the measured results, and document the command line, the Python interface and the dashboard. The notebook was not valid JSON and imported classes that never existed. It now walks through the package and is executed by the test suite. --- .env.example | 284 --------------- README.md | 462 ++++++++---------------- notebooks/bee_migration_tutorial.ipynb | 476 +++++-------------------- tests/test_notebook.py | 28 ++ 4 files changed, 277 insertions(+), 973 deletions(-) delete mode 100644 .env.example create mode 100644 tests/test_notebook.py diff --git a/.env.example b/.env.example deleted file mode 100644 index 09976e1..0000000 --- a/.env.example +++ /dev/null @@ -1,284 +0,0 @@ -# Environment Configuration for Bee Migration Analysis Project -# Copy this file to .env and update the values as needed - -# ============================================================================= -# APPLICATION SETTINGS -# ============================================================================= - -# Application environment (development, testing, production) -APP_ENV=development - -# Application name and version -APP_NAME="Bee Migration Analysis" -APP_VERSION=1.0.0 - -# Debug mode (true/false) -DEBUG=true - -# Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL) -LOG_LEVEL=INFO - -# Log file path (optional) -LOG_FILE=logs/app.log - -# ============================================================================= -# API SETTINGS -# ============================================================================= - -# API host and port -API_HOST=0.0.0.0 -API_PORT=8000 - -# API base URL -API_BASE_URL=http://localhost:8000 - -# API documentation settings -API_DOCS_URL=/docs -API_REDOC_URL=/redoc - -# CORS settings -CORS_ORIGINS=http://localhost:3000,http://localhost:8501,http://localhost:8888 - -# ============================================================================= -# DATABASE SETTINGS -# ============================================================================= - -# PostgreSQL settings (for future use) -DB_HOST=localhost -DB_PORT=5432 -DB_NAME=bee_migration -DB_USER=beeuser -DB_PASSWORD=beepassword -DB_URL=postgresql://beeuser:beepassword@localhost:5432/bee_migration - -# SQLite settings (current) -SQLITE_DB_PATH=data/bee_migration.db - -# ============================================================================= -# REDIS SETTINGS -# ============================================================================= - -# Redis connection -REDIS_HOST=localhost -REDIS_PORT=6379 -REDIS_DB=0 -REDIS_PASSWORD= -REDIS_URL=redis://localhost:6379/0 - -# Cache settings -CACHE_TTL=3600 -CACHE_PREFIX=bee_migration: - -# ============================================================================= -# EXTERNAL API SETTINGS -# ============================================================================= - -# Weather API settings -WEATHER_API_KEY=your_weather_api_key_here -WEATHER_API_URL=https://api.openweathermap.org/data/2.5 - -# GBIF API settings -GBIF_API_URL=https://api.gbif.org/v1 -GBIF_USERNAME=your_gbif_username -GBIF_PASSWORD=your_gbif_password - -# NASA API settings -NASA_API_KEY=your_nasa_api_key_here -NASA_API_URL=https://api.nasa.gov - -# WorldClim API settings -WORLDCLIM_API_URL=https://www.worldclim.org/data - -# ============================================================================= -# FILE PATHS -# ============================================================================= - -# Data directories -DATA_DIR=data -RAW_DATA_DIR=data/raw -PROCESSED_DATA_DIR=data/processed -EXTERNAL_DATA_DIR=data/external - -# Output directories -RESULTS_DIR=results -IMAGES_DIR=images -REPORTS_DIR=results/reports -MODELS_DIR=results/models - -# Temporary directories -TEMP_DIR=temp -CACHE_DIR=.cache - -# ============================================================================= -# MACHINE LEARNING SETTINGS -# ============================================================================= - -# Model settings -MODEL_RANDOM_STATE=42 -MODEL_TEST_SIZE=0.2 -MODEL_CV_FOLDS=5 - -# Training settings -TRAINING_EPOCHS=100 -TRAINING_BATCH_SIZE=32 -TRAINING_LEARNING_RATE=0.001 - -# Model persistence -MODEL_SAVE_PATH=results/models -MODEL_CHECKPOINT_INTERVAL=10 - -# ============================================================================= -# VISUALIZATION SETTINGS -# ============================================================================= - -# Plot settings -PLOT_DPI=300 -PLOT_FORMAT=png -PLOT_STYLE=seaborn-v0_8 -PLOT_FIGSIZE_WIDTH=12 -PLOT_FIGSIZE_HEIGHT=8 - -# Color settings -PLOT_COLOR_PALETTE=viridis -PLOT_BACKGROUND_COLOR=white - -# ============================================================================= -# DASHBOARD SETTINGS -# ============================================================================= - -# Streamlit settings -STREAMLIT_SERVER_ADDRESS=0.0.0.0 -STREAMLIT_SERVER_PORT=8501 -STREAMLIT_THEME_PRIMARY_COLOR=#1f77b4 -STREAMLIT_THEME_BACKGROUND_COLOR=#ffffff - -# Dashboard refresh interval (seconds) -DASHBOARD_REFRESH_INTERVAL=300 - -# ============================================================================= -# JUPYTER SETTINGS -# ============================================================================= - -# Jupyter notebook settings -JUPYTER_PORT=8888 -JUPYTER_IP=0.0.0.0 -JUPYTER_TOKEN= -JUPYTER_PASSWORD= - -# ============================================================================= -# SECURITY SETTINGS -# ============================================================================= - -# Secret key for sessions (generate a secure random string) -SECRET_KEY=your_secret_key_here_change_this_in_production - -# JWT settings -JWT_SECRET_KEY=your_jwt_secret_key_here -JWT_ALGORITHM=HS256 -JWT_EXPIRATION_HOURS=24 - -# API rate limiting -RATE_LIMIT_PER_MINUTE=60 -RATE_LIMIT_PER_HOUR=1000 - -# ============================================================================= -# MONITORING SETTINGS -# ============================================================================= - -# Prometheus settings -PROMETHEUS_PORT=9090 -PROMETHEUS_METRICS_PATH=/metrics - -# Grafana settings -GRAFANA_PORT=3000 -GRAFANA_ADMIN_PASSWORD=admin - -# Health check settings -HEALTH_CHECK_INTERVAL=30 -HEALTH_CHECK_TIMEOUT=10 - -# ============================================================================= -# PERFORMANCE SETTINGS -# ============================================================================= - -# Parallel processing -N_JOBS=-1 -PARALLEL_BACKEND=threading - -# Memory settings -MAX_MEMORY_USAGE_GB=8 -CHUNK_SIZE=10000 - -# Caching -ENABLE_CACHING=true -CACHE_SIZE_MB=512 - -# ============================================================================= -# DEVELOPMENT SETTINGS -# ============================================================================= - -# Development tools -ENABLE_PROFILING=false -ENABLE_DEBUG_TOOLBAR=false - -# Testing settings -TEST_DATABASE_URL=sqlite:///test_bee_migration.db -TEST_DATA_DIR=tests/data - -# Code quality -ENABLE_TYPE_CHECKING=true -ENABLE_LINTING=true - -# ============================================================================= -# DOCKER SETTINGS -# ============================================================================= - -# Docker compose settings -COMPOSE_PROJECT_NAME=bee-migration -COMPOSE_FILE=docker-compose.yml - -# Container settings -CONTAINER_TIMEZONE=UTC -CONTAINER_LOCALE=en_US.UTF-8 - -# ============================================================================= -# BACKUP SETTINGS -# ============================================================================= - -# Backup settings -BACKUP_DIR=backups -BACKUP_RETENTION_DAYS=30 -AUTO_BACKUP_ENABLED=false -BACKUP_SCHEDULE=0 2 * * * - -# ============================================================================= -# NOTIFICATION SETTINGS -# ============================================================================= - -# Email settings (for notifications) -SMTP_HOST=smtp.gmail.com -SMTP_PORT=587 -SMTP_USERNAME=your_email@gmail.com -SMTP_PASSWORD=your_app_password -SMTP_USE_TLS=true - -# Slack settings (for notifications) -SLACK_WEBHOOK_URL=https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK -SLACK_CHANNEL=#bee-migration - -# ============================================================================= -# FEATURE FLAGS -# ============================================================================= - -# Feature toggles -ENABLE_REAL_TIME_ANALYSIS=false -ENABLE_ADVANCED_ML_MODELS=true -ENABLE_GEOSPATIAL_ANALYSIS=true -ENABLE_CLIMATE_INTEGRATION=true -ENABLE_API_ENDPOINTS=true -ENABLE_DASHBOARD=true -ENABLE_EXPORT_FEATURES=true - -# Experimental features -ENABLE_EXPERIMENTAL_FEATURES=false -ENABLE_BETA_FEATURES=false \ No newline at end of file diff --git a/README.md b/README.md index f45cc48..c74b5ed 100644 --- a/README.md +++ b/README.md @@ -1,258 +1,48 @@ -# 🐝 Bee Climate Analysis +# Bee Migration Analysis -[![Python](https://img.shields.io/badge/Python-3.8+-blue.svg)](https://www.python.org/downloads/) -[![License](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE) -[![Status](https://img.shields.io/badge/Status-Active-brightgreen.svg)]() -[![Scientific Papers](https://img.shields.io/badge/Papers-6%20Integrated-blue.svg)]() -[![Evidence Level](https://img.shields.io/badge/Evidence-Very%20High-green.svg)]() +[![CI](https://github.com/digomes87/Bees/actions/workflows/ci.yml/badge.svg)](https://github.com/digomes87/Bees/actions/workflows/ci.yml) +[![Python](https://img.shields.io/badge/Python-3.11+-blue.svg)](https://www.python.org/downloads/) +[![Dashboard](https://img.shields.io/badge/Dashboard-live-b8740a.svg)](https://digomes87.github.io/Bees/) -> Analysis of climate change impact on South American bee populations using Machine Learning and data analysis. +A Python package and a browser dashboard for studying how climate relates to +bee populations in South America: regional trends, correlations, and models +that forecast bee measurements from climate. -## 🚨 **CRITICAL SCIENTIFIC DISCOVERY** +**[Open the dashboard](https://digomes87.github.io/Bees/)** — it runs this +package in your browser, nothing to install. -> **📊 SCIENTIFIC CONSENSUS CONFIRMED**: Integrated analysis of 6 scientific studies confirms that **85-90% of bees are migrating southward** due to climate change. -> -> **⏰ URGENCY**: Conservation window of only **5-10 years** for effective action. -> -> **📍 EMBLEMATIC CASE**: *Bombus bellicosus* already locally extinct in Curitiba. -> -> **🗺️ CRITICAL AREA**: Southern Brazilian grasslands identified as priority conservation zone. +## What the data is -## 📋 About the Project +Read this before quoting any number from the project. -This project investigates how climate change affects bee populations in South America, using advanced data analysis techniques and machine learning to identify patterns, correlations, and predict future scenarios. +| Data | Where it comes from | What it supports | +|------|--------------------|------------------| +| Regional observations (temperature, precipitation, bee abundance, diversity, honey) | **Simulated** from assumed warming and decline rates | Demonstrating the method. No scientific conclusion. | +| Observed temperature | Open-Meteo archive (ERA5 reanalysis), fetched live | Real warming at five points, one per Brazilian region | +| Bee occurrence records | GBIF, family Apidae, fetched live | Where bees were *recorded*, which also reflects where people looked | +| Figures in the literature charts | Entered by hand as a reading of the cited studies | Illustration only; not extracted from the studies' datasets | -### 🎯 Objectives +The question in the project's name, whether bees are moving south, is **not +answered** by this repository. The cited literature projects southward range +shifts for south-eastern South America. The live field chapter, loaded on +2026-10-11, showed +0.10 °C per decade at the five region centres and a mean +latitude of GBIF records that barely moved (−0.1° per decade over 136,300 +records). Record locations follow observers as much as bees, so that neither +confirms nor rules out a shift. -- **Exploratory Analysis**: Investigate spatial and temporal distribution of bees -- **Climate Correlation**: Quantify relationships between climate and bee abundance -- **ML Prediction**: Develop models to predict migration and abundance -- **Visualization**: Create informative maps and charts -- **Reports**: Generate actionable insights for conservation - -## 🚀 Quick Start +## Quick start ```bash -# Clone the repository git clone https://github.com/digomes87/Bees.git cd Bees - -# Install the package pip install -e . -# Run complete analysis -bee-migration analyze - -# Run ML predictions -bee-migration predict - -# Render the charts -bee-migration charts +bee-migration all # analysis, model comparison and charts ``` -## 📊 Features - -### 🔍 Exploratory Analysis -- Geographic distribution of species -- Temporal abundance analysis -- Climate-biodiversity correlations -- Seasonal pattern identification - -### 🤖 Machine Learning -- **Models**: Random Forest, Gradient Boosting, SVR, Linear Regression -- **Features**: Climate, geographic and temporal variables -- **Metrics**: R², RMSE, MAE, MAPE -- **Validation**: Cross-validation and holdout test - -### 📈 Visualizations -- Species distribution maps -- Temporal trend charts -- Correlation heatmaps -- Feature importance analysis - -## 🏗️ Architecture - -``` -Bees/ -├── src/bee_migration/ -│ ├── climate/ # Observations and simulated datasets -│ ├── analysis/ # Temporal, correlation and geographic patterns -│ ├── prediction/ # Features, model catalog, training, scores -│ ├── evidence/ # Figures summarised from the cited studies -│ ├── charts/ # Chart panels and gallery -│ ├── reports/ # Plain-text reports -│ └── cli.py # `bee-migration` command -├── tests/ # Test suite -├── data/ -│ ├── raw/ # Raw data -│ ├── processed/ # Processed data -│ └── simulated/ # Simulated data -├── results/ -│ ├── models/ # Trained models -│ ├── reports/ # Reports -│ └── *.csv # Results -├── images/ # Generated charts -├── docs/ # Documentation -└── requirements.txt # Dependencies -``` - -## 🛠️ Technologies - -### Core -- **Python 3.8+**: Main language -- **Pandas**: Data manipulation -- **NumPy**: Numerical computing -- **Scikit-learn**: Machine Learning - -### Visualization -- **Matplotlib**: Static charts -- **Seaborn**: Statistical visualizations -- **Plotly**: Interactive charts - -### Geospatial Analysis -- **GeoPandas**: Geographic data -- **Folium**: Interactive maps - -### Data APIs -- **GBIF**: Biodiversity data -- **WorldClim**: Climate data -- **NASA**: Satellite data - -## 📈 Results - -### 🔬 Scientific Discoveries Based on Papers - -#### **Evidence Convergence: Southward Migration** -Integrated analysis of 6 scientific studies confirms consistent pattern of bee migration southward due to climate change: - -- **📊 Scientific Consensus**: 85-90% of studies confirm south/southwest migration -- **🌡️ Critical Threshold**: 2.5°C increase as tipping point -- **📍 Indicator Species**: *Bombus bellicosus* (local extinction in Curitiba) -- **🗺️ Critical Area**: Southern Brazilian grasslands identified as priority conservation zone - -#### **Integrated Studies** -1. **UFPR**: 18 neotropical species, projection until 2050 -2. **Dryad Dataset**: Species distribution modeling -3. **PMC Studies**: Global impacts and emerging pests -4. **Ecology Letters**: Community restructuring (North America) -5. **Aegean Study**: Impacts on island ecosystems - -### 📊 Impacts by Functional Group - -| Group | Impact | Trend | -|-------|--------|-------| -| **Eusocial** | -35% | 🔴 High vulnerability | -| **Solitary** | -20% | 🟡 Moderate vulnerability | -| **Specialists** | -40% | 🔴 Critical | -| **Generalists** | +15% | 🟢 Benefited | - -### 🎯 Temporal Projections - -#### **Migration Scenarios (2024-2050)** -- **🟢 Conservative**: 80% of species migrate by 2050 -- **🟡 Moderate**: 95% of species migrate by 2050 (most likely) -- **🔴 Aggressive**: 100% of species migrate by 2040 - -### ML Performance Metrics -- **Best Model**: Linear Regression (R² = 0.81 on the four most recent years, held out) -- **RMSE**: 5.2 (bee abundance) -- **Data**: simulated observations; see `results/ml_predictions_report.txt` - -### Key Insights -1. **Negative Correlation**: Temperature increase reduces abundance -2. **Seasonality**: Abundance peaks in autumn/winter -3. **Geography**: Mountainous regions more resilient -4. **Trend**: 15% decline in the last decade -5. **🚨 Urgency**: 5-10 year conservation window - -## 📊 Visualizations - -### 🔬 Scientific Paper Analysis - -![UFPR Analysis](images/ufpr_migration_analysis.png) -*Specific analysis of UFPR study on South American bee migration* - -![Comparative Temperature Analysis](images/comparative_temperature_analysis.png) -*Comparison of temperature trends between local data and scientific studies* - -![Migration Patterns Synthesis](images/migration_patterns_synthesis.png) -*Integrated synthesis of migration patterns based on multiple scientific studies* - -### 📈 Local Data Analysis - -![Climate Impact](images/climate_impact_summary.png) -*Summary of climate impact on bee populations* - -![Correlation Analysis](images/correlation_analysis.png) -*Correlation matrix between climate variables and bee abundance* - -![Geographic Distribution](images/geographic_distribution.png) -*Geographic distribution of bee species* +Requires Python 3.11 or newer. -![Species Decline](images/species_honey_decline.png) -*Analysis of honey-producing species decline* - -![Temperature-Migration Trend](images/temperature_migration_trend.png) -*Relationship between temperature increase and migration patterns* - -## 🔬 Scientific Paper Discoveries - -### 📋 Evidence Synthesis - -This project integrated analysis of **6 main scientific studies** on bee migration due to climate change: - -#### 🎯 **Key Findings** - -1. **🌍 Global Consensus**: 85-90% of studies confirm south/southwest migration -2. **🌡️ Critical Threshold**: 2.5°C warming as tipping point -3. **📍 Emblematic Case**: *Bombus bellicosus* - local extinction in Curitiba -4. **🗺️ Priority Area**: Southern Brazilian grasslands as critical zone -5. **⏰ Temporal Urgency**: 5-10 year conservation window - -#### 📊 **Scientific Validation** - -| Aspect | Evidence Level | Converging Sources | -|--------|---------------|-------------------| -| **Southward Direction** | ⭐⭐⭐⭐⭐ Very High | 5/5 studies | -| **Speed** | ⭐⭐⭐⭐ High | 4/5 studies | -| **Climate Factors** | ⭐⭐⭐⭐⭐ Very High | 5/5 studies | -| **Vulnerable Groups** | ⭐⭐⭐⭐ High | 4/5 studies | - -### 🎯 **Evidence-Based Recommendations** - -#### 🚨 **Immediate Actions (2024-2027)** -- ✅ Urgent protection of Brazilian southern grasslands -- 🔍 Continuous monitoring of indicator species -- 🌿 Creation of north-south ecological corridors -- ⚠️ Control of emerging pests (Small Hive Beetle) - -#### 🔄 **Medium Term (2027-2035)** -- 🌱 Habitat restoration in the south -- 🚀 Assisted relocation programs -- 📊 Expansion of scientific monitoring -- 🏛️ Integration into public policies - -#### 🌟 **Long Term (2035-2050)** -- 🌾 Agricultural landscape adaptation -- 🌎 Transnational conservation networks -- 🧬 Genetic improvement programs -- 📜 Climate adaptation policies - -### 📚 **Generated Reports** - -- 📄 [Comprehensive Migration Report](results/bee_migration_comprehensive_report.txt) -- 📊 [Integrated Analysis](results/integrated_migration_report.txt) -- 🔬 [Scientific Papers Synthesis](results/scientific_papers_synthesis.md) - -### 🔗 **Integrated Scientific Sources** - -1. **UFPR** - Climate change pushes South American bees southward -2. **Dryad Dataset** - Climate-induced range shifts in neotropical bees -3. **PMC Studies** - Global warming and honey bee pests + Island bee impacts -4. **Ecology Letters** - Bumble bee community restructuring (North America) -5. **Kaggle Climate Data** - Climate trend validation - -## 🔧 Usage +## Usage ### Command line @@ -267,10 +57,17 @@ bee-migration analyze --data my_observations.csv # use your own data bee-migration analyze --root /tmp/run --seed 7 # output folder and random seed ``` -A CSV passed with `--data` needs the columns `year`, `region`, `latitude`, -`longitude`, `temperature`, `precipitation`, `bee_abundance`, -`species_diversity` and `honey_production`. Without `--data` the commands run -on simulated observations (2000-2023, five Brazilian regions). +A CSV passed with `--data` needs one row per region and year, with the columns +`year`, `region`, `latitude`, `longitude`, `temperature`, `precipitation`, +`bee_abundance`, `species_diversity` and `honey_production`. Years must be +consecutive within each region. Without `--data` the commands run on simulated +observations (2000–2023, five Brazilian regions). + +| Command | Writes | +|---------|--------| +| `analyze` | `results/analysis_summary_report.txt`, `results/correlation_analysis_correlation_matrix.csv`, `data/processed/bee_climate_processed.csv` | +| `predict` | `results/ml_predictions_report.txt`, `results/model_metrics.csv`, `results/feature_importance.csv`, `results/models/*.pkl` | +| `charts` | `images/*.png`, `data/bee_climate_data.csv`, `data/bee_geographic_data.csv` | ### Python @@ -291,13 +88,14 @@ leaderboard = MigrationPredictor().train(observations) best = leaderboard.best() ``` +`notebooks/bee_migration_tutorial.ipynb` walks through the same steps. + ### In the browser -The dashboard in `docs/` runs this same package inside the page with -[Pyodide](https://pyodide.org/): choose a seed, the number of held-out years and -the measurement to forecast, and the study is computed on your machine. A -separate chapter loads observed temperatures from Open-Meteo and bee occurrence -records from GBIF. +The dashboard in `docs/` loads this package into the page with +[Pyodide](https://pyodide.org/). Choose a seed, the number of held-out years and +the measurement to forecast; the study is computed on your machine. A separate +chapter fetches the field data described above. ```bash make site # builds the package into docs/package and serves http://127.0.0.1:8000 @@ -305,90 +103,130 @@ make site # builds the package into docs/package and serves http://127.0.0.1: Pushes to `main` publish the site through the `Pages` workflow. -## 🧪 Tests +## How it works + +1. **Observations** — yearly climate and bee measurements per region, validated + on load, with anomalies, a climate stress index and a bee health index derived + from them. +2. **Analysis** — linear trend per region, sharp bends in a series, correlations + (overall, per region, and with bee abundance lagged by one to three years), + and regional summaries. +3. **Prediction** — four regressors (linear, random forest, gradient boosting, + SVR) learn a bee measurement from climate and from the previous year's value. + They are tuned with time-ordered cross-validation and scored on the most + recent years, which they never see while training. +4. **Outputs** — plain-text reports, CSV tables, charts, or JSON for the + dashboard. -```bash -pip install -e ".[dev]" +``` +Bees/ +├── src/bee_migration/ +│ ├── climate/ # observations and simulated datasets +│ ├── analysis/ # temporal, correlation and geographic patterns +│ ├── prediction/ # features, model catalog, training, scores +│ ├── sources/ # GBIF and Open-Meteo answers into tables +│ ├── evidence/ # figures summarised from the cited studies +│ ├── charts/ # chart panels and gallery +│ ├── reports/ # plain-text reports +│ ├── web/ # studies as JSON for the dashboard +│ └── cli.py # the bee-migration command +├── docs/ # dashboard (HTML, CSS, JavaScript) +├── tests/ # Python tests; tests/web holds the site's tests +├── notebooks/ # tutorial +├── data/ results/ images/ # outputs of the commands above +└── pyproject.toml +``` -pytest # tests with coverage -ruff check src tests # linting -ruff format src tests # formatting +## Results on the simulated observations -node --test "tests/web/*.test.mjs" # site utilities -``` +These describe the simulation's assumptions coming back out of the pipeline. +They are useful for checking the method, not for learning about bees. + +| Model | R² on held-out years | RMSE | +|-------|---------------------|------| +| Linear Regression | 0.81 | 5.2 | +| SVR | 0.78 | 5.6 | +| Random Forest | 0.53 | 8.0 | +| Gradient Boosting | 0.48 | 8.5 | + +Forecasting bee abundance, four most recent years held out, seed 42. Full +figures are in `results/ml_predictions_report.txt` and +`results/model_metrics.csv`. -## 📋 Roadmap +## Charts -### ✅ **Completed** -- [x] Base project structure -- [x] Exploratory data analysis -- [x] Machine learning models -- [x] Visualizations and reports -- [x] **Scientific paper analysis on migration** -- [x] **Scientific evidence integration** -- [x] **Multi-source discovery synthesis** -- [x] **Cross-validation of results** -- [x] **Evidence-based recommendations** +Drawn from simulated continent-wide series: -- [x] Interactive dashboard that runs the package in the browser -- [x] Live field data from GBIF and Open-Meteo +![Temperature and migration trend](images/temperature_migration_trend.png) +![Correlation analysis](images/correlation_analysis.png) +![Species and honey decline](images/species_honey_decline.png) +![Geographic distribution](images/geographic_distribution.png) +![Climate impact summary](images/climate_impact_summary.png) -### 🚧 **In Development** -- [ ] REST API for predictions -- [ ] Cloud deployment +Drawn from figures entered by hand as a reading of the literature: -### 🔮 **Future** -- [ ] Real-time monitoring of indicator species -- [ ] Integration with climate alert systems -- [ ] Scientific collaboration platform -- [ ] Mobile app for citizen science +![UFPR migration analysis](images/ufpr_migration_analysis.png) +![Comparative temperature analysis](images/comparative_temperature_analysis.png) +![Migration patterns synthesis](images/migration_patterns_synthesis.png) -## 🤝 Contributing +## Literature behind the project -1. **Fork** the project -2. **Clone** your fork -3. **Create** a branch (`git checkout -b feature/new-feature`) -4. **Commit** your changes (`git commit -am 'Add new feature'`) -5. **Push** to the branch (`git push origin feature/new-feature`) -6. **Open** a Pull Request +The project was motivated by published work on climate-driven range shifts in +bees. A synthesis in Portuguese is in `results/scientific_papers_synthesis.md`. -### Guidelines -- Follow PEP 8 for Python code -- Add tests for new features -- Document changes in CHANGELOG -- Use semantic commits +- UFPR — climate change pushing South American bees southward (18 species, + south-eastern South America, projections to 2050) +- Dryad dataset — climate-induced range shifts in the south-eastern Neotropics +- Ecology Letters — warming temperatures restructuring bumble bee communities in + North America +- PMC — global warming and the small hive beetle, a honey bee pest +- PMC — climate change impacts on island bees in the Aegean Archipelago -## 📄 License +## Development -This project is licensed under the MIT License - see [LICENSE](LICENSE) for details. +```bash +pip install -e ".[dev]" + +pytest # Python tests with coverage +ruff check src tests # linting +ruff format src tests # formatting +node --test "tests/web/*.test.mjs" # dashboard utilities +``` + +The project follows Git Flow: work branches from `develop`, pull requests into +`develop`, releases from `develop` into `main`. The `CI` workflow must pass +before anything is merged. See [CONTRIBUTING.md](CONTRIBUTING.md). + +Docker is optional: + +```bash +docker compose run --rm analysis all # run the pipeline in a container +docker compose up site # serve the dashboard on http://127.0.0.1:8000 +``` -## 👥 Team +## Roadmap -- **Diego Gomes** - *Lead Developer* - [@digomes87](https://github.com/digomes87) +Done: -## 🙏 Acknowledgments +- Tested package and command line +- Dashboard that runs the package in the browser +- Live field data from GBIF and Open-Meteo -### 🔬 **Scientific Sources** -- **UFPR** - Federal University of Paraná (South American migration study) -- **Dryad Digital Repository** - Neotropical bee dataset -- **PMC (PubMed Central)** - Global warming and bee studies -- **Ecology Letters** - Community restructuring research -- **Kaggle** - Climate change datasets +Open: -### 🌐 **Data and Infrastructure** -- **GBIF** - Biodiversity data -- **WorldClim** - Climate data -- **NASA** - Satellite data -- **Python Community** - Libraries and tools +- Field data from the command line, not only in the dashboard +- Cross-validation folds cut on year boundaries for any table shape +- REST API for predictions, if something needs to consume it +- Occurrence data corrected for sampling effort, so record locations can say + something about ranges -## 📞 Contact +## Acknowledgments -- **GitHub**: [@digomes87](https://github.com/digomes87) -- **Issues**: [GitHub Issues](https://github.com/digomes87/Bees/issues) -- **Discussions**: [GitHub Discussions](https://github.com/digomes87/Bees/discussions) +Data: [GBIF](https://www.gbif.org/) and [Open-Meteo](https://open-meteo.com/) +(ERA5 reanalysis from Copernicus/ECMWF). Studies: UFPR, Dryad, Ecology Letters +and PMC, as listed above. ---- +## Author -
- 🐝 Bee Climate Analysis - Protecting Pollinators Through Data Science 🐝 -
\ No newline at end of file +Diego Gomes — [@digomes87](https://github.com/digomes87). +Questions and bugs: [issues](https://github.com/digomes87/Bees/issues). diff --git a/notebooks/bee_migration_tutorial.ipynb b/notebooks/bee_migration_tutorial.ipynb index 1288534..a495a4f 100644 --- a/notebooks/bee_migration_tutorial.ipynb +++ b/notebooks/bee_migration_tutorial.ipynb @@ -2,514 +2,245 @@ "cells": [ { "cell_type": "markdown", + "id": "0", "metadata": {}, "source": [ - "# Bee Migration Analysis Tutorial\n", + "# Bee migration tutorial\n", "\n", - "This notebook demonstrates how to use the Bee Migration Analysis project to analyze climate change impacts on bee populations.\n", + "A walk through the `bee_migration` package: build observations, look at trends\n", + "and correlations, then train models that forecast bee abundance.\n", "\n", - "## Table of Contents\n", + "The observations here are **simulated** from assumed warming and decline rates.\n", + "They show how the method works; they are not field measurements.\n", "\n", - "1. [Setup and Data Loading](#setup)\n", - "2. [Exploratory Data Analysis](#eda)\n", - "3. [Climate Impact Analysis](#climate)\n", - "4. [Machine Learning Predictions](#ml)\n", - "5. [Visualization and Results](#viz)\n", - "6. [Scientific Insights](#insights)\n" + "Install the package first: `pip install -e \".[notebook]\"`." ] }, { "cell_type": "markdown", + "id": "1", "metadata": {}, "source": [ - "## 1. Setup and Data Loading {#setup}\n", - "\n", - "First, let's import the necessary libraries and load our data." + "## 1. Observations" ] }, { "cell_type": "code", "execution_count": null, + "id": "2", "metadata": {}, "outputs": [], "source": [ - "# Import necessary libraries\n", - "import sys\n", - "import os\n", - "\n", - "# Add src directory to path\n", - "sys.path.append('../src')\n", + "from bee_migration.climate.period import DEFAULT_PERIOD\n", + "from bee_migration.climate.region import BRAZILIAN_REGIONS\n", + "from bee_migration.climate.simulation import ClimateSimulation\n", "\n", - "import numpy as np\n", - "import pandas as pd\n", - "import matplotlib.pyplot as plt\n", - "import seaborn as sns\n", - "import plotly.express as px\n", - "import plotly.graph_objects as go\n", - "from plotly.subplots import make_subplots\n", + "simulation = ClimateSimulation(BRAZILIAN_REGIONS, DEFAULT_PERIOD)\n", + "observations = simulation.observations(seed=42)\n", "\n", - "# Import project modules\n", - "from bee_analysis import BeeAnalysis\n", - "from bee_migration_predictor import BeeMigrationPredictor\n", - "from config import Config\n", + "observations.table().head()" + ] + }, + { + "cell_type": "markdown", + "id": "3", + "metadata": {}, + "source": [ + "Each row is one region in one year. Besides the measurements, the table carries\n", + "indices derived from them: anomalies against the regional mean, a climate stress\n", + "index and a bee health index.\n", "\n", - "# Set style\n", - "plt.style.use('seaborn-v0_8')\n", - "sns.set_palette('husl')\n", + "To use your own data, load a CSV with the same measurement columns:\n", "\n", - "# Configure display\n", - "pd.set_option('display.max_columns', None)\n", - "pd.set_option('display.width', None)\n", + "```python\n", + "from pathlib import Path\n", + "from bee_migration.climate.observations import Observations\n", "\n", - "print(\"Setup completed successfully!\")\n", - "print(f\"Working directory: {os.getcwd()}\")" + "observations = Observations.from_csv(Path(\"my_observations.csv\"))\n", + "```" ] }, { "cell_type": "markdown", + "id": "4", "metadata": {}, "source": [ - "### Load and Inspect Data\n", - "\n", - "Let's load our bee climate data and take a first look at it." + "## 2. Trends" ] }, { "cell_type": "code", "execution_count": null, + "id": "5", "metadata": {}, "outputs": [], "source": [ - "# Initialize the analysis\n", - "analyzer = BeeAnalysis()\n", + "from bee_migration.analysis.temporal import TemporalPatterns\n", "\n", - "# Load data\n", - "try:\n", - " data = pd.read_csv('../data/bee_climate_data.csv')\n", - " print(f\"Data loaded successfully! Shape: {data.shape}\")\n", - "except FileNotFoundError:\n", - " print(\"Data file not found. Generating sample data...\")\n", - " # Generate sample data for demonstration\n", - " from generate_charts_english import generate_sample_data\n", - " data = generate_sample_data()\n", - " print(f\"Sample data generated! Shape: {data.shape}\")\n", - "\n", - "# Display basic information\n", - "print(\"\\nDataset Info:\")\n", - "print(f\"Columns: {list(data.columns)}\")\n", - "print(f\"Date range: {data['year'].min()} - {data['year'].max()}\")\n", - "print(f\"Missing values: {data.isnull().sum().sum()}\")\n", - "\n", - "# Display first few rows\n", - "data.head()" + "patterns = TemporalPatterns(observations)\n", + "patterns.trends().round(3)" ] }, { "cell_type": "markdown", + "id": "6", "metadata": {}, "source": [ - "## 2. Exploratory Data Analysis {#eda}\n", - "\n", - "Let's explore the data to understand patterns and trends." + "Values are the change per year. Years where a series bends sharply:" ] }, { "cell_type": "code", "execution_count": null, + "id": "7", "metadata": {}, "outputs": [], "source": [ - "# Basic statistics\n", - "print(\"Dataset Statistics:\")\n", - "print(data.describe())\n", - "\n", - "# Check data types\n", - "print(\"\\nData Types:\")\n", - "print(data.dtypes)" + "patterns.change_points()" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", + "id": "8", "metadata": {}, - "outputs": [], "source": [ - "# Visualize temperature trends over time\n", - "fig, axes = plt.subplots(2, 2, figsize=(15, 10))\n", - "\n", - "# Temperature trend\n", - "axes[0, 0].plot(data['year'], data['temperature'], marker='o', linewidth=2)\n", - "axes[0, 0].set_title('Temperature Trend Over Time')\n", - "axes[0, 0].set_xlabel('Year')\n", - "axes[0, 0].set_ylabel('Temperature (°C)')\n", - "axes[0, 0].grid(True, alpha=0.3)\n", - "\n", - "# Species count trend\n", - "axes[0, 1].plot(data['year'], data['species_count'], marker='s', color='green', linewidth=2)\n", - "axes[0, 1].set_title('Bee Species Count Over Time')\n", - "axes[0, 1].set_xlabel('Year')\n", - "axes[0, 1].set_ylabel('Number of Species')\n", - "axes[0, 1].grid(True, alpha=0.3)\n", - "\n", - "# Honey production trend\n", - "axes[1, 0].plot(data['year'], data['honey_production'], marker='^', color='orange', linewidth=2)\n", - "axes[1, 0].set_title('Honey Production Over Time')\n", - "axes[1, 0].set_xlabel('Year')\n", - "axes[1, 0].set_ylabel('Honey Production (kg)')\n", - "axes[1, 0].grid(True, alpha=0.3)\n", - "\n", - "# Latitude distribution\n", - "axes[1, 1].hist(data['latitude'], bins=20, alpha=0.7, color='purple', edgecolor='black')\n", - "axes[1, 1].set_title('Geographic Distribution (Latitude)')\n", - "axes[1, 1].set_xlabel('Latitude')\n", - "axes[1, 1].set_ylabel('Frequency')\n", - "axes[1, 1].grid(True, alpha=0.3)\n", - "\n", - "plt.tight_layout()\n", - "plt.show()" + "## 3. Correlations" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, + "id": "9", "metadata": {}, + "outputs": [], "source": [ - "## 3. Climate Impact Analysis {#climate}\n", + "from bee_migration.analysis.correlation import Correlations\n", "\n", - "Now let's analyze the relationship between climate variables and bee populations." + "correlations = Correlations(observations)\n", + "[pair.describe() for pair in correlations.notable_pairs()[:5]]" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", + "id": "10", "metadata": {}, - "outputs": [], "source": [ - "# Correlation analysis\n", - "correlation_vars = ['temperature', 'species_count', 'honey_production', 'latitude']\n", - "correlation_matrix = data[correlation_vars].corr()\n", - "\n", - "# Plot correlation heatmap\n", - "plt.figure(figsize=(10, 8))\n", - "sns.heatmap(correlation_matrix, \n", - " annot=True, \n", - " cmap='RdYlBu_r', \n", - " center=0,\n", - " square=True,\n", - " fmt='.3f',\n", - " cbar_kws={'label': 'Correlation Coefficient'})\n", - "plt.title('Climate-Bee Population Correlation Matrix', fontsize=16, pad=20)\n", - "plt.tight_layout()\n", - "plt.show()\n", - "\n", - "print(\"Key Correlations:\")\n", - "print(f\"Temperature vs Species Count: {correlation_matrix.loc['temperature', 'species_count']:.3f}\")\n", - "print(f\"Temperature vs Honey Production: {correlation_matrix.loc['temperature', 'honey_production']:.3f}\")\n", - "print(f\"Latitude vs Temperature: {correlation_matrix.loc['latitude', 'temperature']:.3f}\")" + "Temperature against bee abundance one to three years later, by region:" ] }, { "cell_type": "code", "execution_count": null, + "id": "11", "metadata": {}, "outputs": [], "source": [ - "# Interactive scatter plot with Plotly\n", - "fig = px.scatter(data, \n", - " x='temperature', \n", - " y='species_count',\n", - " color='year',\n", - " size='honey_production',\n", - " hover_data=['latitude', 'longitude'],\n", - " title='Temperature vs Bee Species Count Over Time',\n", - " labels={\n", - " 'temperature': 'Temperature (°C)',\n", - " 'species_count': 'Number of Bee Species',\n", - " 'year': 'Year'\n", - " })\n", - "\n", - "fig.update_layout(\n", - " width=800,\n", - " height=600,\n", - " showlegend=True\n", - ")\n", - "\n", - "fig.show()" + "correlations.lagged().round(2)" ] }, { "cell_type": "markdown", + "id": "12", "metadata": {}, "source": [ - "## 4. Machine Learning Predictions {#ml}\n", - "\n", - "Let's use machine learning to predict bee migration patterns." + "## 4. Regions" ] }, { "cell_type": "code", "execution_count": null, + "id": "13", "metadata": {}, "outputs": [], "source": [ - "# Initialize the predictor\n", - "try:\n", - " predictor = BeeMigrationPredictor()\n", - " \n", - " # Prepare features and target\n", - " features = ['temperature', 'latitude', 'longitude']\n", - " target = 'species_count'\n", - " \n", - " X = data[features]\n", - " y = data[target]\n", - " \n", - " print(f\"Features: {features}\")\n", - " print(f\"Target: {target}\")\n", - " print(f\"Training data shape: {X.shape}\")\n", - " \n", - "except Exception as e:\n", - " print(f\"Error initializing predictor: {e}\")\n", - " print(\"Continuing with basic analysis...\")" + "from bee_migration.analysis.geographic import GeographicPatterns\n", + "\n", + "GeographicPatterns(observations).hotspots().round(2)" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", + "id": "14", "metadata": {}, - "outputs": [], "source": [ - "# Simple linear regression analysis\n", - "from sklearn.linear_model import LinearRegression\n", - "from sklearn.model_selection import train_test_split\n", - "from sklearn.metrics import r2_score, mean_squared_error\n", - "\n", - "# Split data\n", - "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n", - "\n", - "# Train model\n", - "model = LinearRegression()\n", - "model.fit(X_train, y_train)\n", - "\n", - "# Make predictions\n", - "y_pred = model.predict(X_test)\n", - "\n", - "# Calculate metrics\n", - "r2 = r2_score(y_test, y_pred)\n", - "rmse = np.sqrt(mean_squared_error(y_test, y_pred))\n", + "## 5. Forecasting\n", "\n", - "print(f\"Model Performance:\")\n", - "print(f\"R² Score: {r2:.3f}\")\n", - "print(f\"RMSE: {rmse:.3f}\")\n", - "\n", - "# Feature importance\n", - "feature_importance = pd.DataFrame({\n", - " 'feature': features,\n", - " 'coefficient': model.coef_\n", - "})\n", - "\n", - "print(\"\\nFeature Coefficients:\")\n", - "print(feature_importance)" + "Four models learn bee abundance from climate and from the previous year's value.\n", + "The four most recent years are held out: no model sees them while training.\n", + "`QUICK_CANDIDATES` uses fixed hyperparameters; `DEFAULT_CANDIDATES` searches for\n", + "them and takes longer." ] }, { "cell_type": "code", "execution_count": null, + "id": "15", "metadata": {}, "outputs": [], "source": [ - "# Visualize predictions vs actual\n", - "plt.figure(figsize=(10, 6))\n", - "\n", - "plt.subplot(1, 2, 1)\n", - "plt.scatter(y_test, y_pred, alpha=0.7)\n", - "plt.plot([y_test.min(), y_test.max()], [y_test.min(), y_test.max()], 'r--', lw=2)\n", - "plt.xlabel('Actual Species Count')\n", - "plt.ylabel('Predicted Species Count')\n", - "plt.title(f'Predictions vs Actual (R² = {r2:.3f})')\n", - "plt.grid(True, alpha=0.3)\n", - "\n", - "plt.subplot(1, 2, 2)\n", - "residuals = y_test - y_pred\n", - "plt.scatter(y_pred, residuals, alpha=0.7)\n", - "plt.axhline(y=0, color='r', linestyle='--')\n", - "plt.xlabel('Predicted Species Count')\n", - "plt.ylabel('Residuals')\n", - "plt.title('Residual Plot')\n", - "plt.grid(True, alpha=0.3)\n", + "from bee_migration.prediction.catalog import QUICK_CANDIDATES\n", + "from bee_migration.prediction.training import MigrationPredictor\n", "\n", - "plt.tight_layout()\n", - "plt.show()" + "leaderboard = MigrationPredictor(QUICK_CANDIDATES, holdout_years=4).train(observations)\n", + "leaderboard.metrics()[[\"test_r2\", \"test_rmse\", \"train_r2\"]].round(3)" ] }, { "cell_type": "markdown", + "id": "16", "metadata": {}, "source": [ - "## 5. Visualization and Results {#viz}\n", - "\n", - "Let's create comprehensive visualizations of our findings." + "What the best model relies on, measured by shuffling each feature:" ] }, { "cell_type": "code", "execution_count": null, + "id": "17", "metadata": {}, "outputs": [], "source": [ - "# Create a comprehensive dashboard\n", - "fig = make_subplots(\n", - " rows=2, cols=2,\n", - " subplot_titles=('Temperature Trend', 'Species Decline', 'Geographic Distribution', 'Climate Impact'),\n", - " specs=[[{'secondary_y': True}, {'secondary_y': False}],\n", - " [{'type': 'geo'}, {'secondary_y': False}]]\n", - ")\n", - "\n", - "# Temperature and species trend\n", - "fig.add_trace(\n", - " go.Scatter(x=data['year'], y=data['temperature'], name='Temperature', line=dict(color='red')),\n", - " row=1, col=1\n", - ")\n", - "\n", - "fig.add_trace(\n", - " go.Scatter(x=data['year'], y=data['species_count'], name='Species Count', line=dict(color='blue')),\n", - " row=1, col=1, secondary_y=True\n", - ")\n", - "\n", - "# Species decline\n", - "decline_data = data.groupby('year')['species_count'].mean().reset_index()\n", - "fig.add_trace(\n", - " go.Scatter(x=decline_data['year'], y=decline_data['species_count'], \n", - " mode='lines+markers', name='Average Species Count', line=dict(color='green')),\n", - " row=1, col=2\n", - ")\n", - "\n", - "# Geographic distribution\n", - "fig.add_trace(\n", - " go.Scattergeo(\n", - " lat=data['latitude'],\n", - " lon=data['longitude'],\n", - " mode='markers',\n", - " marker=dict(\n", - " size=data['species_count']/5,\n", - " color=data['temperature'],\n", - " colorscale='RdYlBu_r',\n", - " showscale=True,\n", - " colorbar=dict(title='Temperature (°C)')\n", - " ),\n", - " name='Bee Populations'\n", - " ),\n", - " row=2, col=1\n", - ")\n", - "\n", - "# Climate impact summary\n", - "impact_categories = ['High Risk', 'Medium Risk', 'Low Risk']\n", - "impact_values = [30, 45, 25] # Example percentages\n", - "\n", - "fig.add_trace(\n", - " go.Bar(x=impact_categories, y=impact_values, name='Climate Risk', \n", - " marker=dict(color=['red', 'orange', 'green'])),\n", - " row=2, col=2\n", - ")\n", - "\n", - "# Update layout\n", - "fig.update_layout(\n", - " title_text='Bee Migration Analysis Dashboard',\n", - " showlegend=True,\n", - " height=800\n", - ")\n", - "\n", - "# Update geo subplot\n", - "fig.update_geos(\n", - " projection_type='natural earth',\n", - " showland=True,\n", - " landcolor='lightgray',\n", - " showocean=True,\n", - " oceancolor='lightblue'\n", - ")\n", - "\n", - "fig.show()" + "best = leaderboard.best()\n", + "print(best.name)\n", + "best.importance.sort_values(ascending=False).head(8).round(3)" ] }, { "cell_type": "markdown", + "id": "18", "metadata": {}, "source": [ - "## 6. Scientific Insights {#insights}\n", + "## 6. Charts\n", "\n", - "Based on our analysis, let's summarize the key scientific findings." + "The chart functions return matplotlib figures, which Jupyter displays when they\n", + "are the last value of a cell." ] }, { "cell_type": "code", "execution_count": null, + "id": "19", "metadata": {}, "outputs": [], "source": [ - "# Calculate key metrics\n", - "temp_increase = data['temperature'].iloc[-1] - data['temperature'].iloc[0]\n", - "species_decline = ((data['species_count'].iloc[0] - data['species_count'].iloc[-1]) / data['species_count'].iloc[0]) * 100\n", - "honey_decline = ((data['honey_production'].iloc[0] - data['honey_production'].iloc[-1]) / data['honey_production'].iloc[0]) * 100\n", - "\n", - "# Migration pattern analysis\n", - "lat_correlation = correlation_matrix.loc['latitude', 'temperature']\n", - "migration_direction = 'southward' if lat_correlation < 0 else 'northward'\n", - "\n", - "print(\"=== SCIENTIFIC FINDINGS ===")\n", - "print(f\"\\n🌡️ CLIMATE CHANGE IMPACT:\")\n", - "print(f\" • Temperature increase over study period: {temp_increase:.2f}°C\")\n", - "print(f\" • Critical threshold identified: 2.5°C (literature consensus)\")\n", - "\n", - "print(f\"🐝 BEE POPULATION TRENDS:\")\n", - "print(f\" • Species count decline: {species_decline:.1f}%\")\n", - "print(f\" • Honey production decline: {honey_decline:.1f}%\")\n", - "print(f\" • Model prediction accuracy (R²): {r2:.3f}\")\n", + "from bee_migration.charts import observed\n", + "from bee_migration.climate.continental import continental_series\n", "\n", - "print(f\"🗺️ MIGRATION PATTERNS:\")\n", - "print(f\" • Primary migration direction: {migration_direction}\")\n", - "print(f\" • Latitude-temperature correlation: {lat_correlation:.3f}\")\n", - "print(f\" • Geographic resilience varies by region\")\n", - "\n", - "print(f\"⚠️ CONSERVATION URGENCY:\")\n", - "print(f\" • Critical conservation window: 5-10 years\")\n", - "print(f\" • Priority areas: Southern Brazil (literature)\")\n", - "print(f\" • Indicator species: Bombus bellicosus\")\n", - "\n", - "print(f'📊 STATISTICAL CONFIDENCE:')\n", - "print(f' • Literature consensus: 85-90% agreement')\n", - "print(f' • Model validation: Cross-validated')\n", - "print(f' • Data sources: GBIF, WorldClim, NASA')" + "series = continental_series(DEFAULT_PERIOD, seed=42).set_index(\"year\")\n", + "observed.temperature_migration_trend(series)" ] }, { "cell_type": "markdown", + "id": "20", "metadata": {}, "source": [ - "## Conclusions\n", - "\n", - "This analysis demonstrates the significant impact of climate change on bee populations and migration patterns. Key findings include:\n", - "\n", - "### 🔬 **Scientific Evidence**\n", - "- Strong negative correlation between temperature increase and bee abundance\n", - "- Clear southward migration pattern in response to warming\n", - "- Critical temperature threshold of 2.5°C identified\n", - "\n", - "### 📈 **Quantitative Results**\n", - "- 15% decline in bee populations over the last decade\n", - "- Machine learning models achieve >80% prediction accuracy\n", - "- Geographic analysis reveals regional vulnerability patterns\n", - "\n", - "### 🎯 **Conservation Implications**\n", - "- Immediate action required within 5-10 year window\n", - "- Southern Brazil identified as critical conservation area\n", - "- *Bombus bellicosus* serves as key indicator species\n", - "\n", - "### 🔮 **Future Research**\n", - "- Expand analysis to additional species and regions\n", - "- Develop real-time monitoring systems\n", - "- Create predictive models for conservation planning\n", - "\n", - "---\n", + "## Next steps\n", "\n", - "**For more information, see the full project documentation and scientific papers synthesis.**" + "- `bee-migration all` runs everything above from the command line and writes the\n", + " reports, tables and charts.\n", + "- The [dashboard](https://digomes87.github.io/Bees/) runs the same package in the\n", + " browser and adds live field data from GBIF and Open-Meteo." ] } ], @@ -520,18 +251,9 @@ "name": "python3" }, "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.0" + "name": "python" } }, "nbformat": 4, - "nbformat_minor": 4 -} \ No newline at end of file + "nbformat_minor": 5 +} diff --git a/tests/test_notebook.py b/tests/test_notebook.py new file mode 100644 index 0000000..891ad9d --- /dev/null +++ b/tests/test_notebook.py @@ -0,0 +1,28 @@ +import json +from pathlib import Path + +import pytest + +NOTEBOOK = Path(__file__).parent.parent / "notebooks" / "bee_migration_tutorial.ipynb" + + +@pytest.fixture(scope="module") +def cells() -> list[dict]: + return json.loads(NOTEBOOK.read_text(encoding="utf-8"))["cells"] + + +def test_tutorial_runs_from_top_to_bottom(cells): + namespace: dict = {} + sources = ["".join(cell["source"]) for cell in cells if cell["cell_type"] == "code"] + + for source in sources: + exec(compile(source, NOTEBOOK.name, "exec"), namespace) # noqa: S102 + + assert namespace["best"].name in namespace["leaderboard"].metrics().index + assert len(namespace["observations"]) == 120 + + +def test_tutorial_is_committed_without_outputs(cells): + outputs = [cell["outputs"] for cell in cells if cell["cell_type"] == "code"] + + assert all(output == [] for output in outputs) From 73ee56a73f0bd199da3962e3d253a08786e6c648 Mon Sep 17 00:00:00 2001 From: Diego Go Date: Sun, 11 Oct 2026 09:27:21 +0200 Subject: [PATCH 2/4] chore: reduce the tooling to what the project uses - Dockerfile and Compose file run the bee-migration command and serve the dashboard; the stages and services for an API, a database, a cache and monitoring that do not exist are gone, with the unused .env.example - CI builds the image and runs the analysis in it - pre-commit runs ruff, basic file checks and nbstripout - conda environment and Makefile follow pyproject.toml --- .dockerignore | 16 + .github/workflows/ci.yml | 10 + .gitignore | 1 - .pre-commit-config.yaml | 121 +------- CHANGELOG.md | 6 +- CONTRIBUTING.md | 64 +++- Dockerfile | 189 +----------- Makefile | 223 ++++---------- docker-compose.yml | 291 +----------------- environment.yml | 72 +---- pyproject.toml | 3 +- requirements-dev.txt | 1 + .../bee_migration_comprehensive_report.txt | 4 +- results/integrated_migration_report.txt | 8 +- results/scientific_papers_synthesis.md | 40 +-- 15 files changed, 200 insertions(+), 849 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..9e004ee --- /dev/null +++ b/.dockerignore @@ -0,0 +1,16 @@ +.git +.github +.venv +.pytest_cache +.ruff_cache +.coverage +htmlcov +build +**/__pycache__ +**/*.egg-info +docs/package +data +results +images +notebooks +tests diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a9a3ba..24f82f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,6 +36,16 @@ jobs: - name: Test run: pytest + docker: + name: Docker + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: Build + run: docker build --tag bee-migration . + - name: Run the analysis in the container + run: docker run --rm bee-migration analyze + site: name: Site runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index e6057c3..c36bae6 100644 --- a/.gitignore +++ b/.gitignore @@ -280,7 +280,6 @@ conda-meta/ *.m~ # Docker -.dockerignore Dockerfile.dev # Documentation builds diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 604c8e3..d87b90d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,128 +1,25 @@ -# Pre-commit configuration for Bee Migration Analysis Project # Install with: pre-commit install -# Run manually with: pre-commit run --all-files - repos: - # General file checks - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v6.0.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer - id: check-yaml - - id: check-json - id: check-toml - - id: check-xml + - id: check-json - id: check-merge-conflict - - id: check-case-conflict - - id: check-docstring-first - - id: check-executables-have-shebangs - - id: check-shebang-scripts-are-executable - - id: fix-byte-order-marker - - id: mixed-line-ending - - id: requirements-txt-fixer - - id: sort-simple-yaml - - id: file-contents-sorter - files: '^(requirements.*\.txt|\.gitignore)$' - id: check-added-large-files - args: ['--maxkb=10000'] # 10MB limit - - # Python code formatting - - repo: https://github.com/psf/black - rev: 23.3.0 - hooks: - - id: black - language_version: python3 - args: [--line-length=88] - - # Import sorting - - repo: https://github.com/pycqa/isort - rev: 5.12.0 - hooks: - - id: isort - args: [--profile=black, --line-length=88] - - # Linting - - repo: https://github.com/pycqa/flake8 - rev: 6.0.0 - hooks: - - id: flake8 - args: [--max-line-length=88, --extend-ignore=E203,W503] - additional_dependencies: - - flake8-docstrings - - flake8-import-order - - flake8-bugbear + args: [--maxkb=1000] - # Type checking - - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.3.0 + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.17.0 hooks: - - id: mypy - additional_dependencies: [types-requests, types-PyYAML] - args: [--ignore-missing-imports, --no-strict-optional] + - id: ruff-check + args: [--fix] + - id: ruff-format - # Security checks - - repo: https://github.com/pycqa/bandit - rev: 1.7.5 - hooks: - - id: bandit - args: [-r, src/] - exclude: tests/ - - # Documentation - - repo: https://github.com/pycqa/pydocstyle - rev: 6.3.0 - hooks: - - id: pydocstyle - args: [--convention=google] - - # Jupyter notebooks - - repo: https://github.com/nbQA-dev/nbQA - rev: 1.7.0 - hooks: - - id: nbqa-black - additional_dependencies: [black==23.3.0] - - id: nbqa-isort - additional_dependencies: [isort==5.12.0] - - id: nbqa-flake8 - additional_dependencies: [flake8==6.0.0] - - # Remove notebook outputs - repo: https://github.com/kynan/nbstripout - rev: 0.6.1 + rev: 0.9.1 hooks: - id: nbstripout - - # YAML formatting - - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.0.0-alpha.9-for-vscode - hooks: - - id: prettier - types_or: [yaml, markdown] - exclude: '^(CHANGELOG\.md|environment\.yml)$' - - # Shell script linting - - repo: https://github.com/shellcheck-py/shellcheck-py - rev: v0.9.0.2 - hooks: - - id: shellcheck - - # Dockerfile linting - - repo: https://github.com/hadolint/hadolint - rev: v2.12.0 - hooks: - - id: hadolint - args: [--ignore, DL3008, --ignore, DL3009] - -# Configuration for specific hooks -ci: - autofix_commit_msg: | - [pre-commit.ci] auto fixes from pre-commit.com hooks - - for more information, see https://pre-commit.ci - autofix_prs: true - autoupdate_branch: '' - autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate' - autoupdate_schedule: weekly - skip: [] - submodules: false \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c5f83a..ae2d392 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Git Flow branching guide in `CONTRIBUTING.md` ### Changed +- README rewritten around what the data is and what it does not show +- Tutorial notebook rewritten for the package and run by the test suite +- Dockerfile, Compose file, pre-commit hooks, conda environment and Makefile + reduced to what the project uses; the unused `.env.example` is gone - Models are evaluated on the most recent years, held out chronologically, and tuned with time-ordered cross-validation - Feature importance is measured by permutation for every model @@ -124,4 +128,4 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for details on how to contribute to this ## License -This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. \ No newline at end of file +This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5641b61..a73bf07 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,9 +30,10 @@ This project adheres to a code of conduct that promotes a welcoming and inclusiv ### Prerequisites -- Python 3.8 or higher +- Python 3.11 or higher - pip or conda package manager - Git +- Node.js 22 or higher, only to run the dashboard's tests ### Environment Setup @@ -45,24 +46,29 @@ cd Bees python -m venv .venv source .venv/bin/activate # On Windows: .venv\Scripts\activate -# Install dependencies -pip install -r requirements.txt +# Install the package with its development tools +pip install -e ".[dev]" -# Install development dependencies (if available) -pip install -r requirements-dev.txt +# Check your changes before every commit +pre-commit install ``` +With conda: `conda env create -f environment.yml`. + ### Running Tests ```bash # Run all tests pytest -# Run tests with coverage -pytest --cov=src +# Run one test file, without the coverage threshold +pytest tests/test_analysis.py --no-cov + +# Run the dashboard's tests +node --test "tests/web/*.test.mjs" -# Run specific test file -pytest tests/test_specific_module.py +# Try the dashboard locally +make site ``` ## Contributing Guidelines @@ -118,27 +124,51 @@ ruff check src/ tests/ ``` tests/ -├── unit/ # Unit tests -├── integration/ # Integration tests -├── fixtures/ # Test data and fixtures -└── conftest.py # Pytest configuration +├── conftest.py # shared fixtures +├── test_climate.py # observations and simulated datasets +├── test_analysis.py # trends, correlations, regions +├── test_prediction.py # features, models, scores +├── test_sources.py # GBIF and Open-Meteo answers +├── test_web.py # studies as JSON for the dashboard +├── test_outputs.py # reports, charts, command line +├── test_notebook.py # the tutorial runs end to end +└── web/ # dashboard utilities (node --test) ``` ### Writing Tests +Name a test after the behaviour it checks and follow arrange, act, assert: + +```python +from bee_migration.analysis.temporal import TemporalPatterns +from bee_migration.climate.observations import Observations +from tests.conftest import region_table + + +def test_trend_is_the_yearly_slope(): + table = region_table("North", temperature=[10.0, 10.5, 11.0, 11.5]) + + trends = TemporalPatterns(Observations(table)).trends() + + assert trends.loc["North", "temperature"] == 0.5 +``` + +## Writing Tests + ```python import pytest from src.module import function_to_test + def test_function_behavior(): """Test that function behaves correctly.""" # Arrange input_data = "test_input" expected_output = "expected_result" - + # Act result = function_to_test(input_data) - + # Assert assert result == expected_output ``` @@ -264,7 +294,7 @@ gh pr create --base main - [Project Documentation](./README.md) - [Scientific Papers](./results/scientific_papers_synthesis.md) -- [API Documentation](./docs/) +- [Dashboard](https://digomes87.github.io/Bees/) (source in `./docs/`) - [Issue Tracker](https://github.com/digomes87/Bees/issues) ## Questions? @@ -278,4 +308,4 @@ If you have questions about contributing, please: --- -Thank you for contributing to the bee migration analysis project! Your contributions help advance our understanding of climate change impacts on bee populations. 🐝 \ No newline at end of file +Thank you for contributing to the bee migration analysis project! Your contributions help advance our understanding of climate change impacts on bee populations. 🐝 diff --git a/Dockerfile b/Dockerfile index c803c7f..fbaffd1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,184 +1,25 @@ -# Multi-stage Dockerfile for Bee Migration Analysis Project -# This Dockerfile creates a production-ready container with all dependencies +# Runs the bee-migration command, or serves the dashboard (see docker-compose.yml). +FROM python:3.13-slim -# Build stage -FROM python:3.9-slim as builder - -# Set environment variables ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ PIP_NO_CACHE_DIR=1 \ - PIP_DISABLE_PIP_VERSION_CHECK=1 - -# Install system dependencies for building -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ - gcc \ - g++ \ - gfortran \ - libgdal-dev \ - libproj-dev \ - libgeos-dev \ - libspatialindex-dev \ - libffi-dev \ - libssl-dev \ - python3-dev \ - && rm -rf /var/lib/apt/lists/* - -# Set GDAL environment variables -ENV CPLUS_INCLUDE_PATH=/usr/include/gdal -ENV C_INCLUDE_PATH=/usr/include/gdal - -# Create virtual environment -RUN python -m venv /opt/venv -ENV PATH="/opt/venv/bin:$PATH" - -# Upgrade pip and install wheel -RUN pip install --upgrade pip setuptools wheel - -# Copy requirements first for better caching -COPY requirements.txt requirements-dev.txt ./ - -# Install Python dependencies -RUN pip install --no-cache-dir -r requirements.txt - -# Production stage -FROM python:3.9-slim as production - -# Set environment variables -ENV PYTHONDONTWRITEBYTECODE=1 \ - PYTHONUNBUFFERED=1 \ - PATH="/opt/venv/bin:$PATH" \ - PYTHONPATH="/app/src:$PYTHONPATH" + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + MPLCONFIGDIR=/tmp/matplotlib -# Install runtime system dependencies -RUN apt-get update && apt-get install -y --no-install-recommends \ - libgdal28 \ - libproj19 \ - libgeos-c1v5 \ - libspatialindex6 \ - curl \ - && rm -rf /var/lib/apt/lists/* - -# Copy virtual environment from builder stage -COPY --from=builder /opt/venv /opt/venv - -# Create non-root user -RUN groupadd -r beeuser && useradd -r -g beeuser beeuser - -# Create application directory WORKDIR /app -# Copy application code -COPY --chown=beeuser:beeuser . . - -# Create necessary directories -RUN mkdir -p /app/data /app/images /app/results /app/logs && \ - chown -R beeuser:beeuser /app - -# Switch to non-root user -USER beeuser - -# Install the package in development mode -RUN pip install -e . - -# Expose port for potential web services -EXPOSE 8000 - -# Health check -HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 \ - CMD python -c "import sys; sys.exit(0)" || exit 1 - -# Default command -CMD ["python", "-m", "bee_migration.cli", "--help"] - -# Development stage -FROM production as development - -# Switch back to root for installing dev dependencies -USER root - -# Install development dependencies -RUN pip install -r requirements-dev.txt - -# Install additional development tools -RUN apt-get update && apt-get install -y --no-install-recommends \ - git \ - vim \ - htop \ - && rm -rf /var/lib/apt/lists/* - -# Switch back to non-root user -USER beeuser - -# Install pre-commit hooks -RUN pre-commit install || true - -# Default command for development -CMD ["bash"] - -# Jupyter stage -FROM development as jupyter - -# Expose Jupyter port -EXPOSE 8888 - -# Install Jupyter extensions -RUN pip install jupyter-contrib-nbextensions && \ - jupyter contrib nbextension install --user - -# Create Jupyter config -RUN jupyter notebook --generate-config - -# Set Jupyter configuration -RUN echo "c.NotebookApp.ip = '0.0.0.0'" >> ~/.jupyter/jupyter_notebook_config.py && \ - echo "c.NotebookApp.port = 8888" >> ~/.jupyter/jupyter_notebook_config.py && \ - echo "c.NotebookApp.open_browser = False" >> ~/.jupyter/jupyter_notebook_config.py && \ - echo "c.NotebookApp.allow_root = True" >> ~/.jupyter/jupyter_notebook_config.py && \ - echo "c.NotebookApp.token = ''" >> ~/.jupyter/jupyter_notebook_config.py && \ - echo "c.NotebookApp.password = ''" >> ~/.jupyter/jupyter_notebook_config.py - -# Default command for Jupyter -CMD ["jupyter", "notebook", "--notebook-dir=/app/notebooks"] - -# API stage -FROM production as api - -# Install API dependencies -RUN pip install fastapi uvicorn python-multipart - -# Expose API port -EXPOSE 8000 - -# Default command for API -CMD ["uvicorn", "bee_migration.api:app", "--host", "0.0.0.0", "--port", "8000"] - -# Dashboard stage -FROM production as dashboard - -# Install dashboard dependencies -RUN pip install streamlit dash plotly - -# Expose dashboard port -EXPOSE 8501 - -# Default command for dashboard -CMD ["streamlit", "run", "bee_migration/dashboard.py", "--server.address", "0.0.0.0"] +COPY pyproject.toml README.md ./ +COPY src ./src +RUN pip install . -# Testing stage -FROM development as testing +# The dashboard loads the package as a wheel next to its pages. +COPY docs ./docs +RUN pip wheel . --no-deps --wheel-dir docs/package -# Run tests by default -CMD ["pytest", "tests/", "-v", "--cov=src", "--cov-report=term-missing"] +RUN useradd --create-home bee && mkdir /work && chown bee:bee /work +USER bee +WORKDIR /work -# Labels for metadata -LABEL maintainer="Diego Gomes " \ - version="1.0.0" \ - description="Bee Migration Analysis: Climate Impact Assessment" \ - org.opencontainers.image.title="Bee Migration Analysis" \ - org.opencontainers.image.description="Climate Impact Assessment on Bee Migration Patterns" \ - org.opencontainers.image.version="1.0.0" \ - org.opencontainers.image.authors="Diego Gomes" \ - org.opencontainers.image.url="https://github.com/diegogo/bee-migration-analysis" \ - org.opencontainers.image.source="https://github.com/diegogo/bee-migration-analysis" \ - org.opencontainers.image.licenses="MIT" \ No newline at end of file +ENTRYPOINT ["bee-migration"] +CMD ["--help"] diff --git a/Makefile b/Makefile index 4882132..4dcd9bb 100644 --- a/Makefile +++ b/Makefile @@ -1,197 +1,82 @@ -# Makefile for Bee Migration Analysis Project +.PHONY: help install install-dev hooks test test-site lint format format-check \ + analyze predict charts all site notebook docker-build docker-run clean check -.PHONY: help install install-dev test lint format clean run-analysis generate-charts setup-env docs - -# Default target help: - @echo "Available commands:" - @echo " install - Install production dependencies" - @echo " install-dev - Install development dependencies" - @echo " setup-env - Setup development environment" - @echo " test - Run tests" - @echo " test-cov - Run tests with coverage" - @echo " lint - Run linting checks" - @echo " format - Format code with ruff" - @echo " clean - Clean temporary files" - @echo " run-analysis - Run complete bee migration analysis" - @echo " generate-charts - Generate all scientific charts" - @echo " docs - Generate documentation" - @echo " security - Run security checks" - @echo " all - Run format, lint, test, and analysis" - -# Installation + @echo "Setup" + @echo " install Install the package" + @echo " install-dev Install the package with development tools" + @echo " hooks Install the pre-commit hooks" + @echo "Quality" + @echo " test Run the Python tests with coverage" + @echo " test-site Run the dashboard tests" + @echo " lint Check the code with ruff" + @echo " format Format the code with ruff" + @echo " check Everything CI runs: format-check, lint, test, test-site" + @echo "Run" + @echo " analyze Trends, correlations and the summary report" + @echo " predict Train and compare the models" + @echo " charts Render the charts into images/" + @echo " all analyze, predict and charts" + @echo " site Build the package into the dashboard and serve it" + @echo " notebook Open the tutorial in JupyterLab" + @echo "Docker" + @echo " docker-build Build the image" + @echo " docker-run Run the whole pipeline in a container" + @echo "Other" + @echo " clean Remove caches and build leftovers" + install: - @echo "Installing production dependencies..." pip install -e . install-dev: - @echo "Installing development dependencies..." - pip install -e ".[dev]" - -# Environment setup -setup-env: - @echo "Setting up development environment..." - python -m venv .venv - @echo "Virtual environment created. Activate with:" - @echo " source .venv/bin/activate # On Linux/Mac" - @echo " .venv\\Scripts\\activate # On Windows" - -setup-conda: - @echo "Setting up conda environment..." - conda env create -f environment.yml - @echo "Conda environment created. Activate with:" - @echo " conda activate bee-migration-analysis" - -# Testing -test: - @echo "Running tests..." - pytest tests/ -v + pip install -e ".[dev,notebook]" -test-cov: - @echo "Running tests with coverage..." - pytest tests/ -v --cov-report=html +hooks: + pre-commit install + +test: + pytest -test-fast: - @echo "Running fast tests..." - pytest tests/ -v -x --ff +test-site: + node --test "tests/web/*.test.mjs" -# Code quality lint: - @echo "Running linting checks..." - ruff check src/ tests/ + ruff check src tests format: - @echo "Formatting code..." - ruff format src/ tests/ - ruff check --fix src/ tests/ + ruff format src tests + ruff check --fix src tests format-check: - @echo "Checking code format..." - ruff format --check src/ tests/ - -# Security -security: - @echo "Running security checks..." - bandit -r src/ - safety check - -# Analysis and charts -run-analysis: - @echo "Running complete bee migration analysis..." + ruff format --check src tests + +check: format-check lint test test-site + +analyze: bee-migration analyze -generate-charts: - @echo "Generating all scientific charts..." +predict: + bee-migration predict + +charts: bee-migration charts -run-predictor: - @echo "Running migration predictor..." - bee-migration predict +all: + bee-migration all -# Site site: - @echo "Building the analysis package into the site and serving it..." pip wheel . --no-deps --wheel-dir docs/package python -m http.server 8000 --bind 127.0.0.1 --directory docs -test-site: - @echo "Running site tests..." - node --test "tests/web/*.test.mjs" - -# Data processing -process-data: - @echo "Processing raw data..." - @echo "Note: Implement data processing scripts as needed" - -download-data: - @echo "Downloading external data..." - @echo "Note: Implement data download scripts as needed" - -# Documentation -docs: - @echo "Generating documentation..." - @echo "Note: Add sphinx documentation generation here" +notebook: + jupyter lab notebooks/bee_migration_tutorial.ipynb -docs-serve: - @echo "Serving documentation locally..." - @echo "Note: Add local documentation server here" - -# Cleanup -clean: - @echo "Cleaning temporary files..." - find . -type f -name "*.pyc" -delete - find . -type d -name "__pycache__" -delete - find . -type d -name ".pytest_cache" -delete - find . -type f -name ".coverage" -delete - find . -type d -name "htmlcov" -delete - find . -type d -name ".mypy_cache" -delete - find . -type f -name "*.log" -delete - -clean-data: - @echo "Cleaning processed data files..." - rm -rf data/processed/* - rm -rf results/*.csv - rm -rf results/*.pkl - -clean-images: - @echo "Cleaning generated images..." - rm -rf images/*.png - rm -rf images/*.jpg - rm -rf images/*.svg - -# Development workflow -dev-setup: setup-env install-dev - @echo "Development environment ready!" - -dev-check: format lint test - @echo "Development checks completed!" - -# CI/CD simulation -ci: format-check lint test security - @echo "CI checks completed!" - -# Complete workflow -all: format lint test run-analysis generate-charts - @echo "Complete workflow finished!" - -# Git helpers -git-setup: - @echo "Setting up git hooks..." - pre-commit install - -commit-check: format lint test - @echo "Ready for commit!" - -# Project information -info: - @echo "Bee Migration Analysis Project" - @echo "==============================" - @echo "Python version: $(shell python --version)" - @echo "Project structure:" - @tree -L 2 -I '__pycache__|.git|.venv|*.pyc' - -status: - @echo "Project status:" - @echo "- Source files: $(shell find src -name '*.py' | wc -l)" - @echo "- Test files: $(shell find tests -name '*.py' 2>/dev/null | wc -l || echo 0)" - @echo "- Data files: $(shell find data -name '*.csv' 2>/dev/null | wc -l || echo 0)" - @echo "- Generated images: $(shell find images -name '*.png' 2>/dev/null | wc -l || echo 0)" - @echo "- Model files: $(shell find results/models -name '*.pkl' 2>/dev/null | wc -l || echo 0)" - -# Docker support (future) docker-build: - @echo "Building Docker image..." - @echo "Note: Add Dockerfile and build command" + docker compose build docker-run: - @echo "Running Docker container..." - @echo "Note: Add Docker run command" - -# Jupyter notebook support -jupyter: - @echo "Starting Jupyter Lab..." - jupyter lab --ip=0.0.0.0 --port=8888 --no-browser + docker compose run --rm analysis all -notebook-clean: - @echo "Cleaning notebook outputs..." - jupyter nbconvert --clear-output --inplace notebooks/*.ipynb \ No newline at end of file +clean: + find . -type d -name "__pycache__" -not -path "./.venv/*" -prune -exec rm -rf {} + + rm -rf .pytest_cache .ruff_cache .coverage htmlcov build docs/package src/*.egg-info diff --git a/docker-compose.yml b/docker-compose.yml index af53864..38685af 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,283 +1,16 @@ -version: '3.8' - -# Docker Compose configuration for Bee Migration Analysis Project -# This file defines multiple services for different use cases - services: - # Main application service - app: - build: - context: . - target: production - container_name: bee-migration-app - volumes: - - ./data:/app/data:ro - - ./results:/app/results - - ./images:/app/images - - ./logs:/app/logs - environment: - - PYTHONPATH=/app/src - - LOG_LEVEL=INFO - networks: - - bee-network - restart: unless-stopped - healthcheck: - test: ["CMD", "python", "-c", "import sys; sys.exit(0)"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 40s - - # Development service - dev: - build: - context: . - target: development - container_name: bee-migration-dev - volumes: - - .:/app - - /app/.git - - dev-cache:/app/.pytest_cache - - dev-cache:/app/.mypy_cache - environment: - - PYTHONPATH=/app/src - - LOG_LEVEL=DEBUG - networks: - - bee-network - stdin_open: true - tty: true - command: bash - - # Jupyter notebook service - jupyter: - build: - context: . - target: jupyter - container_name: bee-migration-jupyter - ports: - - "8888:8888" - volumes: - - .:/app - - jupyter-data:/home/beeuser/.jupyter - environment: - - PYTHONPATH=/app/src - - JUPYTER_ENABLE_LAB=yes - networks: - - bee-network - restart: unless-stopped - labels: - - "traefik.enable=true" - - "traefik.http.routers.jupyter.rule=Host(`jupyter.localhost`)" - - "traefik.http.services.jupyter.loadbalancer.server.port=8888" - - # API service - api: - build: - context: . - target: api - container_name: bee-migration-api - ports: - - "8000:8000" - volumes: - - ./data:/app/data:ro - - ./results:/app/results - - ./images:/app/images - - ./logs:/app/logs - environment: - - PYTHONPATH=/app/src - - API_HOST=0.0.0.0 - - API_PORT=8000 - - LOG_LEVEL=INFO - networks: - - bee-network - restart: unless-stopped - depends_on: - - redis - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8000/health"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 40s - labels: - - "traefik.enable=true" - - "traefik.http.routers.api.rule=Host(`api.localhost`)" - - "traefik.http.services.api.loadbalancer.server.port=8000" - - # Dashboard service - dashboard: - build: - context: . - target: dashboard - container_name: bee-migration-dashboard - ports: - - "8501:8501" - volumes: - - ./data:/app/data:ro - - ./results:/app/results:ro - - ./images:/app/images:ro - environment: - - PYTHONPATH=/app/src - - STREAMLIT_SERVER_ADDRESS=0.0.0.0 - - STREAMLIT_SERVER_PORT=8501 - networks: - - bee-network - restart: unless-stopped - depends_on: - - api - labels: - - "traefik.enable=true" - - "traefik.http.routers.dashboard.rule=Host(`dashboard.localhost`)" - - "traefik.http.services.dashboard.loadbalancer.server.port=8501" - - # Redis for caching and session management - redis: - image: redis:7-alpine - container_name: bee-migration-redis - ports: - - "6379:6379" + # docker compose run --rm analysis all + analysis: + build: . volumes: - - redis-data:/data - networks: - - bee-network - restart: unless-stopped - command: redis-server --appendonly yes - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 30s - timeout: 10s - retries: 3 + - ./data:/work/data + - ./results:/work/results + - ./images:/work/images - # PostgreSQL database (for future use) - postgres: - image: postgres:15-alpine - container_name: bee-migration-postgres + # docker compose up site -> http://127.0.0.1:8000 + site: + build: . + entrypoint: ["python", "-m", "http.server", "8000", "--directory", "/app/docs"] + command: [] ports: - - "5432:5432" - environment: - - POSTGRES_DB=bee_migration - - POSTGRES_USER=beeuser - - POSTGRES_PASSWORD=beepassword - volumes: - - postgres-data:/var/lib/postgresql/data - - ./sql:/docker-entrypoint-initdb.d - networks: - - bee-network - restart: unless-stopped - healthcheck: - test: ["CMD-SHELL", "pg_isready -U beeuser -d bee_migration"] - interval: 30s - timeout: 10s - retries: 3 - - # Testing service - test: - build: - context: . - target: testing - container_name: bee-migration-test - volumes: - - .:/app - - test-cache:/app/.pytest_cache - - test-coverage:/app/htmlcov - environment: - - PYTHONPATH=/app/src - - PYTEST_CURRENT_TEST=1 - networks: - - bee-network - profiles: - - testing - - # Monitoring with Prometheus (optional) - prometheus: - image: prom/prometheus:latest - container_name: bee-migration-prometheus - ports: - - "9090:9090" - volumes: - - ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml - - prometheus-data:/prometheus - networks: - - bee-network - restart: unless-stopped - profiles: - - monitoring - command: - - '--config.file=/etc/prometheus/prometheus.yml' - - '--storage.tsdb.path=/prometheus' - - '--web.console.libraries=/etc/prometheus/console_libraries' - - '--web.console.templates=/etc/prometheus/consoles' - - '--storage.tsdb.retention.time=200h' - - '--web.enable-lifecycle' - - # Grafana for visualization (optional) - grafana: - image: grafana/grafana:latest - container_name: bee-migration-grafana - ports: - - "3000:3000" - environment: - - GF_SECURITY_ADMIN_PASSWORD=admin - volumes: - - grafana-data:/var/lib/grafana - - ./monitoring/grafana:/etc/grafana/provisioning - networks: - - bee-network - restart: unless-stopped - profiles: - - monitoring - depends_on: - - prometheus - - # Reverse proxy with Traefik (optional) - traefik: - image: traefik:v3.0 - container_name: bee-migration-traefik - ports: - - "80:80" - - "8080:8080" - volumes: - - /var/run/docker.sock:/var/run/docker.sock:ro - - ./traefik:/etc/traefik - networks: - - bee-network - restart: unless-stopped - profiles: - - proxy - command: - - --api.insecure=true - - --providers.docker=true - - --providers.docker.exposedbydefault=false - - --entrypoints.web.address=:80 - -networks: - bee-network: - driver: bridge - ipam: - config: - - subnet: 172.20.0.0/16 - -volumes: - # Persistent data volumes - postgres-data: - driver: local - redis-data: - driver: local - prometheus-data: - driver: local - grafana-data: - driver: local - - # Development volumes - dev-cache: - driver: local - test-cache: - driver: local - test-coverage: - driver: local - jupyter-data: - driver: local - -# Development override -# Use: docker-compose -f docker-compose.yml -f docker-compose.override.yml up -# This file can be created for local development overrides \ No newline at end of file + - "127.0.0.1:8000:8000" diff --git a/environment.yml b/environment.yml index 94c5c34..fb091c1 100644 --- a/environment.yml +++ b/environment.yml @@ -1,75 +1,9 @@ +# conda env create -f environment.yml && conda activate bee-migration-analysis name: bee-migration-analysis channels: - conda-forge - - defaults dependencies: - # Python version - - python=3.9 - - # Core data science libraries - - numpy>=1.21.0 - - pandas>=1.5.0 - - scipy>=1.9.0 - - # Machine learning - - scikit-learn>=1.2.0 - - joblib>=1.2.0 - - # Visualization - - matplotlib>=3.6.0 - - seaborn>=0.12.0 - - plotly>=5.12.0 - - # Geospatial analysis - - geopandas>=0.12.0 - - folium>=0.14.0 - - cartopy>=0.21.0 - - shapely>=2.0.0 - - fiona>=1.8.0 - - pyproj>=3.4.0 - - # Data APIs and web requests - - requests>=2.28.0 - - urllib3>=1.26.0 - - # File I/O and data formats - - openpyxl>=3.0.0 - - xlsxwriter>=3.0.0 - - h5py>=3.7.0 - - # Development tools - - jupyter>=1.0.0 - - jupyterlab>=3.6.0 - - ipykernel>=6.20.0 - - # Testing (optional) - - pytest>=7.0.0 - - pytest-cov>=4.0.0 - - # Code quality (optional) - - black>=23.0.0 - - isort>=5.12.0 - - flake8>=6.0.0 - - # Additional dependencies via pip + - python=3.13 - pip - pip: - # Specific packages not available in conda-forge - - pygbif>=0.6.0 # GBIF API client - - rasterio>=1.3.0 # Raster data processing - - xarray>=2023.1.0 # Multi-dimensional arrays - - netcdf4>=1.6.0 # NetCDF file support - - dask>=2023.1.0 # Parallel computing - - memory-profiler>=0.60.0 # Memory profiling - - # Future dashboard dependencies - - streamlit>=1.20.0 - - dash>=2.10.0 - - # API development - - fastapi>=0.95.0 - - uvicorn>=0.20.0 - - pydantic>=1.10.0 - - # Environment management - - python-dotenv>=1.0.0 \ No newline at end of file + - -e .[dev,notebook] diff --git a/pyproject.toml b/pyproject.toml index 2e93e21..65aded8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,8 @@ dependencies = [ ] [project.optional-dependencies] -dev = ["pytest>=8.0", "pytest-cov>=5.0", "ruff>=0.6"] +dev = ["pytest>=8.0", "pytest-cov>=5.0", "ruff>=0.6", "pre-commit>=4.0"] +notebook = ["jupyterlab>=4.0"] [project.urls] Homepage = "https://github.com/digomes87/Bees" diff --git a/requirements-dev.txt b/requirements-dev.txt index a6de4e2..41897b0 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,3 +3,4 @@ pytest>=8.0 pytest-cov>=5.0 ruff>=0.6 +pre-commit>=4.0 diff --git a/results/bee_migration_comprehensive_report.txt b/results/bee_migration_comprehensive_report.txt index 9a5875a..28ed866 100644 --- a/results/bee_migration_comprehensive_report.txt +++ b/results/bee_migration_comprehensive_report.txt @@ -5,8 +5,8 @@ Data de Geração: 22/08/2025 16:11:50 ## RESUMO EXECUTIVO -Este relatório analisa evidências científicas sobre a migração de abelhas para o sul -devido às mudanças climáticas, baseado em estudos recentes da UFPR, dataset Dryad +Este relatório analisa evidências científicas sobre a migração de abelhas para o sul +devido às mudanças climáticas, baseado em estudos recentes da UFPR, dataset Dryad e pesquisas internacionais. ## 1. ESTUDO UFPR - ABELHAS SUL-AMERICANAS diff --git a/results/integrated_migration_report.txt b/results/integrated_migration_report.txt index bfc2952..a756ca9 100644 --- a/results/integrated_migration_report.txt +++ b/results/integrated_migration_report.txt @@ -5,8 +5,8 @@ Data de Geração: 22/08/2025 16:14:30 ## RESUMO EXECUTIVO -Este relatório integra dados locais do projeto com evidências científicas de múltiplos -estudos sobre migração de abelhas devido às mudanças climáticas, fornecendo uma +Este relatório integra dados locais do projeto com evidências científicas de múltiplos +estudos sobre migração de abelhas devido às mudanças climáticas, fornecendo uma visão abrangente e comparativa dos padrões observados. ## 1. INTEGRAÇÃO DE DADOS @@ -34,7 +34,7 @@ visão abrangente e comparativa dos padrões observados. - **Consenso**: 85-90% das espécies migram para o sul #### 2.2 Fatores Climáticos: -- **Aumento de Temperatura**: +- **Aumento de Temperatura**: - UFPR: 2.5°C projetado - Dados Locais: 2.1-3.2°C observado/projetado - Consenso: 2-3°C de aumento crítico @@ -151,7 +151,7 @@ visão abrangente e comparativa dos padrões observados. --- **Metodologia de Integração:** -Este relatório combina análise quantitativa de dados locais com revisão sistemática +Este relatório combina análise quantitativa de dados locais com revisão sistemática de literatura científica, utilizando métodos de síntese de evidências e validação cruzada. **Fontes Integradas:** diff --git a/results/scientific_papers_synthesis.md b/results/scientific_papers_synthesis.md index e52314a..51e0e9a 100644 --- a/results/scientific_papers_synthesis.md +++ b/results/scientific_papers_synthesis.md @@ -2,7 +2,7 @@ ## Análise Integrada dos Papers Científicos Fornecidos -**Data:** 22 de agosto de 2025 +**Data:** 22 de agosto de 2025 **Análise baseada em:** 6 fontes científicas principais + dados locais --- @@ -257,24 +257,24 @@ ## 📋 CONCLUSÕES FINAIS ### **CERTEZAS CIENTÍFICAS:** -✅ **Migração para o sul é real e acelerada** -✅ **Bombus bellicosus é indicador validado** -✅ **Campos sulinos são área crítica de conservação** -✅ **Grupos funcionais respondem diferentemente** -✅ **Corredores ecológicos são essenciais** +✅ **Migração para o sul é real e acelerada** +✅ **Bombus bellicosus é indicador validado** +✅ **Campos sulinos são área crítica de conservação** +✅ **Grupos funcionais respondem diferentemente** +✅ **Corredores ecológicos são essenciais** ### **INCERTEZAS PRINCIPAIS:** -❓ **Velocidade exata da migração** -❓ **Capacidade de adaptação local** -❓ **Interações com outros fatores** -❓ **Eficácia de medidas de conservação** -❓ **Impactos socioeconômicos precisos** +❓ **Velocidade exata da migração** +❓ **Capacidade de adaptação local** +❓ **Interações com outros fatores** +❓ **Eficácia de medidas de conservação** +❓ **Impactos socioeconômicos precisos** ### **URGÊNCIA DA AÇÃO:** -🚨 **CRÍTICA** - Janela de oportunidade de 5-10 anos -🕐 **TEMPO LIMITADO** - Mudanças já em curso -💰 **CUSTO-EFETIVO** - Prevenção vs. remediação -🤝 **COOPERAÇÃO** - Necessária entre países +🚨 **CRÍTICA** - Janela de oportunidade de 5-10 anos +🕐 **TEMPO LIMITADO** - Mudanças já em curso +💰 **CUSTO-EFETIVO** - Prevenção vs. remediação +🤝 **COOPERAÇÃO** - Necessária entre países --- @@ -283,19 +283,19 @@ 1. **UFPR** - Mudanças climáticas empurram abelhas sul-americanas para o sul 2. **Dryad Dataset** - Climate-induced range shifts in neotropical bees 3. **PMC 6856679** - Global warming promotes biological invasion of honey bee pest -4. **Ecology Letters** - Warming temperatures restructuring bumble bee communities +4. **Ecology Letters** - Warming temperatures restructuring bumble bee communities 5. **PMC 9030098** - Climate change impacts on island bees (Aegean) 6. **Kaggle** - Climate change datasets (validação) --- -**Análise realizada por:** Sistema de Análise Comparativa -**Metodologia:** Síntese de evidências + validação cruzada -**Confiabilidade:** Alta (convergência de múltiplas fontes) +**Análise realizada por:** Sistema de Análise Comparativa +**Metodologia:** Síntese de evidências + validação cruzada +**Confiabilidade:** Alta (convergência de múltiplas fontes) **Próxima atualização:** Conforme novos dados disponíveis --- *"A convergência de evidências científicas de múltiplas fontes e regiões confirma inequivocamente que as abelhas estão migrando para o sul devido às mudanças climáticas. A janela de oportunidade para ação conservacionista é limitada e crítica."* -**- Síntese Científica Integrada, Agosto 2025** \ No newline at end of file +**- Síntese Científica Integrada, Agosto 2025** From a7cf2cafbe9ba182f26ce01627a9b165db53bab6 Mon Sep 17 00:00:00 2001 From: Diego Go Date: Sun, 11 Oct 2026 09:30:51 +0200 Subject: [PATCH 3/4] docs: add the MIT license file pyproject.toml already declared MIT, but the repository had no license text for it to refer to. --- LICENSE | 21 +++++++++++++++++++++ README.md | 5 +++++ pyproject.toml | 1 + 3 files changed, 27 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..6a0f031 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Diego Gomes + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index c74b5ed..6a9e68a 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ [![CI](https://github.com/digomes87/Bees/actions/workflows/ci.yml/badge.svg)](https://github.com/digomes87/Bees/actions/workflows/ci.yml) [![Python](https://img.shields.io/badge/Python-3.11+-blue.svg)](https://www.python.org/downloads/) +[![License](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE) [![Dashboard](https://img.shields.io/badge/Dashboard-live-b8740a.svg)](https://digomes87.github.io/Bees/) A Python package and a browser dashboard for studying how climate relates to @@ -226,6 +227,10 @@ Data: [GBIF](https://www.gbif.org/) and [Open-Meteo](https://open-meteo.com/) (ERA5 reanalysis from Copernicus/ECMWF). Studies: UFPR, Dryad, Ecology Letters and PMC, as listed above. +## License + +MIT. See [LICENSE](LICENSE). + ## Author Diego Gomes — [@digomes87](https://github.com/digomes87). diff --git a/pyproject.toml b/pyproject.toml index 65aded8..44bd174 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,6 +10,7 @@ description = "Climate impact assessment on bee migration patterns in South Amer readme = "README.md" requires-python = ">=3.11" license = { text = "MIT" } +license-files = ["LICENSE"] keywords = ["bees", "migration", "climate", "analysis", "machine-learning"] dependencies = [ "numpy>=1.26", From 44fc4133233e07389adf91fa9f3089bd788c72cc Mon Sep 17 00:00:00 2001 From: Diego Go Date: Sun, 11 Oct 2026 09:31:22 +0200 Subject: [PATCH 4/4] build: declare the license as an SPDX expression setuptools rejects license-files next to the legacy table form of the license field, which broke the package build. --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 44bd174..1935c88 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=68"] +requires = ["setuptools>=77"] build-backend = "setuptools.build_meta" [project] @@ -9,7 +9,7 @@ authors = [{ name = "Diego Gomes", email = "diego@example.com" }] description = "Climate impact assessment on bee migration patterns in South America" readme = "README.md" requires-python = ">=3.11" -license = { text = "MIT" } +license = "MIT" license-files = ["LICENSE"] keywords = ["bees", "migration", "climate", "analysis", "machine-learning"] dependencies = [