From 57e254b8f22dc2cb9e0f4626d6f8e588e4e5082b Mon Sep 17 00:00:00 2001 From: the-icemann Date: Thu, 14 May 2026 23:53:35 +0300 Subject: [PATCH 01/24] Employed the next phase of the ML service to try to pull from the respective service db's instead of a seed --- Makefile | 202 +++++++-- NGINX_502_FIX.md | 73 +++ PHASE2_ANALYSIS.md | 424 ++++++++++++++++++ docker-compose.yml | 3 + execution.md | 237 ++++++++++ services/order/app/kafka_publisher.py | 2 + services/order/app/routers/orders.py | 2 + services/soko-ml/.env.example | 60 ++- .../soko-ml/data-ingestion-service/Dockerfile | 12 + .../data-ingestion-service/requirements.txt | 11 + .../data-ingestion-service/src/__init__.py | 0 .../src/bootstrap/__init__.py | 0 .../src/bootstrap/auth_bootstrap.py | 46 ++ .../src/bootstrap/listing_bootstrap.py | 69 +++ .../src/bootstrap/market_bootstrap.py | 12 + .../src/bootstrap/order_bootstrap.py | 36 ++ .../src/clients/__init__.py | 0 .../src/clients/listing_client.py | 49 ++ .../src/clients/order_client.py | 57 +++ .../src/clients/user_client.py | 75 ++++ .../src/feature_store.py | 336 ++++++++++++++ .../data-ingestion-service/src/health.py | 51 +++ .../data-ingestion-service/src/main.py | 157 +++++++ .../data-ingestion-service/src/schemas.py | 97 ++++ .../src/streams/__init__.py | 0 .../src/streams/transaction_stream.py | 143 ++++++ .../src/transformers/__init__.py | 0 .../src/transformers/buyer_transformer.py | 79 ++++ .../src/transformers/farmer_transformer.py | 180 ++++++++ .../src/transformers/price_transformer.py | 135 ++++++ .../data-ingestion-service/tests/__init__.py | 0 .../tests/test_feature_store.py | 208 +++++++++ .../tests/test_transformers.py | 252 +++++++++++ services/soko-ml/db/schema.sql | 145 ++++++ services/soko-ml/docker-compose.yml | 127 ++++++ services/soko-ml/install_cmdstan.py | 24 + services/soko-ml/kafka-agent/src/agent.py | 4 + .../src/consumers/coverage_gap_consumer.py | 79 ++++ .../consumers/transaction_price_collector.py | 90 ++++ services/soko-ml/location-service/Dockerfile | 12 + .../soko-ml/location-service/requirements.txt | 10 + .../soko-ml/location-service/src/__init__.py | 0 .../soko-ml/location-service/src/cache.py | 140 ++++++ .../soko-ml/location-service/src/fallback.py | 156 +++++++ .../location-service/src/gap_notifier.py | 133 ++++++ .../location-service/src/geo_recommender.py | 119 +++++ .../src/google_maps_client.py | 89 ++++ services/soko-ml/location-service/src/main.py | 118 +++++ .../location-service/src/market_router.py | 202 +++++++++ .../soko-ml/location-service/src/schemas.py | 64 +++ .../location-service/src/sell_signal.py | 85 ++++ .../location-service/src/transport_cost.py | 51 +++ .../location-service/tests/__init__.py | 0 .../tests/test_market_router.py | 182 ++++++++ .../soko-ml/ml-gateway-service/src/main.py | 70 ++- .../soko-ml/ml-gateway-service/src/proxy.py | 40 +- .../src/feature_store_client.py | 77 ++++ .../price-prediction-service/src/predictor.py | 63 ++- .../recommendation-service/requirements.txt | 1 + .../src/feature_store_client.py | 122 +++++ .../recommendation-service/src/geo_filter.py | 86 ++++ .../recommendation-service/src/main.py | 43 +- .../recommendation-service/src/recommender.py | 46 +- services/soko-ml/shared/events.py | 91 ++++ services/user/app/routers/profile.py | 23 + 65 files changed, 5398 insertions(+), 102 deletions(-) create mode 100644 NGINX_502_FIX.md create mode 100644 PHASE2_ANALYSIS.md create mode 100644 execution.md create mode 100644 services/soko-ml/data-ingestion-service/Dockerfile create mode 100644 services/soko-ml/data-ingestion-service/requirements.txt create mode 100644 services/soko-ml/data-ingestion-service/src/__init__.py create mode 100644 services/soko-ml/data-ingestion-service/src/bootstrap/__init__.py create mode 100644 services/soko-ml/data-ingestion-service/src/bootstrap/auth_bootstrap.py create mode 100644 services/soko-ml/data-ingestion-service/src/bootstrap/listing_bootstrap.py create mode 100644 services/soko-ml/data-ingestion-service/src/bootstrap/market_bootstrap.py create mode 100644 services/soko-ml/data-ingestion-service/src/bootstrap/order_bootstrap.py create mode 100644 services/soko-ml/data-ingestion-service/src/clients/__init__.py create mode 100644 services/soko-ml/data-ingestion-service/src/clients/listing_client.py create mode 100644 services/soko-ml/data-ingestion-service/src/clients/order_client.py create mode 100644 services/soko-ml/data-ingestion-service/src/clients/user_client.py create mode 100644 services/soko-ml/data-ingestion-service/src/feature_store.py create mode 100644 services/soko-ml/data-ingestion-service/src/health.py create mode 100644 services/soko-ml/data-ingestion-service/src/main.py create mode 100644 services/soko-ml/data-ingestion-service/src/schemas.py create mode 100644 services/soko-ml/data-ingestion-service/src/streams/__init__.py create mode 100644 services/soko-ml/data-ingestion-service/src/streams/transaction_stream.py create mode 100644 services/soko-ml/data-ingestion-service/src/transformers/__init__.py create mode 100644 services/soko-ml/data-ingestion-service/src/transformers/buyer_transformer.py create mode 100644 services/soko-ml/data-ingestion-service/src/transformers/farmer_transformer.py create mode 100644 services/soko-ml/data-ingestion-service/src/transformers/price_transformer.py create mode 100644 services/soko-ml/data-ingestion-service/tests/__init__.py create mode 100644 services/soko-ml/data-ingestion-service/tests/test_feature_store.py create mode 100644 services/soko-ml/data-ingestion-service/tests/test_transformers.py create mode 100644 services/soko-ml/db/schema.sql create mode 100644 services/soko-ml/install_cmdstan.py create mode 100644 services/soko-ml/kafka-agent/src/consumers/coverage_gap_consumer.py create mode 100644 services/soko-ml/kafka-agent/src/consumers/transaction_price_collector.py create mode 100644 services/soko-ml/location-service/Dockerfile create mode 100644 services/soko-ml/location-service/requirements.txt create mode 100644 services/soko-ml/location-service/src/__init__.py create mode 100644 services/soko-ml/location-service/src/cache.py create mode 100644 services/soko-ml/location-service/src/fallback.py create mode 100644 services/soko-ml/location-service/src/gap_notifier.py create mode 100644 services/soko-ml/location-service/src/geo_recommender.py create mode 100644 services/soko-ml/location-service/src/google_maps_client.py create mode 100644 services/soko-ml/location-service/src/main.py create mode 100644 services/soko-ml/location-service/src/market_router.py create mode 100644 services/soko-ml/location-service/src/schemas.py create mode 100644 services/soko-ml/location-service/src/sell_signal.py create mode 100644 services/soko-ml/location-service/src/transport_cost.py create mode 100644 services/soko-ml/location-service/tests/__init__.py create mode 100644 services/soko-ml/location-service/tests/test_market_router.py create mode 100644 services/soko-ml/price-prediction-service/src/feature_store_client.py create mode 100644 services/soko-ml/recommendation-service/src/feature_store_client.py create mode 100644 services/soko-ml/recommendation-service/src/geo_filter.py diff --git a/Makefile b/Makefile index b615206..d59fc38 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,8 @@ REC_VENV := $(ML_DIR)/recommendation-service/.venv GATEWAY_VENV := $(ML_DIR)/ml-gateway-service/.venv AGENT_VENV := $(ML_DIR)/kafka-agent/.venv DATA_VENV := $(ML_DIR)/data-generator/.venv +INGEST_VENV := $(ML_DIR)/data-ingestion-service/.venv +LOC_VENV := $(ML_DIR)/location-service/.venv # ── Core services that need .env files ─────────────────────────────────────── CORE_SERVICES := auth user produce order payment message notification blog ussd @@ -34,12 +36,14 @@ CORE_SERVICES := auth user produce order payment message notification blog ussd bridge-network \ ml-up ml-down ml-logs \ core-up core-down core-logs core-restart \ - install generate-data train \ - dev dev-price dev-rec dev-gateway \ + install generate-data train cold-start \ + dev dev-price dev-rec dev-gateway dev-location dev-ingest \ + db-up db-shell db-reset \ infra-up infra-down kafka-topics kafka-ui redis-cli \ - logs logs-price logs-rec logs-gateway logs-agent \ - test test-price test-rec test-gateway \ - health smoke-test \ + ingest-bootstrap ingest-status gaps-summary gaps-reset \ + logs logs-price logs-rec logs-gateway logs-agent logs-location logs-ingest \ + test test-price test-rec test-gateway test-location test-ingest \ + health smoke-test smoke-route smoke-discover smoke-fallback smoke-tier3 smoke-ingest \ clean clean-models clean-docker \ help @@ -74,11 +78,27 @@ help: @echo "► ML STACK" @echo " ─────────────────────────────────────────────────────────" @echo " make ml-up Build and start the ML stack" - @echo " ↳ ML gateway → http://localhost:8080" - @echo " ↳ Price service → http://localhost:8094/docs" - @echo " ↳ Rec service → http://localhost:8095/docs" + @echo " ↳ ML gateway → http://localhost:8080" + @echo " ↳ Price service → http://localhost:8094/docs" + @echo " ↳ Rec service → http://localhost:8095/docs" + @echo " ↳ Location service → http://localhost:8003/docs" + @echo " ↳ Ingest service → http://localhost:8096/docs" @echo " make ml-down Stop and remove ML containers + volumes" @echo " make ml-logs Tail logs for all ML containers" + @echo " make cold-start First-time: bring up infra, bootstrap DB, then full ML stack" + @echo "" + @echo "► ML — DATABASE (Feature Store)" + @echo " ─────────────────────────────────────────────────────────" + @echo " make db-up Start only the ML Postgres (soko-ml-db) + run schema" + @echo " make db-shell Open psql shell into soko_ml_db" + @echo " make db-reset Drop and re-create schema (destructive!)" + @echo "" + @echo "► ML — DATA INGESTION" + @echo " ─────────────────────────────────────────────────────────" + @echo " make ingest-bootstrap Trigger initial data sync from backend services" + @echo " make ingest-status Show bootstrap progress" + @echo " make gaps-summary Show crop/market coverage gap report" + @echo " make gaps-reset Reset all gap counters (dev only)" @echo "" @echo "► CORE STACK" @echo " ─────────────────────────────────────────────────────────" @@ -90,13 +110,15 @@ help: @echo "" @echo "► ML — LOCAL DEVELOPMENT" @echo " ─────────────────────────────────────────────────────────" - @echo " make install Create Python venvs and install ML dependencies" + @echo " make install Create Python venvs and install all ML dependencies" @echo " make generate-data Generate synthetic training data (farmers/buyers/prices)" @echo " make train Train price-prediction models locally" @echo " make dev Run ML stack with hot-reload (docker compose dev override)" @echo " make dev-price Run price service locally with uvicorn on :8094" @echo " make dev-rec Run recommendation service locally with uvicorn on :8095" @echo " make dev-gateway Run ML gateway locally with uvicorn on :8080" + @echo " make dev-location Run location service locally with uvicorn on :8003" + @echo " make dev-ingest Run data-ingestion service locally with uvicorn on :8096" @echo "" @echo "► ML — INFRASTRUCTURE" @echo " ─────────────────────────────────────────────────────────" @@ -113,6 +135,8 @@ help: @echo " make logs-rec Tail recommendation-service logs" @echo " make logs-gateway Tail ml-gateway-service logs" @echo " make logs-agent Tail kafka-agent logs" + @echo " make logs-location Tail location-service logs" + @echo " make logs-ingest Tail data-ingestion-service logs" @echo "" @echo "► TESTING" @echo " ─────────────────────────────────────────────────────────" @@ -120,11 +144,18 @@ help: @echo " make test-price Run price-prediction-service tests only" @echo " make test-rec Run recommendation-service tests only" @echo " make test-gateway Run ml-gateway-service tests only" + @echo " make test-location Run location-service tests only" + @echo " make test-ingest Run data-ingestion-service tests only" @echo "" @echo "► HEALTH & SMOKE" @echo " ─────────────────────────────────────────────────────────" - @echo " make health Hit /health on API gateway + all ML services" - @echo " make smoke-test End-to-end: price prediction + recommendation calls" + @echo " make health Hit /health on API gateway + all ML services" + @echo " make smoke-test Price prediction + recommendation calls" + @echo " make smoke-route Location /route with a sample farmer payload" + @echo " make smoke-discover Location /discover buyer→farmers query" + @echo " make smoke-fallback Location /route Tier 2 fallback (rare crop)" + @echo " make smoke-tier3 Location /route Tier 3 unknown crop" + @echo " make smoke-ingest POST a synthetic order event to /ingest/order-event" @echo "" @echo "► CLEAN" @echo " ─────────────────────────────────────────────────────────" @@ -266,22 +297,16 @@ core-logs: # ============================================================================= install: - python3.12 -m venv $(PRICE_VENV) && $(PRICE_VENV)/bin/pip install -q -r $(ML_DIR)/price-prediction-service/requirements.txt - python3.12 -m venv $(REC_VENV) && $(REC_VENV)/bin/pip install -q -r $(ML_DIR)/recommendation-service/requirements.txt - python3.12 -m venv $(GATEWAY_VENV) && $(GATEWAY_VENV)/bin/pip install -q -r $(ML_DIR)/ml-gateway-service/requirements.txt - python3.12 -m venv $(AGENT_VENV) && $(AGENT_VENV)/bin/pip install -q -r $(ML_DIR)/kafka-agent/requirements.txt - python3.12 -m venv $(DATA_VENV) && $(DATA_VENV)/bin/pip install -q -r $(ML_DIR)/data-generator/requirements.txt + python3.12 -m venv $(PRICE_VENV) && $(PRICE_VENV)/bin/pip install -q --timeout 120 -r $(ML_DIR)/price-prediction-service/requirements.txt + python3.12 -m venv $(REC_VENV) && $(REC_VENV)/bin/pip install -q --timeout 120 -r $(ML_DIR)/recommendation-service/requirements.txt + python3.12 -m venv $(GATEWAY_VENV) && $(GATEWAY_VENV)/bin/pip install -q --timeout 120 -r $(ML_DIR)/ml-gateway-service/requirements.txt + python3.12 -m venv $(AGENT_VENV) && $(AGENT_VENV)/bin/pip install -q --timeout 120 -r $(ML_DIR)/kafka-agent/requirements.txt + python3.12 -m venv $(DATA_VENV) && $(DATA_VENV)/bin/pip install -q --timeout 120 -r $(ML_DIR)/data-generator/requirements.txt + python3.12 -m venv $(INGEST_VENV) && $(INGEST_VENV)/bin/pip install -q --timeout 120 -r $(ML_DIR)/data-ingestion-service/requirements.txt + python3.12 -m venv $(LOC_VENV) && $(LOC_VENV)/bin/pip install -q --timeout 120 -r $(ML_DIR)/location-service/requirements.txt @echo "All ML dependencies installed." @echo "Installing CmdStan 2.33.1 into Prophet's internal path (one-time, ~400 MB)..." - $(PRICE_VENV)/bin/python -c " \ - import prophet, pathlib, cmdstanpy; \ - d = pathlib.Path(prophet.__file__).parent / 'stan_model'; \ - target = d / 'cmdstan-2.33.1'; \ - d.mkdir(parents=True, exist_ok=True); \ - (print('CmdStan 2.33.1 already present, skipping.') if (target / 'Makefile').exists() \ - else (print('Downloading + compiling CmdStan 2.33.1...'), \ - cmdstanpy.install_cmdstan(dir=str(d), version='2.33.1'), \ - print('CmdStan 2.33.1 installed.')))" + $(PRICE_VENV)/bin/python $(ML_DIR)/install_cmdstan.py generate-data: @mkdir -p $(ML_DIR)/recommendation-service/data/raw @@ -319,17 +344,81 @@ dev-gateway: cd $(ML_DIR)/ml-gateway-service && \ $(abspath $(GATEWAY_VENV))/bin/uvicorn src.main:app --host 0.0.0.0 --port 8080 --reload +dev-location: + cd $(ML_DIR)/location-service && \ + $(abspath $(LOC_VENV))/bin/uvicorn src.main:app --host 0.0.0.0 --port 8003 --reload + +dev-ingest: + cd $(ML_DIR)/data-ingestion-service && \ + $(abspath $(INGEST_VENV))/bin/uvicorn src.main:app --host 0.0.0.0 --port 8096 --reload + +# ============================================================================= +# ML — DATABASE (Feature Store) +# ============================================================================= + +db-up: + $(COMPOSE_ML) up -d soko-ml-db db-init + @echo "Waiting for schema init to complete..." + @sleep 5 + @$(COMPOSE_ML) logs db-init + +db-shell: + $(COMPOSE_ML) exec soko-ml-db psql -U $${POSTGRES_USER:-soko_ml} -d soko_ml_db + +db-reset: + @echo "WARNING: This will drop and re-apply the schema. Press Ctrl-C to abort (5s)..." + @sleep 5 + $(COMPOSE_ML) exec -T soko-ml-db psql -U $${POSTGRES_USER:-soko_ml} -d soko_ml_db \ + -c "DROP SCHEMA public CASCADE; CREATE SCHEMA public;" + $(COMPOSE_ML) exec -T soko-ml-db psql -U $${POSTGRES_USER:-soko_ml} -d soko_ml_db \ + -f /schema/schema.sql + @echo "Schema reset complete." + +# ============================================================================= +# ML — DATA INGESTION +# ============================================================================= + +ingest-bootstrap: + @curl -sf -X POST http://localhost:8096/bootstrap | python3 -m json.tool || \ + curl -sf -X POST http://localhost:8080/ingest/bootstrap | python3 -m json.tool + +ingest-status: + @curl -sf http://localhost:8096/bootstrap/status | python3 -m json.tool || \ + curl -sf http://localhost:8080/ingest/status | python3 -m json.tool + +gaps-summary: + @curl -sf http://localhost:8096/gaps/summary | python3 -m json.tool || \ + curl -sf http://localhost:8080/gaps/summary | python3 -m json.tool + +gaps-reset: + @$(COMPOSE_ML) exec -T soko-ml-db psql -U $${POSTGRES_USER:-soko_ml} -d soko_ml_db \ + -c "TRUNCATE coverage_gaps;" + @echo "Gap counters reset." + +cold-start: bridge-network db-up + @echo "Waiting for DB to be fully ready..." + @sleep 10 + $(COMPOSE_ML) up --build -d + @echo "Stack up — triggering bootstrap..." + @sleep 20 + @$(MAKE) ingest-bootstrap + @echo "" + @echo "Cold start complete:" + @echo " ML gateway → http://localhost:8080" + @echo " Ingest → http://localhost:8096/docs" + @echo " Location → http://localhost:8003/docs" + # ============================================================================= # ML — INFRASTRUCTURE HELPERS # ============================================================================= infra-up: - $(COMPOSE_ML) up -d zookeeper kafka kafka-init redis + $(COMPOSE_ML) up -d zookeeper kafka kafka-init redis soko-ml-db db-init @echo "ML infrastructure starting (Kafka may take ~30s to be ready)." infra-down: - $(COMPOSE_ML) stop zookeeper kafka kafka-init redis - $(COMPOSE_ML) rm -f zookeeper kafka kafka-init redis + $(COMPOSE_ML) stop zookeeper kafka kafka-init redis soko-ml-db db-init + $(COMPOSE_ML) rm -f zookeeper kafka kafka-init redis soko-ml-db db-init kafka-topics: $(COMPOSE_ML) exec kafka kafka-topics --bootstrap-server localhost:9092 \ @@ -344,6 +433,8 @@ kafka-topics: --create --if-not-exists --topic soko.ml.events --partitions 2 --replication-factor 1 $(COMPOSE_ML) exec kafka kafka-topics --bootstrap-server localhost:9092 \ --create --if-not-exists --topic soko.dlq --partitions 2 --replication-factor 1 + $(COMPOSE_ML) exec kafka kafka-topics --bootstrap-server localhost:9092 \ + --create --if-not-exists --topic soko.gaps --partitions 2 --replication-factor 1 @echo "All Kafka topics created." kafka-ui: @@ -371,11 +462,17 @@ logs-gateway: logs-agent: $(COMPOSE_ML) logs -f kafka-agent +logs-location: + $(COMPOSE_ML) logs -f location-service + +logs-ingest: + $(COMPOSE_ML) logs -f data-ingestion-service + # ============================================================================= # TESTING # ============================================================================= -test: test-price test-rec test-gateway +test: test-price test-rec test-gateway test-location test-ingest test-price: $(PRICE_VENV)/bin/pytest $(ML_DIR)/price-prediction-service/tests/ -v @@ -386,6 +483,12 @@ test-rec: test-gateway: $(GATEWAY_VENV)/bin/pytest $(ML_DIR)/ml-gateway-service/tests/ -v +test-location: + $(LOC_VENV)/bin/pytest $(ML_DIR)/location-service/tests/ -v + +test-ingest: + $(INGEST_VENV)/bin/pytest $(ML_DIR)/data-ingestion-service/tests/ -v + # ============================================================================= # HEALTH & SMOKE # ============================================================================= @@ -399,6 +502,10 @@ health: curl -sf http://localhost:8094/health | python3 -m json.tool || echo "UNREACHABLE" @echo "=== Recommendation Service ===" && \ curl -sf http://localhost:8095/health | python3 -m json.tool || echo "UNREACHABLE" + @echo "=== Location Service ===" && \ + curl -sf http://localhost:8003/health | python3 -m json.tool || echo "UNREACHABLE" + @echo "=== Data Ingestion Service ===" && \ + curl -sf http://localhost:8096/health | python3 -m json.tool || echo "UNREACHABLE" smoke-test: @echo "=== Smoke: Price Prediction ===" @@ -411,6 +518,41 @@ smoke-test: @echo "=== Smoke: Buyers for Farmer ===" @curl -sf "http://localhost:8080/recommend/buyers-for-farmer/F0001?top_n=3" | python3 -m json.tool +smoke-route: + @echo "=== Smoke: Market Route (farmer sell signal) ===" + @curl -sf -X POST http://localhost:8080/location/route \ + -H 'Content-Type: application/json' \ + -d '{"farmer_id":"F0001","crop":"maize_grain","quantity_kg":500,"harvest_month":8}' \ + | python3 -m json.tool + +smoke-discover: + @echo "=== Smoke: Discover Farmers Near Buyer ===" + @curl -sf -X POST http://localhost:8080/location/discover \ + -H 'Content-Type: application/json' \ + -d '{"buyer_id":"B0001","crop":"maize_grain","max_distance_km":150,"max_price_ugx":2000,"top_n":5}' \ + | python3 -m json.tool + +smoke-fallback: + @echo "=== Smoke: Tier 2 fallback (sesame seed — limited coverage) ===" + @curl -sf -X POST http://localhost:8080/location/route \ + -H 'Content-Type: application/json' \ + -d '{"farmer_id":"F0001","crop":"sesame","quantity_kg":200,"harvest_month":10}' \ + | python3 -m json.tool + +smoke-tier3: + @echo "=== Smoke: Tier 3 unknown crop ===" + @curl -sf -X POST http://localhost:8080/location/route \ + -H 'Content-Type: application/json' \ + -d '{"farmer_id":"F0001","crop":"moringa","quantity_kg":50,"harvest_month":6}' \ + | python3 -m json.tool + +smoke-ingest: + @echo "=== Smoke: POST synthetic order event to ingest ===" + @curl -sf -X POST http://localhost:8096/ingest/order-event \ + -H 'Content-Type: application/json' \ + -d '{"event_type":"purchase_completed","order_id":"TEST-001","product_name":"Maize (Dry)","crop":"Grains","market":"Kampala","price_per_kg_ugx":1400,"quantity_kg":50,"total_ugx":70000,"farmer_id":"F0001","buyer_id":"B0001","timestamp":"2026-05-14T10:00:00Z"}' \ + | python3 -m json.tool + # ============================================================================= # CLEAN # ============================================================================= @@ -418,7 +560,7 @@ smoke-test: clean: find $(ML_DIR) -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true find $(ML_DIR) -name "*.pyc" -delete 2>/dev/null || true - rm -rf $(PRICE_VENV) $(REC_VENV) $(GATEWAY_VENV) $(AGENT_VENV) $(DATA_VENV) + rm -rf $(PRICE_VENV) $(REC_VENV) $(GATEWAY_VENV) $(AGENT_VENV) $(DATA_VENV) $(INGEST_VENV) $(LOC_VENV) rm -f $(ML_DIR)/recommendation-service/data/raw/*.csv @echo "Cleaned." diff --git a/NGINX_502_FIX.md b/NGINX_502_FIX.md new file mode 100644 index 0000000..4a9b1f3 --- /dev/null +++ b/NGINX_502_FIX.md @@ -0,0 +1,73 @@ +# nginx 502 Bad Gateway — Root Cause & Fix + +## What's happening + +When `make core-up` rebuilds and restarts backend containers (e.g. `user_service`, `produce_service`), those containers can come up with different Docker-assigned IPs than they had before. nginx resolves static `upstream` block hostnames **once at startup** and caches the IPs — it does not re-resolve them when upstream containers restart. + +Result: nginx sends `user_service` requests to produce_service's old IP and vice versa → `Connection refused (111)` → **502 Bad Gateway**. + +Confirmed from nginx error log: +``` +connect() failed (111) upstream: "http://172.20.0.12:8002/docs" ← /users/ route +connect() failed (111) upstream: "http://172.20.0.15:8003/docs" ← /listings/ route +``` +But docker network shows `produce_service=172.20.0.12` and `user_service=172.20.0.15` — the ports are crossed because nginx cached the pre-rebuild IPs. + +--- + +## Quick fix (immediate) + +Restart nginx so it re-resolves all upstream hostnames: + +```bash +docker restart api_gateway +``` + +Also add this to `make core-up` so it always reloads nginx after rebuilding services: + +```makefile +core-up: + $(COMPOSE_CORE) up --build -d + @docker restart api_gateway # re-resolve upstream IPs after container rebuilds + ... +``` + +--- + +## Proper fix (nginx config) + +Convert all static `upstream` blocks in `nginx/nginx.conf` to variable-based resolution, the same pattern already used for the ML gateway. Variable-based `proxy_pass` forces nginx to re-query the Docker DNS resolver (`127.0.0.11`) on every request. + +**Current (broken on rebuild):** +```nginx +upstream user_service { server user_service:8002; } + +location /users/ { + proxy_pass http://user_service/; +} +``` + +**Fixed:** +```nginx +# Remove the upstream block entirely, use a variable instead: + +location /users/ { + set $user_svc "user_service:8002"; + proxy_pass http://$user_svc/; +} +``` + +The `resolver 127.0.0.11 valid=30s;` directive already in `nginx.conf` handles the periodic re-resolution — no other changes needed. + +Apply the same pattern to every service: `auth_service`, `user_service`, `produce_service`, `order_service`, `payment_service`, `message_service`, `notification_service`, `blog_service`, `ussd_service`. + +--- + +## Why this doesn't affect the ML gateway + +The ML gateway already uses variable-based resolution: +```nginx +set $ml_gw "ml-gateway:8000"; +proxy_pass http://$ml_gw/...; +``` +That's why it survives ML stack restarts without nginx needing a reload. diff --git a/PHASE2_ANALYSIS.md b/PHASE2_ANALYSIS.md new file mode 100644 index 0000000..dc1d922 --- /dev/null +++ b/PHASE2_ANALYSIS.md @@ -0,0 +1,424 @@ +# Soko ML Phase 2 — Complete System Analysis + +**Date of Analysis:** 2026-05-14 +**Scope:** Phase 2 extension of the Soko ML service layer — transforming a static CSV-driven price prediction system into a live, data-ingesting, location-aware farmer decision support system. + +--- + +## 1. What Phase 2 Actually Builds + +Phase 1 delivered Prophet-based price prediction and collaborative-filtering recommendations, both trained on synthetic CSV data. Phase 2 replaces every static data source with live backend data and adds two new services. The result is a system that: + +- Learns from real transactions as they happen +- Recommends the best market to sell at (not just the price) +- Gives the farmer a GO/WAIT sell signal based on price trajectory +- Falls back gracefully when ML coverage is insufficient +- Tracks its own blind spots and escalates them for remediation + +--- + +## 2. Discovery Findings (What the Code Actually Contains) + +### Auth Service (`services/auth`) +- Stores only credentials (email, hashed password, role). No profile data, no GPS. +- All profile operations delegated to user-service at registration time. +- **Impact:** Bootstrap clients must call user-service, not auth-service. + +### User Service (`services/user`) +- `UserProfile` has `district` and `village` as plain strings. No lat/lng. +- `specialties` is a comma-separated string (e.g., `"maize,beans,coffee"`). +- `GET /users/farmers` already existed (public). `GET /users/buyers` did not — added. +- **Impact:** All GPS must come from district-centroid lookup tables, not user records. + +### Order Service (`services/order`) +- `OrderStatus` enum: `pending → confirmed → processing → dispatched → delivered → cancelled`. No "completed". +- `checkout()` publishes to `soko.transactions` with `crop = product.category` (e.g., `"Grains"`) — not the specific crop name. +- **Impact:** (1) Bootstrap must filter by `delivered` status. (2) `product_name` must be added to the Kafka payload to enable specific crop identification. + +### Produce Service (`services/produce`) +- Service is named `produce-service` in Docker. Prompt called it `listing-service` — all client code uses the actual name. +- No Kafka publisher. Listings are only accessible via HTTP. +- **Impact:** Periodic re-bootstrap every 15 minutes for profile sync (no push-based update path). + +--- + +## 3. Architectural Decisions and Their Implications + +### Decision 1: Add `product_name` to the Kafka Transaction Payload + +**What changed:** `services/order/app/kafka_publisher.py` and `routers/orders.py` now include `product_name` (the listing title, e.g., `"Maize (Dry)"`) alongside the existing `crop` (the category, e.g., `"Grains"`). + +**Why it was needed:** The ML price model is trained on specific crops (`maize_grain`, `yellow_beans`). The category field alone is ambiguous — `"Grains"` maps to 5+ crops. Without `product_name`, every streaming transaction event would need to guess the crop, producing noise in training data. + +**How it works:** `normalise_crop_from_order()` in `price_transformer.py` tries `product_name` first via the `CROP_NAME_NORMALISER` dict, then falls back to category-level mapping if the product name is unrecognisable. This means a transition period where older orders (no `product_name`) still produce usable (if slightly noisier) observations. + +**Risk:** Requires a coordinated deploy — data-ingestion-service must be updated after order-service, or it will receive events without `product_name` and fall back gracefully. + +--- + +### Decision 2: No Kafka Publishers in User/Produce Services → Periodic Re-Bootstrap Instead + +**What was considered:** Adding Kafka `profile_updated` events to user-service and produce-service, so the ML layer would get push-based profile updates. + +**Why rejected:** These services have no Kafka infrastructure. Adding it would be a significant cross-service change with no benefit to the core product, only to the ML layer. + +**What was built instead:** `recommendation-service` calls `GET /users/farmers` and `GET /users/buyers` every `PROFILE_REFRESH_INTERVAL_SECONDS` (default: 15 min) via `asyncio.create_task` background loop. Data is loaded directly into the in-memory `ProfileStore`. + +**Implication:** A farmer who updates their specialties will not be reflected in recommendations for up to 15 minutes. This is acceptable for a smallholder market with low-frequency profile changes. + +**Implication for bootstrap:** `data-ingestion-service` runs a full HTTP bootstrap of all farmers, buyers, and delivered orders at startup, then listens to `soko.transactions` for incremental updates. The Postgres feature store (`farmer_features`, `buyer_features`) is the ground truth; the recommendation service's in-memory store is a read cache of it. + +--- + +### Decision 3: Uganda District Centroids as GPS Approximation + +**What was built:** `DISTRICT_COORDINATES` in `farmer_transformer.py` maps 25 Uganda districts to approximate lat/lng centroid coordinates. `DISTRICT_TO_MARKET` maps districts to ML market node IDs (used by `price_transformer.py` and `geo_recommender.py`). + +**Why it was needed:** User profiles store `district` (e.g., `"Gulu"`) as a string. No GPS field exists. The location-service and recommendation-service both need coordinates for distance calculations. + +**Known limitation:** District centroids can be 20-50 km from where the farmer actually is. The `GEO_FILTER_RELAX_FACTOR` (default: 1.5) compensates by expanding the Haversine pre-filter radius by 50%. All API responses include a `location_precision: "district_centroid"` flag so callers know the coordinates are approximate. + +**Upgrade path:** If the user-service adds a GPS field to `UserProfile` in the future, `farmer_transformer.py` can use it directly — the `district_to_coords()` function is only called when no explicit coordinates are present. + +--- + +## 4. New Services + +### `data-ingestion-service` (port 8096) + +**Purpose:** Single entry point for all data flowing into the ML Feature Store. Owns the Postgres `soko_ml_db` database exclusively. + +**Startup sequence:** +1. Connect to Postgres +2. Apply schema if needed (`db-init` container runs schema.sql separately) +3. If `BOOTSTRAP_ON_STARTUP=true` AND all three core tables are empty: trigger full HTTP bootstrap from user/order/produce services +4. Start `TransactionStream` background thread consuming `soko.transactions` + +**Key endpoints:** +- `POST /bootstrap` — Trigger or re-trigger full sync +- `GET /bootstrap/status` — Check if bootstrap is in progress / complete +- `POST /ingest/order-event` — Accept a single transaction event (called by kafka-agent's `TransactionPriceCollector`) +- `GET /gaps/summary` — Show which crop/market pairs need more data +- `GET /coverage` — Full coverage map + +**Data quality mechanisms:** +- Outlier rejection: 3σ rolling window (last 30 observations per crop/market pair) +- Deduplication: Postgres `UNIQUE` constraint on `order_id` in `price_observations` +- Both streaming path (`TransactionStream`) and HTTP path (`/ingest/order-event`) exist simultaneously; deduplication handles double-writes + +--- + +### `location-service` (port 8003) + +**Purpose:** Given a farmer + crop, recommend which markets to sell at and when. + +**`POST /route` — The core endpoint:** +1. Load market registry from `soko_ml_db` (Redis-cached for 6h) +2. Get distances to all markets (Google Maps batch call, or Haversine fallback; Redis-cached per farmer×market pair for 30 days) +3. Fetch Prophet price predictions from `ml-gateway-service` for each market +4. Compute net value = `predicted_price × quantity - transport_cost` +5. Derive GO/WAIT sell signal from price trend, perishability, harvest month +6. Return ranked market list with signal + +**Three-tier fallback:** +- **Tier 1:** Full ML — Prophet predictions available for this crop/market pair (`observation_count >= 30`) +- **Tier 2:** Category price band — crop is known but specific market has insufficient data. Returns historical price range for the crop category +- **Tier 3:** Unknown crop — crop not in the ML catalogue at all. Returns generic advice, records a coverage gap event, publishes to `soko.gaps` + +**`POST /discover` — Buyer-side discovery:** +Finds farmers near a buyer who grow a requested crop. Uses `farmer_features` table (district centroids + Haversine filter). + +--- + +## 5. Modified Existing Services + +### `price-prediction-service` +- Added `train_all_models_from_feature_store()`: reads real price observations from Postgres instead of CSV; triggered when `retrain_requested` event arrives on `soko.ml.events` +- Original `train_all_models()` (CSV-based) retained as cold-start fallback + +### `recommendation-service` +- `ProfileStore` rewritten: `async reload()` fetches from Postgres feature store instead of reading CSV files +- Startup fails fast (SystemExit) if DB is unreachable — no silent degradation with empty profiles +- Background reload task runs every 15 min + +### `ml-gateway-service` +- Added proxying for `/location/*`, `/gaps/*`, `/coverage`, `/ingest/*` routes +- Added circuit breakers for `location` and `ingestion` services +- Health check now aggregates status from all 4 downstream services (price, rec, location, ingest); only price + rec are required for `overall: ok` + +### `kafka-agent` +- Added `CoverageGapConsumer`: consumes `soko.gaps` for monitoring/logging +- Added `TransactionPriceCollector`: consumes `soko.transactions`, forwards `purchase_completed` events to `data-ingestion-service /ingest/order-event` via HTTP (alternative path to the internal `TransactionStream`) + +--- + +## 6. Data Flow Diagrams + +### Transaction → Price Observation Flow + +``` +Order Service + └─ checkout() → publish to soko.transactions + │ + ├─► TransactionStream (data-ingestion-service internal thread) + │ └─ insert_price_observation() → price_observations (Postgres) + │ └─ trigger: trg_update_coverage → updates coverage_map + │ └─ if observation_count >= 30 → publish retrain_requested + │ └─ price-prediction-service retrains Prophet model + │ + └─► TransactionPriceCollector (kafka-agent) + └─ POST /ingest/order-event → data-ingestion-service + └─ insert_price_observation() (deduplicated by order_id) +``` + +### Farmer Market Routing Flow + +``` +Client → POST /ml/location/route + └─ ml-gateway-service → POST /route → location-service + │ + ├─ Load market registry (soko_ml_db → Redis cache) + ├─ Get farmer GPS (district centroid if no lat/lng) + ├─ Fetch road distances (Google Maps API → Redis cache 30 days) + ├─ Fetch Prophet predictions (ml-gateway → price-prediction-service) + │ └─ Tier check: coverage_map.is_model_ready? + │ ├─ Tier 1: full ML prediction → compute net value + │ ├─ Tier 2: category price band (insufficient data) + │ └─ Tier 3: unknown crop → gap_notifier → soko.gaps + ├─ Estimate transport cost (rate band lookup) + ├─ Derive sell signal (perishability → harvest month → trend) + └─ Return ranked_markets + signal +``` + +### Coverage Gap → Retraining Flow + +``` +location-service (Tier 3) + └─ gap_notifier.record_and_notify_gap() + ├─ INSERT / UPDATE coverage_gaps (Postgres) + └─ publish CoverageGapEvent → soko.gaps + ├─► CoverageGapConsumer (kafka-agent) — logs for ops monitoring + └─► (future) admin notification service + +data-ingestion-service (TransactionStream) + └─ When coverage_map.observation_count reaches MIN_OBSERVATIONS_FOR_MODEL (30) + └─ publish RetrainRequestedEvent → soko.ml.events + └─► price-prediction-service + └─ train_all_models_from_feature_store(crop, market) +``` + +--- + +## 7. Kafka Topics Reference + +| Topic | Partitions | Retention | Produced by | Consumed by | +|---|---|---|---|---| +| `soko.transactions` | 6 | 7 days | order-service | TransactionStream, TransactionPriceCollector, TransactionConsumer | +| `soko.interactions` | 6 | 3 days | (future frontend) | InteractionConsumer | +| `soko.price.requests` | 3 | 1 day | kafka-agent | PriceRequestConsumer | +| `soko.price.results` | 3 | 1 day | PriceRequestConsumer | kafka-agent | +| `soko.ml.events` | 2 | 14 days | data-ingestion-service | price-prediction-service | +| `soko.gaps` | 2 | 30 days | location-service | CoverageGapConsumer | +| `soko.dlq` | 2 | 30 days | all consumers on failure | (manual remediation) | + +--- + +## 8. Postgres Schema Summary (`soko_ml_db`) + +| Table | Purpose | Key Columns | +|---|---|---| +| `farmer_features` | ML-ready farmer profiles | `farmer_id`, `crops_offered TEXT[]`, `lat`, `lng`, `avg_rating` | +| `buyer_features` | ML-ready buyer profiles | `buyer_id`, `crop_interests TEXT[]`, `total_purchases` | +| `price_observations` | Real transaction prices | `crop`, `market`, `price_ugx_per_kg`, `order_id UNIQUE` | +| `coverage_map` | Model readiness per crop/market | `is_model_ready`, `observation_count`, `last_trained_at` | +| `market_registry` | All known markets with GPS | `market_id`, `lat`, `lng`, `active` | +| `coverage_gaps` | Crops/markets with no ML data | `crop`, `priority LOW/MEDIUM/HIGH`, `gap_count` | + +**Postgres trigger:** `trg_update_coverage` fires on every `price_observations` INSERT, incrementing `coverage_map.observation_count` and flipping `is_model_ready = TRUE` when count reaches `min_observations_needed` (default 30). This is the mechanism by which the system self-heals from Tier 2/3 to Tier 1 over time. + +--- + +## 9. Redis Key Patterns and TTLs + +| Key Pattern | TTL | Purpose | +|---|---|---| +| `dist:{farmer_id}:{market_id}` | 30 days | Road distance (km) from farmer to market | +| `route:{farmer_id}:{crop}` | 6 hours | Full ranked market list | +| `discover:{buyer_id}:{crop}:{radius}` | 1 hour | Nearby farmers for buyer | +| `market_registry` | 6 hours | Market list from Postgres | + +--- + +## 10. Environment Variables Reference + +See `services/soko-ml/.env.example` for the complete list with defaults and comments. + +**Critical variables that must be set before production:** + +| Variable | Default | Why it matters | +|---|---|---| +| `POSTGRES_PASSWORD` | `changeme` | Feature store security | +| `INTERNAL_API_KEY` | `internal-secret` | Auth between ML layer and core services | +| `USER_SERVICE_URL` | `http://user-service:8002` | Bootstrap farmer/buyer data | +| `ORDER_SERVICE_URL` | `http://order-service:8003` | Bootstrap historical price observations | +| `PRODUCE_SERVICE_URL` | `http://produce-service:8004` | Coverage map seeding | +| `GOOGLE_MAPS_API_KEY` | (empty) | Road distances; Haversine fallback used if absent | + +--- + +## 11. Port Map + +| Service | Container Port | Default Host Port | +|---|---|---| +| ml-gateway-service | 8000 | 8080 | +| price-prediction-service | 8001 | 8094 | +| recommendation-service | 8002 | 8095 | +| location-service | 8003 | 8003 | +| data-ingestion-service | 8004 | 8096 | +| Kafka (external) | 9092 | 29092 | +| Postgres (soko_ml_db) | 5432 | (not exposed) | +| Redis | 6379 | (not exposed) | + +--- + +## 12. Startup Order (Docker Compose `depends_on`) + +``` +zookeeper + └─ kafka + └─ kafka-init (creates topics, exits) + └─ price-prediction-service + └─ recommendation-service + └─ kafka-agent (waits for price + rec + data-ingestion) + +soko-ml-db + └─ db-init (applies schema.sql, exits) + └─ data-ingestion-service + └─ (bootstrap runs at startup if tables empty) + +redis + └─ price-prediction-service + └─ recommendation-service + └─ location-service + +data-ingestion-service + location-service + └─ ml-gateway-service (waits for all 4 downstream services) + └─ kafka-agent +``` + +--- + +## 13. Known Limitations and Future Work + +| Limitation | Impact | Recommended Fix | +|---|---|---| +| District centroid GPS (~20-50 km error) | Distance-ranked markets may be slightly wrong | Add `lat/lng` to `UserProfile` in user-service | +| No real-time profile push | Up to 15 min lag for updated farmer specialties | Add Kafka publisher to user-service `PUT /me` | +| Produce-service listing prices not in feature store | Farmer asking price not used in market routing | Periodic sync of active listing prices | +| Google Maps quota | Distance calls are batched and cached; if quota exhausted, Haversine is used | Monitor API quota; upgrade plan if needed | +| Prophet cold-start for new crop/market pairs | Tier 2 until 30 observations accumulate | Lower `MIN_OBSERVATIONS_FOR_MODEL` to 10 during initial rollout | +| `kafka-agent` `TransactionPriceCollector` doubles write load | Acceptable due to Postgres deduplication | Can be disabled if data-ingestion-service `TransactionStream` is confirmed reliable | + +--- + +## 14. File Manifest (Phase 2 New / Modified Files) + +### New Services +``` +services/soko-ml/data-ingestion-service/ + src/ + main.py FastAPI app, lifespan bootstrap + schemas.py Pydantic request/response models + feature_store.py All asyncpg DB operations + health.py Health checks (DB + backend services) + transformers/ + farmer_transformer.py Profile → farmer_features row + buyer_transformer.py Profile → buyer_features row + price_transformer.py Transaction event → price_observations row + clients/ + user_client.py GET /users/farmers, /users/buyers + order_client.py GET /orders?status=delivered + listing_client.py GET /listings + bootstrap/ + auth_bootstrap.py Bulk farmer + buyer sync + order_bootstrap.py Bulk price observation sync + listing_bootstrap.py Coverage map seeding + market_bootstrap.py No-op (markets seeded in schema.sql) + streams/ + transaction_stream.py Kafka consumer thread + tests/ + test_transformers.py Pure unit tests (no DB) + test_feature_store.py Integration tests (skipped if DB absent) + requirements.txt + Dockerfile + +services/soko-ml/location-service/ + src/ + main.py FastAPI app + schemas.py Pydantic models + market_router.py Core routing logic + geo_recommender.py Buyer→farmer discovery + google_maps_client.py Distance Matrix API + Haversine fallback + transport_cost.py Rate-band cost estimation + sell_signal.py GO/WAIT signal derivation + fallback.py Tier 1/2/3 fallback logic + gap_notifier.py Gap recording + Kafka publish + cache.py Redis key patterns + helpers + tests/ + test_market_router.py Unit + integration tests + requirements.txt + Dockerfile +``` + +### New Infrastructure +``` +services/soko-ml/db/ + schema.sql Postgres schema + seed data + trigger + +services/soko-ml/.env.example Updated with all Phase 2 variables +``` + +### Modified Existing Files +``` +services/order/app/kafka_publisher.py Added product_name field +services/order/app/routers/orders.py Passes product_name on checkout + cancel + +services/user/app/routers/profile.py Added GET /users/buyers endpoint + +services/soko-ml/shared/events.py Added SokoTransactionEvent, CoverageGapEvent, + RetrainRequestedEvent + +services/soko-ml/price-prediction-service/src/predictor.py + Added train_all_models_from_feature_store() + +services/soko-ml/recommendation-service/ + src/recommender.py ProfileStore rewritten (async reload from Postgres) + src/main.py Removed CSV paths; added periodic reload + src/feature_store_client.py New — asyncpg reads from farmer_features/buyer_features + src/geo_filter.py New — Haversine pre-filter helper + requirements.txt Added asyncpg + +services/soko-ml/ml-gateway-service/ + src/proxy.py Added location + ingestion circuit breakers + src/main.py Added /location/*, /gaps/*, /coverage, /ingest/* routes + +services/soko-ml/kafka-agent/ + src/agent.py Added CoverageGapConsumer + TransactionPriceCollector + src/consumers/coverage_gap_consumer.py New + src/consumers/transaction_price_collector.py New + +services/soko-ml/docker-compose.yml + Added: soko-ml-db, db-init, data-ingestion-service, location-service + Added: soko.gaps Kafka topic + Updated: ml-gateway depends_on, kafka-agent depends_on + DATA_INGESTION_SERVICE_URL + Updated: volumes (soko_ml_db_data) + +Makefile (root) + Added: db-up, db-shell, db-reset, cold-start + Added: ingest-bootstrap, ingest-status, gaps-summary, gaps-reset + Added: dev-location, dev-ingest, logs-location, logs-ingest + Added: test-location, test-ingest + Added: smoke-route, smoke-discover, smoke-fallback, smoke-tier3, smoke-ingest + Added: INGEST_VENV, LOC_VENV + Updated: install, infra-up, infra-down, kafka-topics, health, clean +``` diff --git a/docker-compose.yml b/docker-compose.yml index 8157d7b..e7e5f9b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -130,6 +130,7 @@ services: condition: service_healthy networks: - soko_net + - soko-ml-bridge restart: on-failure # ───────────────────────────────────────── @@ -170,6 +171,7 @@ services: condition: service_healthy networks: - soko_net + - soko-ml-bridge restart: on-failure # ───────────────────────────────────────── @@ -215,6 +217,7 @@ services: condition: service_started networks: - soko_net + - soko-ml-bridge restart: on-failure # ───────────────────────────────────────── diff --git a/execution.md b/execution.md new file mode 100644 index 0000000..5c74101 --- /dev/null +++ b/execution.md @@ -0,0 +1,237 @@ +# Soko Phase 2 — Execution & Testing Walkthrough + +## Prerequisites + +Three config changes are required before running the stack end-to-end: + +- `data-ingestion-service` joined to `soko-ml-bridge` (so it can reach core services) +- `user_service`, `order_service`, `produce_service` joined to `soko-ml-bridge` +- `services/soko-ml/.env` updated with all Phase 2 variables + +All three are already applied in the codebase. + +--- + +## Step 1 — Unit tests (no Docker needed) + +Pure function tests — crop normalisers, transport rates, sell signal logic. Always pass offline. + +```bash +# Install Python deps (only needed once, or after adding a new service) +make install + +# Transformer tests: crop normalisation, market mapping, UGX price field names +make test-ingest + +# Location tests: haversine distance, transport cost bands, sell signal derivation +make test-location +``` + +Expected: all green. DB-dependent tests auto-skip with `pytest.skip("Postgres unreachable")` — that is correct behaviour offline. + +--- + +## Step 2 — Tear down any existing ML stack + +```bash +make ml-down +``` + +--- + +## Step 3 — Cold-start Phase 2 + +Starts the ML Postgres, applies the schema, builds and starts all seven ML services, then auto-triggers the initial data bootstrap from core services. + +```bash +make cold-start +``` + +Takes ~90 seconds. Internal sequence: +1. Ensures `soko-ml-bridge` network exists +2. Starts `soko-ml-db` and runs `db/schema.sql` +3. Builds and starts all ML containers +4. Waits 20 s for healthchecks to pass +5. POSTs to `/bootstrap` on the ingest service + +Follow progress: +```bash +make ml-logs +# Ctrl-C when all services show "Application startup complete" +``` + +--- + +## Step 4 — Restart core services on the new network + +Core service containers were started before `soko-ml-bridge` was added to them. Recreate them so they join it. + +```bash +make core-restart +``` + +After this, `data-ingestion-service` can reach `http://user_service:8002`, `http://order_service:8004`, and `http://produce_service:8003` directly over `soko-ml-bridge`. + +--- + +## Step 5 — Health check every service + +```bash +make health +``` + +| Service | Port | +|---|---| +| API gateway (nginx) | `:80` | +| ML gateway | `:8080` | +| Price prediction | `:8094` | +| Recommendation | `:8095` | +| Location service | `:8003` | +| Data ingestion | `:8096` | + +All should return `{"status":"ok"}`. If any show `UNREACHABLE`, tail their logs: + +```bash +make logs-location +make logs-ingest +make logs-gateway +``` + +--- + +## Step 6 — Check bootstrap status + +```bash +make ingest-status +``` + +Returns a JSON breakdown of farmers, produce listings, and price observations pulled from core services. If `bootstrap_complete: false`, trigger it manually: + +```bash +make ingest-bootstrap +``` + +--- + +## Step 7 — Smoke test Phase 1 features (regression check) + +Verify price predictions and recommendations are unaffected: + +```bash +make smoke-test +``` + +Hits `/price/predict` with `maize_grain` at `Kisenyi_Kampala` and both recommendation endpoints. + +--- + +## Step 8 — Smoke test Phase 2 location endpoints + +**Market routing** — ranked markets with sell signal and transport cost for a farmer: + +```bash +make smoke-route +``` + +Response includes: +- `tier` — 1 (full ML), 2 (category band), or 3 (unknown crop) +- `ranked_markets` — each with `ugx_per_kg`, `mode` (e.g. `boda_cargo`, `pickup_truck`), and `sell_signal` + +**Buyer-to-farmer discovery** — farmers near a buyer within 150 km under 2 000 UGX/kg: + +```bash +make smoke-discover +``` + +**Tier 2 fallback** — niche crop with thin ML coverage falls back to category price band: + +```bash +make smoke-fallback +``` + +**Tier 3** — completely unknown crop (`moringa`) returns a gap notification and publishes to `soko.gaps`: + +```bash +make smoke-tier3 +``` + +--- + +## Step 9 — Smoke test the ingest endpoint + +Posts a synthetic `purchase_completed` event and writes a `price_observation` row: + +```bash +make smoke-ingest +``` + +Confirm it landed in the DB: + +```bash +make db-shell +``` + +Inside psql: + +```sql +SELECT crop, market, price_per_kg, currency, source +FROM price_observations +ORDER BY observed_at DESC +LIMIT 5; +\q +``` + +Expected: `currency = UGX`, `price_per_kg = 1400` for event `TEST-001`. + +--- + +## Step 10 — Gap report + +After the Tier 3 smoke test, `moringa` will appear here: + +```bash +make gaps-summary +``` + +Returns coverage counts per crop/market and a `gap_level` (`low` / `medium` / `high`) based on the thresholds in `services/soko-ml/.env`. + +--- + +## Quick reference — troubleshooting + +| Problem | Command | +|---|---| +| DB schema missing | `make db-reset` *(destructive)* | +| Kafka topics not created | `make kafka-topics` | +| Redis full / stale cache | `make redis-cli` → `FLUSHALL` | +| Re-run bootstrap | `make ingest-bootstrap` | +| Wipe everything and restart | `make clean-docker` then `make cold-start` | +| Rebuild one service only | `docker compose -f services/soko-ml/docker-compose.yml up --build -d ` | + +--- + +## Port map + +| Service | Host port | Container port | +|---|---|---| +| nginx (API gateway) | 80 | 80 | +| ML gateway | 8080 | 8000 | +| Price prediction | 8094 | 8001 | +| Recommendation | 8095 | 8002 | +| Location service | 8003 | 8003 | +| Data ingestion | 8096 | 8004 | +| Kafka (external) | 29092 | 29092 | + +--- + +## Transport cost reference (FarasUG / SafeBoda benchmarks) + +| Distance | Mode | UGX/kg | +|---|---|---| +| 0 – 25 km | boda_cargo | 290 | +| 25 – 80 km | taxi_van | 420 | +| 80 – 200 km | pickup_truck | 620 | +| 200 – 400 km | shared_lorry | 850 | +| 400+ km | cross_region | 1 100 | + +Rates reflect partial-load pricing for 100–500 kg loads. All monetary values in UGX. diff --git a/services/order/app/kafka_publisher.py b/services/order/app/kafka_publisher.py index 9e37117..116f062 100644 --- a/services/order/app/kafka_publisher.py +++ b/services/order/app/kafka_publisher.py @@ -41,6 +41,7 @@ def publish_transaction( quantity_kg: float, price_per_kg_ugx: float, total_ugx: float, + product_name: str = "", ) -> None: """Fire-and-forget publish to soko.transactions. Silently skips if broker unreachable.""" producer = _get_producer() @@ -53,6 +54,7 @@ def publish_transaction( "buyer_id": buyer_id, "farmer_id": farmer_id, "crop": crop, + "product_name": product_name, "market": market, "quantity_kg": quantity_kg, "price_per_kg_ugx": price_per_kg_ugx, diff --git a/services/order/app/routers/orders.py b/services/order/app/routers/orders.py index 62f0a44..3eced37 100644 --- a/services/order/app/routers/orders.py +++ b/services/order/app/routers/orders.py @@ -235,6 +235,7 @@ async def checkout( buyer_id=str(order.buyer_id), farmer_id=str(product.get("farmerId", "")), crop=str(product.get("category", "unknown")), + product_name=str(product.get("name", "")), market=order.delivery_district or "Kisenyi_Kampala", quantity_kg=float(item.quantity), price_per_kg_ugx=float(item.unitPrice), @@ -319,6 +320,7 @@ async def cancel_order( buyer_id=str(order.buyer_id), farmer_id=str(item.farmer_id), crop=str(item.category), + product_name=str(item.product_name), market=order.delivery_district or "Kisenyi_Kampala", quantity_kg=float(item.quantity), price_per_kg_ugx=float(item.unit_price), diff --git a/services/soko-ml/.env.example b/services/soko-ml/.env.example index 8bc09a5..54991dd 100644 --- a/services/soko-ml/.env.example +++ b/services/soko-ml/.env.example @@ -1,31 +1,69 @@ -# ── Service Ports ───────────────────────────────────────────────────────────── -PRICE_SERVICE_PORT=8094 -REC_SERVICE_PORT=8095 -GATEWAY_PORT=8080 +# ============================================================================= +# Soko ML Stack — environment variables +# +# Copy to .env and fill in real values before running `make ml-up`. +# All variables have safe defaults for local development; items marked +# REQUIRED must be set before the stack will work correctly end-to-end. +# ============================================================================= + +# ── ML Feature Store (Postgres) ─────────────────────────────────────────────── +POSTGRES_USER=soko_ml +POSTGRES_PASSWORD=changeme # REQUIRED: change before production # ── Redis ───────────────────────────────────────────────────────────────────── REDIS_HOST=redis REDIS_PORT=6379 REDIS_DB=0 -REDIS_PASSWORD= +REDIS_PASSWORD= # leave empty for no-auth (default) # ── Kafka ───────────────────────────────────────────────────────────────────── KAFKA_BOOTSTRAP_SERVERS=kafka:9092 -KAFKA_PRICE_REQUEST_TOPIC=soko.price.requests -KAFKA_PRICE_RESULT_TOPIC=soko.price.results KAFKA_TRANSACTION_TOPIC=soko.transactions KAFKA_INTERACTION_TOPIC=soko.interactions +KAFKA_PRICE_REQUEST_TOPIC=soko.price.requests +KAFKA_PRICE_RESULT_TOPIC=soko.price.results KAFKA_ML_EVENTS_TOPIC=soko.ml.events KAFKA_DLQ_TOPIC=soko.dlq +KAFKA_GAPS_TOPIC=soko.gaps + +# ── Backend service URLs (reachable via soko-ml-bridge network) ─────────────── +USER_SERVICE_URL=http://user-service:8002 # REQUIRED +ORDER_SERVICE_URL=http://order-service:8003 # REQUIRED +PRODUCE_SERVICE_URL=http://produce-service:8004 # REQUIRED + +# Shared internal API key — must match INTERNAL_SECRET in core services +INTERNAL_API_KEY=internal-secret # REQUIRED: change before production + +# ── Google Maps ─────────────────────────────────────────────────────────────── +# If empty, location-service falls back to Haversine straight-line distances. +GOOGLE_MAPS_API_KEY= # optional but recommended +MAPS_DISTANCE_CACHE_TTL_SECONDS=2592000 # 30 days + +# ── Port bindings (host-side) ───────────────────────────────────────────────── +GATEWAY_PORT=8080 +PRICE_SERVICE_PORT=8094 +REC_SERVICE_PORT=8095 +LOCATION_SERVICE_PORT=8097 +INGEST_SERVICE_PORT=8096 + +# ── Data ingestion behaviour ────────────────────────────────────────────────── +BOOTSTRAP_ON_STARTUP=true +PROFILE_REFRESH_INTERVAL_SECONDS=900 # 15 min — recommendation-service re-sync +MIN_OBSERVATIONS_FOR_MODEL=30 +PRICE_ANOMALY_SIGMA_THRESHOLD=3.0 + +# ── Coverage gap thresholds ─────────────────────────────────────────────────── +GAP_MEDIUM_THRESHOLD=5 +GAP_HIGH_THRESHOLD=10 + +# ── Geo filter ──────────────────────────────────────────────────────────────── +GEO_FILTER_RELAX_FACTOR=1.5 -# ── ML Config ───────────────────────────────────────────────────────────────── +# ── ML service config ───────────────────────────────────────────────────────── MODEL_DIR=/app/models -FARMERS_DATA_PATH=/app/data/raw/farmers.csv -BUYERS_DATA_PATH=/app/data/raw/buyers.csv PRICE_CACHE_TTL_SECONDS=86400 REC_CACHE_TTL_SECONDS=3600 DEFAULT_TOP_N=5 # ── Logging ─────────────────────────────────────────────────────────────────── LOG_LEVEL=INFO -SERVICE_NAME=soko-ml diff --git a/services/soko-ml/data-ingestion-service/Dockerfile b/services/soko-ml/data-ingestion-service/Dockerfile new file mode 100644 index 0000000..0f3faa6 --- /dev/null +++ b/services/soko-ml/data-ingestion-service/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.11-slim + +WORKDIR /app + +RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY src/ ./src/ + +CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8004", "--log-level", "info"] diff --git a/services/soko-ml/data-ingestion-service/requirements.txt b/services/soko-ml/data-ingestion-service/requirements.txt new file mode 100644 index 0000000..f7afc81 --- /dev/null +++ b/services/soko-ml/data-ingestion-service/requirements.txt @@ -0,0 +1,11 @@ +fastapi==0.111.0 +uvicorn[standard]==0.29.0 +pydantic==2.7.1 +asyncpg==0.29.0 +confluent-kafka==2.4.0 +httpx==0.27.0 +structlog==24.2.0 +redis[asyncio]==5.0.4 +numpy==1.26.4 +pytest==8.2.0 +pytest-asyncio==0.23.6 diff --git a/services/soko-ml/data-ingestion-service/src/__init__.py b/services/soko-ml/data-ingestion-service/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/services/soko-ml/data-ingestion-service/src/bootstrap/__init__.py b/services/soko-ml/data-ingestion-service/src/bootstrap/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/services/soko-ml/data-ingestion-service/src/bootstrap/auth_bootstrap.py b/services/soko-ml/data-ingestion-service/src/bootstrap/auth_bootstrap.py new file mode 100644 index 0000000..9c46aed --- /dev/null +++ b/services/soko-ml/data-ingestion-service/src/bootstrap/auth_bootstrap.py @@ -0,0 +1,46 @@ +""" +Pulls all farmer and buyer profiles from user-service and populates the feature store. +Called once at startup (or on make ingest-bootstrap) when tables are empty. +""" +import logging + +from ..clients.user_client import fetch_all_farmers, fetch_all_buyers +from ..transformers.farmer_transformer import transform_farmer +from ..transformers.buyer_transformer import transform_buyer +from ..feature_store import bulk_upsert_farmers, bulk_upsert_buyers + +log = logging.getLogger(__name__) + + +async def bootstrap_farmers() -> int: + records = [] + async for profile in fetch_all_farmers(): + try: + records.append(transform_farmer(profile)) + except Exception as exc: + log.warning(f"Could not transform farmer {profile.get('id')}: {exc}") + + if not records: + log.warning("No farmer profiles returned from user-service — skipping farmer bootstrap") + return 0 + + count = await bulk_upsert_farmers(records) + log.info(f"Farmer bootstrap complete: {count}/{len(records)} upserted") + return count + + +async def bootstrap_buyers() -> int: + records = [] + async for profile in fetch_all_buyers(): + try: + records.append(transform_buyer(profile)) + except Exception as exc: + log.warning(f"Could not transform buyer {profile.get('id')}: {exc}") + + if not records: + log.warning("No buyer profiles returned from user-service — skipping buyer bootstrap") + return 0 + + count = await bulk_upsert_buyers(records) + log.info(f"Buyer bootstrap complete: {count}/{len(records)} upserted") + return count diff --git a/services/soko-ml/data-ingestion-service/src/bootstrap/listing_bootstrap.py b/services/soko-ml/data-ingestion-service/src/bootstrap/listing_bootstrap.py new file mode 100644 index 0000000..c5435fc --- /dev/null +++ b/services/soko-ml/data-ingestion-service/src/bootstrap/listing_bootstrap.py @@ -0,0 +1,69 @@ +""" +Scans active listings from produce-service to seed the coverage map +with what crops are actually being sold, even before orders arrive. +""" +import logging + +from ..clients.listing_client import fetch_all_listings +from ..transformers.farmer_transformer import CROP_NAME_NORMALISER, DISTRICT_TO_MARKET, DEFAULT_MARKET +from ..feature_store import get_pool + +log = logging.getLogger(__name__) + +CATEGORY_TO_CROP: dict[str, list[str]] = { + "Grains": ["maize_grain", "sorghum", "millet"], + "Vegetables": ["tomatoes", "kale"], + "Fruits": ["matoke"], + "Tubers": ["irish_potatoes", "cassava_chips"], + "Legumes": ["yellow_beans"], +} + + +async def bootstrap_listings() -> int: + """ + For each active listing, ensure the crop-market pair exists in coverage_map. + Does not insert price observations — that's done from real orders. + """ + pairs_seen: set[tuple[str, str]] = set() + + async for listing in fetch_all_listings(): + name = listing.get("name", "") + category = listing.get("category", "") + district = listing.get("district", "") + + market = DISTRICT_TO_MARKET.get(district, DEFAULT_MARKET) + + # Try to identify specific crop from product name first + norm_name = CROP_NAME_NORMALISER.get(name.lower().strip()) + if norm_name: + pairs_seen.add((norm_name, market)) + else: + # Fall back to all crops in this category + for crop in CATEGORY_TO_CROP.get(category, []): + pairs_seen.add((crop, market)) + + if not pairs_seen: + log.info("No active listings found — coverage map already seeded from schema.sql") + return 0 + + pool = await get_pool() + count = 0 + async with pool.acquire() as conn: + for crop, market in pairs_seen: + await conn.execute( + """ + INSERT INTO coverage_map (crop, market) + VALUES ($1, $2) + ON CONFLICT (crop, market) DO NOTHING + """, + crop, market, + ) + count += 1 + + log.info(f"Listing bootstrap complete: {count} coverage_map pairs ensured") + return count + + +async def bootstrap_market_bootstrap() -> None: + """No-op: market registry is seeded in schema.sql at db-init time.""" + log.info("Market registry loaded from schema.sql — no runtime bootstrap needed") diff --git a/services/soko-ml/data-ingestion-service/src/bootstrap/market_bootstrap.py b/services/soko-ml/data-ingestion-service/src/bootstrap/market_bootstrap.py new file mode 100644 index 0000000..1ce68dd --- /dev/null +++ b/services/soko-ml/data-ingestion-service/src/bootstrap/market_bootstrap.py @@ -0,0 +1,12 @@ +""" +Market registry bootstrap — no-op because market data is seeded in schema.sql. +Module exists to maintain the file structure described in the system spec. +""" +import logging + +log = logging.getLogger(__name__) + + +async def bootstrap_markets() -> int: + log.info("Market registry is seeded in db/schema.sql — no HTTP bootstrap required") + return 0 diff --git a/services/soko-ml/data-ingestion-service/src/bootstrap/order_bootstrap.py b/services/soko-ml/data-ingestion-service/src/bootstrap/order_bootstrap.py new file mode 100644 index 0000000..b4ec237 --- /dev/null +++ b/services/soko-ml/data-ingestion-service/src/bootstrap/order_bootstrap.py @@ -0,0 +1,36 @@ +""" +Pulls all delivered orders from order-service and inserts price observations. +Falls back gracefully if the internal endpoint is not yet exposed. +""" +import logging + +from ..clients.order_client import fetch_delivered_orders +from ..transformers.price_transformer import transform_order_item +from ..feature_store import bulk_insert_price_observations + +log = logging.getLogger(__name__) + + +async def bootstrap_orders() -> int: + records = [] + async for order in fetch_delivered_orders(): + order_id = order.get("id", "") + delivery_district = order.get("delivery_district", "") + completed_at = order.get("updated_at", "") # closest to completion timestamp + items = order.get("items", []) + + for item in items: + try: + rec = transform_order_item(order_id, item, delivery_district, completed_at) + if rec: + records.append(rec) + except Exception as exc: + log.warning(f"Could not transform order item {item.get('id')}: {exc}") + + if not records: + log.info("No historical price observations from orders — will populate from live events") + return 0 + + count = await bulk_insert_price_observations(records) + log.info(f"Order bootstrap complete: {count} price observations inserted") + return count diff --git a/services/soko-ml/data-ingestion-service/src/clients/__init__.py b/services/soko-ml/data-ingestion-service/src/clients/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/services/soko-ml/data-ingestion-service/src/clients/listing_client.py b/services/soko-ml/data-ingestion-service/src/clients/listing_client.py new file mode 100644 index 0000000..2f399e7 --- /dev/null +++ b/services/soko-ml/data-ingestion-service/src/clients/listing_client.py @@ -0,0 +1,49 @@ +""" +Async HTTP client for produce-service (listing-service). +Used to scan the active listing catalogue for coverage map seeding. +""" +import os +import logging +from typing import AsyncIterator + +import httpx + +log = logging.getLogger(__name__) + +LISTING_SERVICE_URL = os.getenv("LISTING_SERVICE_URL", "http://produce-service:3004") +INTERNAL_API_KEY = os.getenv("INTERNAL_API_KEY", "") +PAGE_LIMIT = 100 + + +def _headers() -> dict: + return {"x-internal-secret": INTERNAL_API_KEY} + + +async def fetch_all_listings() -> AsyncIterator[dict]: + """ + Paginates through GET /listings (public endpoint) and yields each listing dict. + Produce-service uses /listings not /internal/listings. + """ + page = 1 + async with httpx.AsyncClient(timeout=30.0) as client: + while True: + try: + resp = await client.get( + f"{LISTING_SERVICE_URL}/listings", + params={"page": page, "limit": PAGE_LIMIT}, + ) + resp.raise_for_status() + listings = resp.json() + except httpx.HTTPError as exc: + log.error(f"Failed to fetch listings page {page}: {exc}") + return + + if not listings: + return + + for listing in listings: + yield listing + + if len(listings) < PAGE_LIMIT: + return + page += 1 diff --git a/services/soko-ml/data-ingestion-service/src/clients/order_client.py b/services/soko-ml/data-ingestion-service/src/clients/order_client.py new file mode 100644 index 0000000..873c68b --- /dev/null +++ b/services/soko-ml/data-ingestion-service/src/clients/order_client.py @@ -0,0 +1,57 @@ +""" +Async HTTP client for order-service internal API. +Fetches delivered orders for price observation bootstrap. +""" +import os +import logging +from typing import AsyncIterator + +import httpx + +log = logging.getLogger(__name__) + +ORDER_SERVICE_URL = os.getenv("ORDER_SERVICE_URL", "http://order-service:3002") +INTERNAL_API_KEY = os.getenv("INTERNAL_API_KEY", "") +PAGE_LIMIT = 100 + + +def _headers() -> dict: + return {"x-internal-secret": INTERNAL_API_KEY} + + +async def fetch_delivered_orders() -> AsyncIterator[dict]: + """ + Paginates through GET /internal/orders?status=delivered and yields each order dict. + Falls back gracefully if the internal endpoint is not yet available. + """ + page = 1 + async with httpx.AsyncClient(timeout=30.0) as client: + while True: + try: + resp = await client.get( + f"{ORDER_SERVICE_URL}/internal/orders", + params={"status": "delivered", "page": page, "limit": PAGE_LIMIT}, + headers=_headers(), + ) + if resp.status_code == 404: + log.warning( + "order-service /internal/orders endpoint not found — " + "price bootstrap from orders skipped. " + "Price observations will come from live soko.transactions events." + ) + return + resp.raise_for_status() + orders = resp.json() + except httpx.HTTPError as exc: + log.error(f"Failed to fetch orders page {page}: {exc}") + return + + if not orders: + return + + for order in orders: + yield order + + if len(orders) < PAGE_LIMIT: + return + page += 1 diff --git a/services/soko-ml/data-ingestion-service/src/clients/user_client.py b/services/soko-ml/data-ingestion-service/src/clients/user_client.py new file mode 100644 index 0000000..9b7842d --- /dev/null +++ b/services/soko-ml/data-ingestion-service/src/clients/user_client.py @@ -0,0 +1,75 @@ +""" +Async HTTP client for user-service. +Fetches farmer and buyer profiles for bootstrap. +""" +import os +import logging +from typing import AsyncIterator + +import httpx + +log = logging.getLogger(__name__) + +USER_SERVICE_URL = os.getenv("USER_SERVICE_URL", "http://user-service:3003") +INTERNAL_API_KEY = os.getenv("INTERNAL_API_KEY", "") +PAGE_LIMIT = 100 + + +def _headers() -> dict: + return {"x-internal-secret": INTERNAL_API_KEY} + + +async def fetch_all_farmers() -> AsyncIterator[dict]: + """Paginates through GET /users/farmers and yields each farmer profile dict.""" + page = 1 + async with httpx.AsyncClient(timeout=30.0) as client: + while True: + try: + resp = await client.get( + f"{USER_SERVICE_URL}/users/farmers", + params={"page": page, "limit": PAGE_LIMIT}, + headers=_headers(), + ) + resp.raise_for_status() + profiles = resp.json() + except httpx.HTTPError as exc: + log.error(f"Failed to fetch farmers page {page}: {exc}") + return + + if not profiles: + return + + for profile in profiles: + yield profile + + if len(profiles) < PAGE_LIMIT: + return + page += 1 + + +async def fetch_all_buyers() -> AsyncIterator[dict]: + """Paginates through GET /users/buyers and yields each buyer profile dict.""" + page = 1 + async with httpx.AsyncClient(timeout=30.0) as client: + while True: + try: + resp = await client.get( + f"{USER_SERVICE_URL}/users/buyers", + params={"page": page, "limit": PAGE_LIMIT}, + headers=_headers(), + ) + resp.raise_for_status() + profiles = resp.json() + except httpx.HTTPError as exc: + log.error(f"Failed to fetch buyers page {page}: {exc}") + return + + if not profiles: + return + + for profile in profiles: + yield profile + + if len(profiles) < PAGE_LIMIT: + return + page += 1 diff --git a/services/soko-ml/data-ingestion-service/src/feature_store.py b/services/soko-ml/data-ingestion-service/src/feature_store.py new file mode 100644 index 0000000..7bf1b9c --- /dev/null +++ b/services/soko-ml/data-ingestion-service/src/feature_store.py @@ -0,0 +1,336 @@ +""" +Postgres read/write for all ML feature tables. +Uses asyncpg connection pooling. All queries are parameterised — no string formatting in SQL. +""" +import os +import logging +from typing import Optional + +import asyncpg + +log = logging.getLogger(__name__) + +_pool: Optional[asyncpg.Pool] = None + + +async def get_pool() -> asyncpg.Pool: + global _pool + if _pool is None: + dsn = os.getenv("POSTGRES_DSN", "postgresql://soko_ml:changeme@soko-ml-db:5432/soko_ml_db") + _pool = await asyncpg.create_pool(dsn, min_size=2, max_size=10) + return _pool + + +async def close_pool() -> None: + global _pool + if _pool: + await _pool.close() + _pool = None + + +# ── Bootstrap guards ────────────────────────────────────────────────────────── + +async def is_bootstrap_needed() -> bool: + """Returns True if all feature tables are empty (fresh environment).""" + pool = await get_pool() + async with pool.acquire() as conn: + farmer_count = await conn.fetchval("SELECT COUNT(*) FROM farmer_features") + buyer_count = await conn.fetchval("SELECT COUNT(*) FROM buyer_features") + price_count = await conn.fetchval( + "SELECT COUNT(*) FROM price_observations WHERE source = 'soko_order'" + ) + return farmer_count == 0 and buyer_count == 0 and price_count == 0 + + +# ── Farmer features ─────────────────────────────────────────────────────────── + +async def upsert_farmer(record: dict) -> None: + pool = await get_pool() + async with pool.acquire() as conn: + await conn.execute( + """ + INSERT INTO farmer_features ( + farmer_id, name, lat, lng, district, + crops_offered, markets_served, + avg_rating, fulfillment_rate, avg_response_time_hrs, + total_orders_completed, total_orders_cancelled, + total_listings, last_active_at, synced_at + ) VALUES ( + $1, $2, $3, $4, $5, + $6, $7, + $8, $9, $10, + $11, $12, + $13, $14, NOW() + ) + ON CONFLICT (farmer_id) DO UPDATE SET + name = EXCLUDED.name, + lat = EXCLUDED.lat, + lng = EXCLUDED.lng, + district = EXCLUDED.district, + crops_offered = EXCLUDED.crops_offered, + markets_served = EXCLUDED.markets_served, + avg_rating = EXCLUDED.avg_rating, + fulfillment_rate = EXCLUDED.fulfillment_rate, + avg_response_time_hrs = EXCLUDED.avg_response_time_hrs, + total_orders_completed = EXCLUDED.total_orders_completed, + total_orders_cancelled = EXCLUDED.total_orders_cancelled, + total_listings = EXCLUDED.total_listings, + last_active_at = EXCLUDED.last_active_at, + synced_at = NOW() + """, + record["farmer_id"], + record["name"], + record.get("lat"), + record.get("lng"), + record.get("district"), + record.get("crops_offered", []), + record.get("markets_served", []), + record.get("avg_rating", 0.0), + record.get("fulfillment_rate", 1.0), + record.get("avg_response_time_hrs", 24.0), + record.get("total_orders_completed", 0), + record.get("total_orders_cancelled", 0), + record.get("total_listings", 0), + record.get("last_active_at"), + ) + + +async def bulk_upsert_farmers(records: list[dict]) -> int: + count = 0 + for record in records: + try: + await upsert_farmer(record) + count += 1 + except Exception as exc: + log.error(f"Failed to upsert farmer {record.get('farmer_id')}: {exc}") + return count + + +# ── Buyer features ──────────────────────────────────────────────────────────── + +async def upsert_buyer(record: dict) -> None: + pool = await get_pool() + async with pool.acquire() as conn: + await conn.execute( + """ + INSERT INTO buyer_features ( + buyer_id, name, lat, lng, district, + preferred_crops, preferred_markets, + avg_order_volume_kg, payment_reliability, + avg_spend_per_order, total_purchases, + last_active_at, synced_at + ) VALUES ( + $1, $2, $3, $4, $5, + $6, $7, + $8, $9, + $10, $11, + $12, NOW() + ) + ON CONFLICT (buyer_id) DO UPDATE SET + name = EXCLUDED.name, + lat = EXCLUDED.lat, + lng = EXCLUDED.lng, + district = EXCLUDED.district, + preferred_crops = EXCLUDED.preferred_crops, + preferred_markets = EXCLUDED.preferred_markets, + avg_order_volume_kg = EXCLUDED.avg_order_volume_kg, + payment_reliability = EXCLUDED.payment_reliability, + avg_spend_per_order = EXCLUDED.avg_spend_per_order, + total_purchases = EXCLUDED.total_purchases, + last_active_at = EXCLUDED.last_active_at, + synced_at = NOW() + """, + record["buyer_id"], + record["name"], + record.get("lat"), + record.get("lng"), + record.get("district"), + record.get("preferred_crops", []), + record.get("preferred_markets", []), + record.get("avg_order_volume_kg", 0.0), + record.get("payment_reliability", 1.0), + record.get("avg_spend_per_order", 0.0), + record.get("total_purchases", 0), + record.get("last_active_at"), + ) + + +async def bulk_upsert_buyers(records: list[dict]) -> int: + count = 0 + for record in records: + try: + await upsert_buyer(record) + count += 1 + except Exception as exc: + log.error(f"Failed to upsert buyer {record.get('buyer_id')}: {exc}") + return count + + +# ── Price observations ──────────────────────────────────────────────────────── + +async def get_rolling_mean_and_std(market: str, crop: str, last_n: int = 30) -> tuple[float, float]: + """Returns (mean, std) of last_n price observations for outlier rejection.""" + pool = await get_pool() + async with pool.acquire() as conn: + rows = await conn.fetch( + """ + SELECT price_per_kg FROM price_observations + WHERE market = $1 AND crop = $2 + ORDER BY observed_at DESC + LIMIT $3 + """, + market, crop, last_n, + ) + if len(rows) < 2: + return 0.0, 0.0 + prices = [float(r["price_per_kg"]) for r in rows] + mean = sum(prices) / len(prices) + variance = sum((p - mean) ** 2 for p in prices) / len(prices) + return mean, variance ** 0.5 + + +async def insert_price_observation(record: dict) -> bool: + """ + Inserts a price observation. Returns False and logs a warning if the price + is a statistical outlier (> 3 sigma from rolling mean of last 30 observations). + Never raises. + """ + market = record["market"] + crop = record["crop"] + price = float(record["price_per_kg"]) + + sigma_threshold = float(os.getenv("PRICE_ANOMALY_SIGMA_THRESHOLD", "3.0")) + mean, std = await get_rolling_mean_and_std(market, crop) + if std > 0 and abs(price - mean) > sigma_threshold * std: + log.warning( + f"Outlier price rejected: {crop}@{market} price={price} " + f"mean={mean:.0f} std={std:.0f} (>{sigma_threshold}σ)" + ) + return False + + pool = await get_pool() + async with pool.acquire() as conn: + await conn.execute( + """ + INSERT INTO price_observations + (observed_at, market, crop, price_per_kg, currency, source, order_id, quantity_kg) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + """, + record["observed_at"], + market, + crop, + price, + record.get("currency", "UGX"), + record.get("source", "soko_order"), + record.get("order_id"), + record.get("quantity_kg"), + ) + return True + + +async def bulk_insert_price_observations(records: list[dict]) -> int: + count = 0 + for record in records: + try: + inserted = await insert_price_observation(record) + if inserted: + count += 1 + except Exception as exc: + log.error(f"Failed to insert price observation: {exc}") + return count + + +# ── Coverage map ────────────────────────────────────────────────────────────── + +async def get_coverage_status(crop: str, market: str) -> Optional[dict]: + pool = await get_pool() + async with pool.acquire() as conn: + row = await conn.fetchrow( + """ + SELECT crop, market, observation_count, min_observations_needed, + is_model_ready, last_retrain_at + FROM coverage_map + WHERE crop = $1 AND market = $2 + """, + crop, market, + ) + if row is None: + return None + return dict(row) + + +async def get_all_coverage() -> list[dict]: + pool = await get_pool() + async with pool.acquire() as conn: + rows = await conn.fetch( + "SELECT crop, market, observation_count, min_observations_needed, is_model_ready " + "FROM coverage_map ORDER BY crop, market" + ) + return [dict(r) for r in rows] + + +async def mark_retrained(crop: str, market: str) -> None: + pool = await get_pool() + async with pool.acquire() as conn: + await conn.execute( + "UPDATE coverage_map SET last_retrain_at = NOW() WHERE crop = $1 AND market = $2", + crop, market, + ) + + +# ── Coverage gaps ───────────────────────────────────────────────────────────── + +PRIORITY_THRESHOLDS = {"low": (1, 5), "medium": (5, 15), "high": (15, 99999)} + + +def _compute_priority(frequency: int) -> str: + for priority, (lo, hi) in PRIORITY_THRESHOLDS.items(): + if lo <= frequency < hi: + return priority + return "high" + + +async def record_coverage_gap(crop_submitted: str, category_guess: str, reported_by: str) -> dict: + """Upsert a coverage gap record. Returns the updated record including current frequency.""" + pool = await get_pool() + async with pool.acquire() as conn: + existing = await conn.fetchrow( + "SELECT frequency FROM coverage_gaps WHERE crop_submitted = $1", + crop_submitted, + ) + if existing: + new_freq = existing["frequency"] + 1 + priority = _compute_priority(new_freq) + await conn.execute( + """ + UPDATE coverage_gaps + SET frequency = $1, last_reported_at = NOW(), priority = $2 + WHERE crop_submitted = $3 + """, + new_freq, priority, crop_submitted, + ) + return {"crop_submitted": crop_submitted, "frequency": new_freq, "priority": priority} + else: + await conn.execute( + """ + INSERT INTO coverage_gaps + (crop_submitted, category_guess, frequency, first_reported_by, priority) + VALUES ($1, $2, 1, $3, 'low') + """, + crop_submitted, category_guess, reported_by, + ) + return {"crop_submitted": crop_submitted, "frequency": 1, "priority": "low"} + + +async def get_gap_summary() -> list[dict]: + pool = await get_pool() + async with pool.acquire() as conn: + rows = await conn.fetch( + """ + SELECT crop_submitted, category_guess, frequency, priority, status, + first_reported_at, last_reported_at + FROM coverage_gaps + ORDER BY frequency DESC + """ + ) + return [dict(r) for r in rows] diff --git a/services/soko-ml/data-ingestion-service/src/health.py b/services/soko-ml/data-ingestion-service/src/health.py new file mode 100644 index 0000000..6ba4d81 --- /dev/null +++ b/services/soko-ml/data-ingestion-service/src/health.py @@ -0,0 +1,51 @@ +""" +Health checks for all external dependencies of the data-ingestion-service. +""" +import logging +import os + +import asyncpg +import httpx + +log = logging.getLogger(__name__) + +USER_SERVICE_URL = os.getenv("USER_SERVICE_URL", "http://user-service:3003") +ORDER_SERVICE_URL = os.getenv("ORDER_SERVICE_URL", "http://order-service:3002") +LISTING_SERVICE_URL = os.getenv("LISTING_SERVICE_URL", "http://produce-service:3004") +POSTGRES_DSN = os.getenv("POSTGRES_DSN", "postgresql://soko_ml:changeme@soko-ml-db:5432/soko_ml_db") + + +async def check_postgres() -> str: + try: + conn = await asyncpg.connect(POSTGRES_DSN) + await conn.fetchval("SELECT 1") + await conn.close() + return "ok" + except Exception as exc: + log.warning(f"Postgres health check failed: {exc}") + return "unreachable" + + +async def check_service(url: str, name: str) -> str: + try: + async with httpx.AsyncClient(timeout=3.0) as client: + resp = await client.get(f"{url}/health") + return "ok" if resp.status_code == 200 else "degraded" + except Exception: + return "unreachable" + + +async def full_health_check() -> dict: + postgres = await check_postgres() + user = await check_service(USER_SERVICE_URL, "user-service") + order = await check_service(ORDER_SERVICE_URL, "order-service") + listing = await check_service(LISTING_SERVICE_URL, "produce-service") + + overall = "ok" if all(s == "ok" for s in [postgres, user, order, listing]) else "degraded" + return { + "status": overall, + "postgres": postgres, + "user-service": user, + "order-service": order, + "produce-service": listing, + } diff --git a/services/soko-ml/data-ingestion-service/src/main.py b/services/soko-ml/data-ingestion-service/src/main.py new file mode 100644 index 0000000..3a6ccf3 --- /dev/null +++ b/services/soko-ml/data-ingestion-service/src/main.py @@ -0,0 +1,157 @@ +""" +data-ingestion-service — entry point. + +Modes of operation: + Bootstrap — on startup, pulls all profiles and historical orders from backend services. + Only runs when the feature store tables are completely empty. + Streaming — runs continuously, consuming soko.transactions for live price observations. + HTTP API — exposes /health, /bootstrap, /ingest/* endpoints. +""" +import asyncio +import logging +import os +from contextlib import asynccontextmanager + +import structlog +from fastapi import FastAPI, BackgroundTasks, HTTPException +from fastapi.responses import JSONResponse + +from .feature_store import get_pool, close_pool, is_bootstrap_needed, get_all_coverage, get_gap_summary +from .health import full_health_check +from .schemas import BootstrapStatusResponse, IngestOrderEventPayload +from .streams.transaction_stream import TransactionStream + +structlog.configure( + processors=[ + structlog.processors.TimeStamper(fmt="iso"), + structlog.processors.add_log_level, + structlog.processors.JSONRenderer(), + ] +) +log = structlog.get_logger() + +BOOTSTRAP_ON_STARTUP = os.getenv("BOOTSTRAP_ON_STARTUP", "true").lower() == "true" +SERVICE_NAME = os.getenv("SERVICE_NAME", "data-ingestion-service") + +_stream: TransactionStream | None = None +_bootstrap_lock = asyncio.Lock() + + +async def _run_bootstrap() -> dict: + from .bootstrap.auth_bootstrap import bootstrap_farmers, bootstrap_buyers + from .bootstrap.order_bootstrap import bootstrap_orders + from .bootstrap.listing_bootstrap import bootstrap_listings, bootstrap_market_bootstrap + + await bootstrap_market_bootstrap() + farmers = await bootstrap_farmers() + buyers = await bootstrap_buyers() + orders = await bootstrap_orders() + listings = await bootstrap_listings() + return {"farmers": farmers, "buyers": buyers, "orders": orders, "listings": listings} + + +@asynccontextmanager +async def lifespan(app: FastAPI): + global _stream + + # Initialise connection pool + await get_pool() + log.info("postgres_pool_ready") + + # Bootstrap if tables are empty and flag is set + if BOOTSTRAP_ON_STARTUP: + async with _bootstrap_lock: + needed = await is_bootstrap_needed() + if needed: + log.info("bootstrap_starting") + try: + result = await _run_bootstrap() + log.info("bootstrap_complete", **result) + except Exception as exc: + log.error(f"bootstrap_failed: {exc}") + else: + log.info("bootstrap_skipped_tables_not_empty") + + # Start transaction stream consumer + _stream = TransactionStream() + _stream.start() + log.info("transaction_stream_started") + + yield + + if _stream: + _stream.stop() + await close_pool() + log.info("data_ingestion_service_stopped") + + +app = FastAPI(title="Soko Data Ingestion Service", version="2.0.0", lifespan=lifespan) + + +@app.get("/health") +async def health(): + return await full_health_check() + + +@app.post("/bootstrap") +async def trigger_bootstrap(background_tasks: BackgroundTasks): + """ + Manually triggers a full bootstrap regardless of table state. + Used by `make ingest-bootstrap`. + """ + async def _do_bootstrap(): + async with _bootstrap_lock: + log.info("manual_bootstrap_triggered") + result = await _run_bootstrap() + log.info("manual_bootstrap_complete", **result) + + background_tasks.add_task(_do_bootstrap) + return {"message": "Bootstrap triggered — running in background"} + + +@app.get("/bootstrap/status", response_model=BootstrapStatusResponse) +async def bootstrap_status(): + from .feature_store import get_pool + pool = await get_pool() + async with pool.acquire() as conn: + farmers = await conn.fetchval("SELECT COUNT(*) FROM farmer_features") + buyers = await conn.fetchval("SELECT COUNT(*) FROM buyer_features") + orders = await conn.fetchval( + "SELECT COUNT(*) FROM price_observations WHERE source = 'soko_order'" + ) + coverage = await conn.fetchval("SELECT COUNT(*) FROM coverage_map") + + return BootstrapStatusResponse( + farmers_ingested=farmers, + buyers_ingested=buyers, + orders_ingested=orders, + coverage_pairs=coverage, + already_bootstrapped=(farmers > 0 or buyers > 0), + ) + + +@app.post("/ingest/order-event") +async def ingest_order_event(payload: IngestOrderEventPayload): + """ + Accepts a transaction event forwarded by kafka-agent (or called directly). + Processes as a price observation. + """ + from .transformers.price_transformer import transform_transaction_event + from .feature_store import insert_price_observation + + rec = transform_transaction_event(payload.model_dump()) + if rec is None: + return {"status": "skipped", "reason": "not a purchase_completed or zero price"} + + inserted = await insert_price_observation(rec) + return {"status": "inserted" if inserted else "rejected_outlier"} + + +@app.get("/gaps/summary") +async def gap_summary(): + return await get_gap_summary() + + +@app.get("/coverage") +async def coverage(): + return await get_all_coverage() diff --git a/services/soko-ml/data-ingestion-service/src/schemas.py b/services/soko-ml/data-ingestion-service/src/schemas.py new file mode 100644 index 0000000..6a08658 --- /dev/null +++ b/services/soko-ml/data-ingestion-service/src/schemas.py @@ -0,0 +1,97 @@ +from datetime import datetime +from typing import Optional +from pydantic import BaseModel + + +class FarmerProfileIn(BaseModel): + """Shape of a farmer profile as returned by user-service GET /users/farmers.""" + id: str + name: str + initials: str + avatarUrl: Optional[str] = None + district: str + village: Optional[str] = None + verified: bool = False + farmerBio: Optional[str] = None + farmName: Optional[str] = None + specialties: list[str] = [] + memberSince: str = "" + totalListings: int = 0 + totalSales: int = 0 + averageRating: float = 0.0 + totalReviews: int = 0 + responseTime: Optional[str] = None + + +class BuyerProfileIn(BaseModel): + """Shape of a buyer profile as returned by user-service GET /users/buyers.""" + id: str + name: str + initials: str + email: str + phone: str + avatarUrl: Optional[str] = None + district: str + village: Optional[str] = None + role: str + verified: bool = False + memberSince: str = "" + totalOrders: Optional[int] = 0 + totalSpent: Optional[int] = 0 + wishlistCount: Optional[int] = 0 + + +class OrderItemIn(BaseModel): + """Shape of an order item from order-service GET /internal/orders.""" + id: str + product_id: str + product_name: str + farmer_id: str + farmer_name: str + unit: str + category: str + unit_price: float + quantity: float + subtotal: float + + +class OrderIn(BaseModel): + """Shape of a completed order from order-service GET /internal/orders.""" + id: str + buyer_id: str + status: str + delivery_district: str + currency: str = "UGX" + updated_at: str + items: list[OrderItemIn] = [] + + +class IngestAuthEventPayload(BaseModel): + """Forwarded from kafka-agent for user profile events.""" + event_type: str + user_id: str + role: str + data: dict = {} + + +class IngestOrderEventPayload(BaseModel): + """Forwarded from soko.transactions consumer.""" + event_type: str + order_id: str + buyer_id: str + farmer_id: str + crop: str + product_name: str = "" + market: str + quantity_kg: float + price_per_kg_ugx: float + total_ugx: float + timestamp: str = "" + + +class BootstrapStatusResponse(BaseModel): + farmers_ingested: int + buyers_ingested: int + orders_ingested: int + coverage_pairs: int + already_bootstrapped: bool diff --git a/services/soko-ml/data-ingestion-service/src/streams/__init__.py b/services/soko-ml/data-ingestion-service/src/streams/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/services/soko-ml/data-ingestion-service/src/streams/transaction_stream.py b/services/soko-ml/data-ingestion-service/src/streams/transaction_stream.py new file mode 100644 index 0000000..5c2db44 --- /dev/null +++ b/services/soko-ml/data-ingestion-service/src/streams/transaction_stream.py @@ -0,0 +1,143 @@ +""" +Consumes soko.transactions in real time. +Processes purchase_completed events → price observations in the feature store. +Also updates buyer purchase counts to improve recommendation quality. +""" +import json +import logging +import os +import threading + +from confluent_kafka import Consumer, KafkaError, Producer + +from ..transformers.price_transformer import transform_transaction_event +from ..transformers.farmer_transformer import DISTRICT_TO_MARKET, DEFAULT_MARKET + +log = logging.getLogger(__name__) + +BOOTSTRAP_SERVERS = os.getenv("KAFKA_BOOTSTRAP_SERVERS", "kafka:9092") +TRANSACTION_TOPIC = os.getenv("KAFKA_TRANSACTION_TOPIC", "soko.transactions") +ML_EVENTS_TOPIC = os.getenv("KAFKA_ML_EVENTS_TOPIC", "soko.ml.events") +MIN_OBSERVATIONS_FOR_RETRAIN = int(os.getenv("MIN_OBSERVATIONS_FOR_MODEL", "52")) + + +class TransactionStream: + """ + Runs in its own thread. Consumes soko.transactions and writes price observations. + Publishes retrain_requested to soko.ml.events when a pair hits the threshold. + """ + + def __init__(self) -> None: + self._stop_event = threading.Event() + self._consumer = Consumer({ + "bootstrap.servers": BOOTSTRAP_SERVERS, + "group.id": "soko-ml-ingest-transactions", + "auto.offset.reset": "earliest", + "enable.auto.commit": False, + }) + self._consumer.subscribe([TRANSACTION_TOPIC]) + + self._producer = Producer({ + "bootstrap.servers": BOOTSTRAP_SERVERS, + "socket.timeout.ms": 3000, + "message.timeout.ms": 5000, + }) + + def start(self) -> threading.Thread: + t = threading.Thread(target=self._run, daemon=True, name="transaction-stream") + t.start() + return t + + def stop(self) -> None: + self._stop_event.set() + + def _run(self) -> None: + import asyncio + + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + log.info(f"transaction_stream_started topic={TRANSACTION_TOPIC}") + try: + while not self._stop_event.is_set(): + msg = self._consumer.poll(timeout=1.0) + if msg is None: + continue + if msg.error(): + if msg.error().code() != KafkaError._PARTITION_EOF: + log.error(f"transaction_stream_error: {msg.error()}") + continue + + raw = msg.value().decode("utf-8") + try: + event = json.loads(raw) + loop.run_until_complete(self._process(event)) + self._consumer.commit(asynchronous=False) + except Exception as exc: + log.error(f"transaction_processing_error: {exc}") + finally: + self._consumer.close() + loop.close() + log.info("transaction_stream_stopped") + + async def _process(self, event: dict) -> None: + from ..feature_store import insert_price_observation, get_coverage_status, mark_retrained + from datetime import datetime + + rec = transform_transaction_event(event) + if rec is None: + return + + inserted = await insert_price_observation(rec) + if not inserted: + return + + # Check if this pair just crossed the retrain threshold + coverage = await get_coverage_status(rec["crop"], rec["market"]) + if ( + coverage + and coverage["is_model_ready"] + and coverage["last_retrain_at"] is None + ): + await mark_retrained(rec["crop"], rec["market"]) + self._publish_retrain_event(rec["crop"], rec["market"], coverage["observation_count"]) + + # Increment buyer purchase count in feature store + buyer_id = event.get("buyer_id", "") + if buyer_id: + await self._update_buyer_purchases(buyer_id) + + async def _update_buyer_purchases(self, buyer_id: str) -> None: + from ..feature_store import get_pool + pool = await get_pool() + async with pool.acquire() as conn: + await conn.execute( + """ + UPDATE buyer_features + SET total_purchases = total_purchases + 1, + last_active_at = NOW() + WHERE buyer_id = $1 + """, + buyer_id, + ) + + def _publish_retrain_event(self, crop: str, market: str, observation_count: int) -> None: + from datetime import datetime + payload = json.dumps({ + "event_type": "retrain_requested", + "market": market, + "crop": crop, + "reason": f"{observation_count} real transaction observations reached", + "data_source": "soko_order", + "timestamp": datetime.utcnow().isoformat() + "Z", + }).encode() + try: + self._producer.produce( + ML_EVENTS_TOPIC, + key=f"{market}__{crop}".encode(), + value=payload, + ) + self._producer.poll(0) + log.info(f"retrain_requested published: {market}/{crop}") + except Exception as exc: + log.warning(f"Failed to publish retrain event: {exc}") diff --git a/services/soko-ml/data-ingestion-service/src/transformers/__init__.py b/services/soko-ml/data-ingestion-service/src/transformers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/services/soko-ml/data-ingestion-service/src/transformers/buyer_transformer.py b/services/soko-ml/data-ingestion-service/src/transformers/buyer_transformer.py new file mode 100644 index 0000000..f9d74a2 --- /dev/null +++ b/services/soko-ml/data-ingestion-service/src/transformers/buyer_transformer.py @@ -0,0 +1,79 @@ +""" +Converts user-service buyer profile payloads into ML buyer_features records. +""" +from .farmer_transformer import ( + CROP_NAME_NORMALISER, DISTRICT_TO_MARKET, DEFAULT_MARKET, district_to_coords +) + + +def normalise_interest(raw: str) -> list[str]: + """ + Buyer interests are stored as categories (e.g. "Grains", "Vegetables"). + Expand each category to the known ML crops within it. + Falls back to returning the interest as-is if unrecognised. + """ + CATEGORY_CROPS: dict[str, list[str]] = { + "grains": ["maize_grain", "sorghum", "millet", "rice"], + "vegetables": ["tomatoes", "kale", "nakati", "cabbage", "onions", "eggplant"], + "fruits": ["matoke"], + "tubers": ["irish_potatoes", "cassava_chips", "sweet_potatoes", "yams"], + "legumes": ["yellow_beans", "groundnuts", "soybeans"], + "dairy": [], + "livestock": [], + "poultry": [], + "fish": [], + "other": [], + } + key = raw.lower().strip() + if key in CATEGORY_CROPS: + return CATEGORY_CROPS[key] + # Try normalising as a specific crop name + normalised = CROP_NAME_NORMALISER.get(key, key) + return [normalised] if normalised else [] + + +def transform_buyer(profile: dict) -> dict: + """ + Converts a user-service buyer profile response dict into a buyer_features record. + Input shape matches GET /users/buyers (AuthenticatedUser schema). + """ + buyer_id = profile.get("id", "") + name = profile.get("name", "") + district = profile.get("district", "") + + lat, lng = district_to_coords(district) + + raw_interests = profile.get("interests", []) + if isinstance(raw_interests, str): + raw_interests = [i.strip() for i in raw_interests.split(",") if i.strip()] + + # Expand category interests to specific crop keys + preferred_crops: list[str] = [] + for interest in raw_interests: + preferred_crops.extend(normalise_interest(interest)) + preferred_crops = list(dict.fromkeys(preferred_crops)) # deduplicate, preserve order + + primary_market = DISTRICT_TO_MARKET.get(district, DEFAULT_MARKET) + preferred_markets = [primary_market] + + total_purchases = int(profile.get("totalOrders", 0) or 0) + total_spent = float(profile.get("totalSpent", 0) or 0) + avg_spend = round(total_spent / max(total_purchases, 1), 2) + + # payment_reliability — not directly available; default to neutral + payment_reliability = 1.0 + + return { + "buyer_id": buyer_id, + "name": name, + "lat": lat, + "lng": lng, + "district": district, + "preferred_crops": preferred_crops, + "preferred_markets": preferred_markets, + "avg_order_volume_kg": 0.0, + "payment_reliability": payment_reliability, + "avg_spend_per_order": avg_spend, + "total_purchases": total_purchases, + "last_active_at": None, + } diff --git a/services/soko-ml/data-ingestion-service/src/transformers/farmer_transformer.py b/services/soko-ml/data-ingestion-service/src/transformers/farmer_transformer.py new file mode 100644 index 0000000..1328a17 --- /dev/null +++ b/services/soko-ml/data-ingestion-service/src/transformers/farmer_transformer.py @@ -0,0 +1,180 @@ +""" +Converts user-service farmer profile payloads into ML farmer_features records. +""" +from datetime import datetime +from typing import Optional + +# Maps raw crop name variants (from profile specialties) to ML crop keys. +# Unknown names pass through as-is and trigger the coverage gap check downstream. +CROP_NAME_NORMALISER: dict[str, str] = { + "maize": "maize_grain", + "corn": "maize_grain", + "posho": "maize_grain", + "maize grain": "maize_grain", + "maize_grain": "maize_grain", + "beans": "yellow_beans", + "yellow beans": "yellow_beans", + "yellow_beans": "yellow_beans", + "potatoes": "irish_potatoes", + "irish potatoes": "irish_potatoes", + "irish_potatoes": "irish_potatoes", + "cooking banana": "matoke", + "banana": "matoke", + "matoke": "matoke", + "cassava": "cassava_chips", + "cassava chips": "cassava_chips", + "cassava_chips": "cassava_chips", + "millet": "millet", + "sorghum": "sorghum", + "tomato": "tomatoes", + "tomatoes": "tomatoes", + "coffee": "coffee", + "vanilla": "vanilla", + "cotton": "cotton", + "groundnuts": "groundnuts", + "peanuts": "groundnuts", + "soybeans": "soybeans", + "soya beans": "soybeans", + "sunflower": "sunflower", + "rice": "rice", + "sweet potatoes": "sweet_potatoes", + "yams": "yams", + "onions": "onions", + "cabbage": "cabbage", + "kale": "kale", + "nakati": "nakati", + "eggplant": "eggplant", + "aubergine": "eggplant", +} + +# Approximate GPS centroids for Uganda districts. +# Used as best-available location when user profiles have no lat/lng field. +DISTRICT_COORDINATES: dict[str, tuple[float, float]] = { + "Kampala": (0.3476, 32.5825), + "Wakiso": (0.4040, 32.4591), + "Mukono": (0.3536, 32.7552), + "Jinja": (0.4244, 33.2041), + "Mbarara": (-0.6072, 30.6545), + "Gulu": (2.7747, 32.2990), + "Lira": (2.2499, 32.8998), + "Mbale": (1.0824, 34.1754), + "Masaka": (-0.3390, 31.7369), + "Arua": (3.0200, 30.9100), + "Fort Portal": (0.6710, 30.2750), + "Soroti": (1.7150, 33.6110), + "Tororo": (0.6920, 34.1800), + "Kabale": (-1.2490, 29.9900), + "Hoima": (1.4340, 31.3520), + "Kasese": (0.1830, 30.0860), + "Iganga": (0.6100, 33.4720), + "Bushenyi": (-0.5470, 30.1910), + "Mityana": (0.4280, 32.0420), + "Mubende": (0.5770, 31.3700), + "Ntungamo": (-0.8820, 30.2640), + "Rukungiri": (-0.8420, 29.9440), + "Kyenjojo": (0.6210, 30.6330), + "Rakai": (-0.7200, 31.4200), + "Kiboga": (0.9200, 31.7700), +} + +# Approximate mapping from delivery district to the nearest ML market node. +# Used when transforming order data that only carries a district name. +DISTRICT_TO_MARKET: dict[str, str] = { + "Kampala": "Kisenyi_Kampala", + "Wakiso": "Kisenyi_Kampala", + "Mukono": "Kisenyi_Kampala", + "Gulu": "Gulu", + "Mbarara": "Mbarara", + "Bushenyi": "Mbarara", + "Ntungamo": "Mbarara", + "Mbale": "Mbale", + "Tororo": "Mbale", + "Iganga": "Mbale", + "Lira": "Lira", + "Soroti": "Lira", + "Arua": "Gulu", + "Masaka": "Masaka", + "Rakai": "Masaka", + "Jinja": "Kisenyi_Kampala", +} +DEFAULT_MARKET = "Kisenyi_Kampala" + + +def normalise_crop(raw: str) -> str: + return CROP_NAME_NORMALISER.get(raw.lower().strip(), raw.lower().strip()) + + +def district_to_coords(district: str) -> tuple[Optional[float], Optional[float]]: + coords = DISTRICT_COORDINATES.get(district) + if coords: + return coords + # Try case-insensitive lookup + for k, v in DISTRICT_COORDINATES.items(): + if k.lower() == district.lower(): + return v + return None, None + + +def transform_farmer(profile: dict) -> dict: + """ + Converts a user-service FarmerProfile response dict into a farmer_features record. + Input shape matches GET /users/farmers or GET /users/{id} responses. + """ + farmer_id = profile.get("id", "") + name = profile.get("name", "") + district = profile.get("district", "") + + lat, lng = district_to_coords(district) + + raw_specialties = profile.get("specialties", []) + if isinstance(raw_specialties, str): + raw_specialties = [s.strip() for s in raw_specialties.split(",") if s.strip()] + + crops_offered = [normalise_crop(c) for c in raw_specialties if c] + + # Derive which markets this farmer likely serves from their district + primary_market = DISTRICT_TO_MARKET.get(district, DEFAULT_MARKET) + markets_served = list({primary_market}) + + avg_rating = float(profile.get("averageRating", 0.0) or 0.0) + total_sales = int(profile.get("totalSales", 0) or 0) + total_listings = int(profile.get("totalListings", 0) or 0) + + # fulfillment_rate not available from profile API — default to a neutral value + fulfillment_rate = 1.0 + + # response_time from profile is a string like "< 1 hr" — parse to hours + avg_response_time_hrs = _parse_response_time(profile.get("responseTime")) + + return { + "farmer_id": farmer_id, + "name": name, + "lat": lat, + "lng": lng, + "district": district, + "crops_offered": crops_offered, + "markets_served": markets_served, + "avg_rating": avg_rating, + "fulfillment_rate": fulfillment_rate, + "avg_response_time_hrs": avg_response_time_hrs, + "total_orders_completed": total_sales, + "total_orders_cancelled": 0, + "total_listings": total_listings, + "last_active_at": None, + } + + +def _parse_response_time(value: Optional[str]) -> float: + """Parse a response time string like '< 1 hr', '2 hrs', '24 hrs' into float hours.""" + if not value: + return 24.0 + v = value.lower().replace("<", "").replace(">", "").strip() + try: + # Extract leading number + import re + match = re.search(r"(\d+\.?\d*)", v) + if match: + return float(match.group(1)) + except Exception: + pass + return 24.0 diff --git a/services/soko-ml/data-ingestion-service/src/transformers/price_transformer.py b/services/soko-ml/data-ingestion-service/src/transformers/price_transformer.py new file mode 100644 index 0000000..0d043f8 --- /dev/null +++ b/services/soko-ml/data-ingestion-service/src/transformers/price_transformer.py @@ -0,0 +1,135 @@ +""" +Converts order-service order payloads into ML price_observation records. +""" +from datetime import datetime, date +from typing import Optional + +from .farmer_transformer import CROP_NAME_NORMALISER, DISTRICT_TO_MARKET + + +def normalise_crop_from_order(product_name: str, category: str) -> str: + """ + Determine the ML crop key from an order item. + Priority: product_name (specific) → category (broad fallback). + """ + if product_name: + key = product_name.lower().strip() + if key in CROP_NAME_NORMALISER: + return CROP_NAME_NORMALISER[key] + for word in key.split(): + if word in CROP_NAME_NORMALISER: + return CROP_NAME_NORMALISER[word] + + cat_map = { + "grains": "maize_grain", + "vegetables": "tomatoes", + "fruits": "matoke", + "tubers": "irish_potatoes", + "legumes": "yellow_beans", + "herbs": None, + "dairy": None, + "poultry": None, + "livestock": None, + "fish": None, + "other": None, + } + cat_key = category.lower().strip() + crop = cat_map.get(cat_key) + if crop: + return crop + + return CROP_NAME_NORMALISER.get(product_name.lower().strip(), product_name.lower().strip() or "unknown") + + +def normalise_market(district: str) -> Optional[str]: + """ + Maps a delivery district string to an ML market node ID. + Returns None for districts not in DISTRICT_TO_MARKET — callers should skip + such events rather than mis-attributing them to a default market. + """ + return DISTRICT_TO_MARKET.get(district) or DISTRICT_TO_MARKET.get(district.title()) + + +def transform_transaction_event(event: dict) -> Optional[dict]: + """ + Converts a soko.transactions Kafka event (purchase_completed) into a + price_observations record. Returns None if the event should be skipped. + + The order-service publishes price_per_kg_ugx (UGX per kg) directly. + All monetary values stored in UGX — never USD. + """ + if event.get("event_type") != "purchase_completed": + return None + + price_per_kg = float(event.get("price_per_kg_ugx", 0)) + if price_per_kg <= 0: + return None + + quantity_kg = float(event.get("quantity_kg", 0)) + product_name = event.get("product_name", "") + category = event.get("crop", "") # soko.transactions uses 'crop' for category + + crop = normalise_crop_from_order(product_name, category) + market = normalise_market(event.get("market", "")) + + if not crop or crop == "unknown": + return None + if not market: + return None + + raw_ts = event.get("timestamp", "") + try: + observed_at = datetime.fromisoformat(raw_ts.replace("Z", "+00:00")).date() + except (ValueError, AttributeError): + observed_at = date.today() + + return { + "observed_at": observed_at, + "market": market, + "crop": crop, + "price_per_kg": price_per_kg, + "currency": "UGX", + "source": "soko_order", + "order_id": event.get("order_id"), + "quantity_kg": quantity_kg if quantity_kg > 0 else None, + } + + +def transform_order_item( + order_id: str, + item: dict, + delivery_district: str, + completed_at: str, +) -> Optional[dict]: + """ + Converts an order item from the order bootstrap API into a price_observations record. + Used during bootstrap, not streaming. + """ + price_per_kg = float(item.get("unit_price", 0)) + if price_per_kg <= 0: + return None + + product_name = item.get("product_name", "") + category = item.get("category", "") + + crop = normalise_crop_from_order(product_name, category) + market = normalise_market(delivery_district) + + if crop == "unknown" or not crop: + return None + + try: + observed_at = datetime.fromisoformat(completed_at.replace("Z", "+00:00")).date() + except (ValueError, AttributeError): + observed_at = date.today() + + return { + "observed_at": observed_at, + "market": market, + "crop": crop, + "price_per_kg": price_per_kg, + "currency": "UGX", + "source": "soko_order", + "order_id": order_id, + "quantity_kg": float(item.get("quantity", 0)) or None, + } diff --git a/services/soko-ml/data-ingestion-service/tests/__init__.py b/services/soko-ml/data-ingestion-service/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/services/soko-ml/data-ingestion-service/tests/test_feature_store.py b/services/soko-ml/data-ingestion-service/tests/test_feature_store.py new file mode 100644 index 0000000..17cd0f5 --- /dev/null +++ b/services/soko-ml/data-ingestion-service/tests/test_feature_store.py @@ -0,0 +1,208 @@ +""" +Integration tests for the data-ingestion-service feature store. +Requires a live Postgres instance (set POSTGRES_DSN in environment). +Skipped automatically when the DB is unreachable. +""" +import os +import pytest +import pytest_asyncio +import asyncpg + +pytestmark = pytest.mark.asyncio + +DSN = os.getenv( + "POSTGRES_DSN", + "postgresql://soko_ml:changeme@localhost:5432/soko_ml_db", +) + + +async def _db_reachable() -> bool: + try: + conn = await asyncpg.connect(DSN, timeout=3) + await conn.close() + return True + except Exception: + return False + + +@pytest.fixture(scope="module") +def anyio_backend(): + return "asyncio" + + +@pytest_asyncio.fixture(scope="module") +async def pool(): + if not await _db_reachable(): + pytest.skip("Postgres unreachable — skipping feature store integration tests") + p = await asyncpg.create_pool(DSN, min_size=1, max_size=2) + yield p + await p.close() + + +# ── Farmer upsert ───────────────────────────────────────────────────────────── + +@pytest.mark.asyncio +async def test_upsert_farmer_inserts_new_row(pool): + from src.feature_store import upsert_farmer + + farmer = { + "farmer_id": "TEST_F001", + "name": "Test Farmer", + "district": "Kampala", + "lat": 0.3476, + "lng": 32.5825, + "crops_offered": ["maize_grain", "yellow_beans"], + "avg_rating": 4.0, + "fulfillment_rate": 0.95, + "response_time_hours": 8.0, + "total_sales": 0, + } + async with pool.acquire() as conn: + await conn.execute("DELETE FROM farmer_features WHERE farmer_id = $1", farmer["farmer_id"]) + + await upsert_farmer(farmer) + + async with pool.acquire() as conn: + row = await conn.fetchrow( + "SELECT * FROM farmer_features WHERE farmer_id = $1", farmer["farmer_id"] + ) + assert row is not None + assert row["name"] == "Test Farmer" + assert "maize_grain" in row["crops_offered"] + + async with pool.acquire() as conn: + await conn.execute("DELETE FROM farmer_features WHERE farmer_id = $1", farmer["farmer_id"]) + + +@pytest.mark.asyncio +async def test_upsert_farmer_updates_existing_row(pool): + from src.feature_store import upsert_farmer + + farmer = { + "farmer_id": "TEST_F002", + "name": "Initial Name", + "district": "Gulu", + "lat": 2.7747, + "lng": 32.2990, + "crops_offered": ["sorghum"], + "avg_rating": 3.5, + "fulfillment_rate": 0.80, + "response_time_hours": 24.0, + "total_sales": 0, + } + async with pool.acquire() as conn: + await conn.execute("DELETE FROM farmer_features WHERE farmer_id = $1", farmer["farmer_id"]) + + await upsert_farmer(farmer) + farmer["name"] = "Updated Name" + farmer["avg_rating"] = 4.8 + await upsert_farmer(farmer) + + async with pool.acquire() as conn: + row = await conn.fetchrow( + "SELECT * FROM farmer_features WHERE farmer_id = $1", farmer["farmer_id"] + ) + assert row["name"] == "Updated Name" + assert float(row["avg_rating"]) == pytest.approx(4.8, abs=0.01) + + async with pool.acquire() as conn: + await conn.execute("DELETE FROM farmer_features WHERE farmer_id = $1", farmer["farmer_id"]) + + +# ── Outlier rejection ───────────────────────────────────────────────────────── + +@pytest.mark.asyncio +async def test_insert_price_observation_accepts_normal_price(pool): + from src.feature_store import insert_price_observation + + obs = { + "order_id": "TEST-OBS-001", + "crop": "maize_grain", + "market": "Kisenyi_Kampala", + "price_ugx_per_kg": 1400.0, + "quantity_kg": 100.0, + "observed_at": "2026-05-14T10:00:00+00:00", + "source": "soko_order", + } + async with pool.acquire() as conn: + await conn.execute( + "DELETE FROM price_observations WHERE order_id = $1", obs["order_id"] + ) + + accepted = await insert_price_observation(obs) + assert accepted is True + + async with pool.acquire() as conn: + row = await conn.fetchrow( + "SELECT * FROM price_observations WHERE order_id = $1", obs["order_id"] + ) + assert row is not None + assert float(row["price_ugx_per_kg"]) == pytest.approx(1400.0) + + async with pool.acquire() as conn: + await conn.execute( + "DELETE FROM price_observations WHERE order_id = $1", obs["order_id"] + ) + + +@pytest.mark.asyncio +async def test_insert_price_observation_rejects_outlier(pool): + """ + Seeds enough observations to establish a mean, then tries a 10σ outlier. + """ + from src.feature_store import insert_price_observation, bulk_insert_price_observations + + seed_obs = [ + { + "order_id": f"SEED-OBS-{i:03d}", + "crop": "sorghum", + "market": "Gulu", + "price_ugx_per_kg": 900.0 + i, + "quantity_kg": 50.0, + "observed_at": f"2026-04-{i+1:02d}T10:00:00+00:00", + "source": "soko_order", + } + for i in range(30) + ] + async with pool.acquire() as conn: + await conn.execute( + "DELETE FROM price_observations WHERE order_id LIKE 'SEED-OBS-%'" + ) + + await bulk_insert_price_observations(seed_obs) + + outlier = { + "order_id": "OUTLIER-OBS-001", + "crop": "sorghum", + "market": "Gulu", + "price_ugx_per_kg": 999999.0, # obviously anomalous + "quantity_kg": 50.0, + "observed_at": "2026-05-14T11:00:00+00:00", + "source": "soko_order", + } + accepted = await insert_price_observation(outlier) + assert accepted is False + + async with pool.acquire() as conn: + await conn.execute( + "DELETE FROM price_observations WHERE order_id LIKE 'SEED-OBS-%' OR order_id = 'OUTLIER-OBS-001'" + ) + + +# ── Coverage map ────────────────────────────────────────────────────────────── + +@pytest.mark.asyncio +async def test_get_coverage_status_returns_dict(pool): + from src.feature_store import get_coverage_status + + status = await get_coverage_status("maize_grain", "Kisenyi_Kampala") + # May return None if the seed data is not yet present; just assert no crash + assert status is None or isinstance(status, dict) + + +@pytest.mark.asyncio +async def test_get_all_coverage_returns_list(pool): + from src.feature_store import get_all_coverage + + coverage = await get_all_coverage() + assert isinstance(coverage, list) diff --git a/services/soko-ml/data-ingestion-service/tests/test_transformers.py b/services/soko-ml/data-ingestion-service/tests/test_transformers.py new file mode 100644 index 0000000..c886b40 --- /dev/null +++ b/services/soko-ml/data-ingestion-service/tests/test_transformers.py @@ -0,0 +1,252 @@ +""" +Unit tests for the data-ingestion-service transformer layer. +No DB, no Kafka, no HTTP — pure function tests. +""" +import pytest + +from src.transformers.farmer_transformer import ( + CROP_NAME_NORMALISER, + DISTRICT_COORDINATES, + DISTRICT_TO_MARKET, + normalise_crop, + district_to_coords, + transform_farmer, +) +from src.transformers.buyer_transformer import normalise_interest, transform_buyer +from src.transformers.price_transformer import ( + normalise_crop_from_order, + normalise_market, + transform_transaction_event, +) + + +# ── CROP_NAME_NORMALISER ────────────────────────────────────────────────────── + +class TestNormaliseCrop: + def test_canonical_keys_pass_through(self): + assert normalise_crop("maize_grain") == "maize_grain" + assert normalise_crop("yellow_beans") == "yellow_beans" + + def test_common_aliases(self): + assert normalise_crop("maize") == "maize_grain" + assert normalise_crop("corn") == "maize_grain" + assert normalise_crop("posho") == "maize_grain" + assert normalise_crop("beans") == "yellow_beans" + assert normalise_crop("potatoes") == "irish_potatoes" + assert normalise_crop("banana") == "matoke" + assert normalise_crop("cassava") == "cassava_chips" + assert normalise_crop("peanuts") == "groundnuts" + assert normalise_crop("soya beans") == "soybeans" + + def test_case_insensitive(self): + assert normalise_crop("Maize") == "maize_grain" + assert normalise_crop("COFFEE") == "coffee" + assert normalise_crop("Tomatoes") == "tomatoes" + + def test_unknown_crop_passes_through(self): + assert normalise_crop("moringa") == "moringa" + assert normalise_crop("avocado") == "avocado" + + def test_whitespace_stripped(self): + assert normalise_crop(" maize ") == "maize_grain" + + def test_all_normaliser_values_are_lowercase_underscore(self): + for val in CROP_NAME_NORMALISER.values(): + assert " " not in val, f"Value '{val}' contains space" + assert val == val.lower(), f"Value '{val}' is not lowercase" + + +# ── DISTRICT COORDINATES ────────────────────────────────────────────────────── + +class TestDistrictCoords: + def test_known_district(self): + lat, lng = district_to_coords("Kampala") + assert abs(lat - 0.3476) < 0.001 + assert abs(lng - 32.5825) < 0.001 + + def test_unknown_district_returns_none(self): + lat, lng = district_to_coords("Atlantis") + assert lat is None + assert lng is None + + def test_all_coordinates_plausible_for_uganda(self): + for district, (lat, lng) in DISTRICT_COORDINATES.items(): + assert -2.0 <= lat <= 4.5, f"{district} lat {lat} out of Uganda range" + assert 29.5 <= lng <= 35.0, f"{district} lng {lng} out of Uganda range" + + +# ── DISTRICT_TO_MARKET ──────────────────────────────────────────────────────── + +class TestDistrictToMarket: + def test_kampala_maps_to_kisenyi(self): + assert DISTRICT_TO_MARKET["Kampala"] == "Kisenyi_Kampala" + + def test_gulu_maps_to_gulu(self): + assert DISTRICT_TO_MARKET["Gulu"] == "Gulu" + + def test_all_market_ids_are_non_empty_strings(self): + for district, market_id in DISTRICT_TO_MARKET.items(): + assert isinstance(market_id, str) and market_id, f"{district} has empty market" + + +# ── TRANSFORM_FARMER ────────────────────────────────────────────────────────── + +class TestTransformFarmer: + BASE_PAYLOAD = { + "id": "F001", + "name": "Alice Nakato", + "district": "Kampala", + "village": "Mulago", + "specialties": "maize,beans,tomatoes", + "average_rating": 4.2, + "response_time_hours": 6, + "is_verified": True, + } + + def test_basic_transform(self): + result = transform_farmer(self.BASE_PAYLOAD) + assert result["farmer_id"] == "F001" + assert result["name"] == "Alice Nakato" + assert result["district"] == "Kampala" + assert "maize_grain" in result["crops_offered"] + assert "yellow_beans" in result["crops_offered"] + assert "tomatoes" in result["crops_offered"] + + def test_lat_lng_from_district_centroid(self): + result = transform_farmer(self.BASE_PAYLOAD) + expected_lat, expected_lng = DISTRICT_COORDINATES["Kampala"] + assert abs(result["lat"] - expected_lat) < 0.001 + assert abs(result["lng"] - expected_lng) < 0.001 + + def test_rating_clamped_to_zero_when_missing(self): + payload = {**self.BASE_PAYLOAD, "average_rating": None} + result = transform_farmer(payload) + assert result["avg_rating"] == 0.0 + + def test_response_time_string_parsed(self): + payload = {**self.BASE_PAYLOAD, "responseTime": "12h"} + result = transform_farmer(payload) + assert result["avg_response_time_hrs"] == 12.0 + + +# ── BUYER TRANSFORMER ───────────────────────────────────────────────────────── + +class TestBuyerTransformer: + def test_normalise_grains_category(self): + crops = normalise_interest("Grains") + assert "maize_grain" in crops + assert "sorghum" in crops + + def test_normalise_vegetables(self): + crops = normalise_interest("Vegetables") + assert "tomatoes" in crops or "kale" in crops + + def test_unknown_interest_passes_through(self): + crops = normalise_interest("Moringa") + assert "moringa" in crops + + def test_transform_buyer_basic(self): + payload = { + "id": "B001", + "name": "John Ssemakula", + "district": "Wakiso", + "interests": "Grains,Legumes", + } + result = transform_buyer(payload) + assert result["buyer_id"] == "B001" + assert "maize_grain" in result["preferred_crops"] + assert "yellow_beans" in result["preferred_crops"] + + +# ── PRICE TRANSFORMER ───────────────────────────────────────────────────────── + +class TestPriceTransformer: + def test_normalise_crop_from_product_name(self): + crop = normalise_crop_from_order(product_name="Maize (Dry)", category="Grains") + assert crop == "maize_grain" + + def test_falls_back_to_category_mapping(self): + crop = normalise_crop_from_order(product_name="", category="Grains") + assert crop is not None + + def test_normalise_market_kampala(self): + market = normalise_market("Kampala") + assert market == "Kisenyi_Kampala" + + def test_normalise_market_unknown_returns_none(self): + # Unknown districts return None — callers skip them rather than + # mis-attributing prices to a default market. + market = normalise_market("Atlantis") + assert market is None + + def test_normalise_market_case_insensitive_via_title(self): + assert normalise_market("gulu") == "Gulu" + + def test_transform_transaction_event_valid(self): + # Event shape mirrors what order-service publishes to soko.transactions. + # price_per_kg_ugx is the UGX-per-kg price — never USD. + event = { + "event_type": "purchase_completed", + "order_id": "ORD-001", + "product_name": "Beans", + "crop": "Legumes", + "market": "Gulu", + "price_per_kg_ugx": 1500.0, + "quantity_kg": 100, + "farmer_id": "F001", + "buyer_id": "B001", + "timestamp": "2026-05-14T10:00:00Z", + } + result = transform_transaction_event(event) + assert result is not None + assert result["crop"] == "yellow_beans" + assert result["market"] == "Gulu" + assert float(result["price_per_kg"]) == 1500.0 + assert result["currency"] == "UGX" + + def test_transform_transaction_event_skips_unmappable_market(self): + # "Atlantis" is not in DISTRICT_TO_MARKET → normalise_market returns None → skipped + event = { + "event_type": "purchase_completed", + "order_id": "ORD-002", + "product_name": "Maize", + "crop": "Grains", + "market": "Atlantis", + "price_per_kg_ugx": 1300.0, + "quantity_kg": 50, + "farmer_id": "F001", + "buyer_id": "B001", + "timestamp": "2026-05-14T10:00:00Z", + } + result = transform_transaction_event(event) + assert result is None + + def test_transform_event_zero_price_returns_none(self): + event = { + "event_type": "purchase_completed", + "order_id": "ORD-003", + "product_name": "Maize", + "crop": "Grains", + "market": "Kampala", + "price_per_kg_ugx": 0, + "quantity_kg": 50, + "farmer_id": "F001", + "buyer_id": "B001", + "timestamp": "2026-05-14T10:00:00Z", + } + result = transform_transaction_event(event) + assert result is None + + def test_transform_event_non_purchase_returns_none(self): + event = { + "event_type": "order_cancelled", + "order_id": "ORD-004", + "product_name": "Maize", + "crop": "Grains", + "market": "Kampala", + "price_per_kg_ugx": 1300.0, + "quantity_kg": 50, + "timestamp": "2026-05-14T10:00:00Z", + } + result = transform_transaction_event(event) + assert result is None diff --git a/services/soko-ml/db/schema.sql b/services/soko-ml/db/schema.sql new file mode 100644 index 0000000..b0ef066 --- /dev/null +++ b/services/soko-ml/db/schema.sql @@ -0,0 +1,145 @@ +-- Soko ML Feature Store — PostgreSQL 16 +-- Owned exclusively by the ML layer. No backend service has direct access. + +-- ── Price observations — feeds Prophet model training ───────────────────────── +CREATE TABLE IF NOT EXISTS price_observations ( + id SERIAL PRIMARY KEY, + observed_at DATE NOT NULL, + market VARCHAR(50) NOT NULL, + crop VARCHAR(50) NOT NULL, + price_per_kg NUMERIC(10, 2) NOT NULL, + currency CHAR(3) DEFAULT 'UGX', + source VARCHAR(20) NOT NULL, -- 'soko_order' | 'farmgain_seed' + order_id VARCHAR(100), -- NULL for seed data + quantity_kg NUMERIC(10, 2), + created_at TIMESTAMP DEFAULT NOW() +); +CREATE INDEX IF NOT EXISTS idx_price_market_crop + ON price_observations(market, crop, observed_at); + +-- ── Farmer features — feeds recommendation-service ─────────────────────────── +CREATE TABLE IF NOT EXISTS farmer_features ( + farmer_id VARCHAR(100) PRIMARY KEY, + name VARCHAR(200), + lat NUMERIC(10, 7), + lng NUMERIC(10, 7), + district VARCHAR(100), + crops_offered TEXT[], + markets_served TEXT[], + avg_rating NUMERIC(3, 2) DEFAULT 0.0, + fulfillment_rate NUMERIC(5, 4) DEFAULT 1.0, + avg_response_time_hrs NUMERIC(6, 2) DEFAULT 24.0, + price_competitiveness NUMERIC(5, 4) DEFAULT 0.5, + repeat_buyer_rate NUMERIC(5, 4) DEFAULT 0.0, + total_orders_completed INTEGER DEFAULT 0, + total_orders_cancelled INTEGER DEFAULT 0, + total_listings INTEGER DEFAULT 0, + last_active_at TIMESTAMP, + synced_at TIMESTAMP DEFAULT NOW() +); + +-- ── Buyer features — feeds recommendation-service ──────────────────────────── +CREATE TABLE IF NOT EXISTS buyer_features ( + buyer_id VARCHAR(100) PRIMARY KEY, + name VARCHAR(200), + lat NUMERIC(10, 7), + lng NUMERIC(10, 7), + district VARCHAR(100), + preferred_crops TEXT[], + preferred_markets TEXT[], + avg_order_volume_kg NUMERIC(10, 2) DEFAULT 0.0, + payment_reliability NUMERIC(5, 4) DEFAULT 1.0, + purchase_frequency_days NUMERIC(6, 2) DEFAULT 30.0, + avg_spend_per_order NUMERIC(12, 2) DEFAULT 0.0, + total_purchases INTEGER DEFAULT 0, + last_active_at TIMESTAMP, + synced_at TIMESTAMP DEFAULT NOW() +); + +-- ── Market registry — static seed populated at init time ───────────────────── +-- No market-service exists; this is the authoritative source. +CREATE TABLE IF NOT EXISTS market_registry ( + market_id VARCHAR(50) PRIMARY KEY, + name VARCHAR(200) NOT NULL, + lat NUMERIC(10, 7) NOT NULL, + lng NUMERIC(10, 7) NOT NULL, + district VARCHAR(100), + active BOOLEAN DEFAULT TRUE +); + +INSERT INTO market_registry (market_id, name, lat, lng, district) VALUES + ('Kisenyi_Kampala', 'Kisenyi Market, Kampala', 0.3136, 32.5811, 'Kampala'), + ('Gulu', 'Gulu Main Market', 2.7747, 32.2990, 'Gulu'), + ('Mbarara', 'Mbarara Market', -0.6072, 30.6545, 'Mbarara'), + ('Mbale', 'Mbale Central Market', 1.0824, 34.1754, 'Mbale'), + ('Lira', 'Lira Market', 2.2499, 32.8998, 'Lira'), + ('Masaka', 'Masaka Market', -0.3390, 31.7369, 'Masaka') +ON CONFLICT (market_id) DO NOTHING; + +-- ── Coverage map — tracks which crop-market pairs have enough data ───────────── +CREATE TABLE IF NOT EXISTS coverage_map ( + crop VARCHAR(50) NOT NULL, + market VARCHAR(50) NOT NULL, + observation_count INTEGER DEFAULT 0, + min_observations_needed INTEGER DEFAULT 52, + is_model_ready BOOLEAN DEFAULT FALSE, + last_retrain_at TIMESTAMP, + PRIMARY KEY (crop, market) +); + +-- Seed coverage_map with all known pairs at zero observations +INSERT INTO coverage_map (crop, market) VALUES + ('maize_grain', 'Kisenyi_Kampala'), ('maize_grain', 'Gulu'), + ('maize_grain', 'Mbarara'), ('maize_grain', 'Mbale'), + ('maize_grain', 'Lira'), ('maize_grain', 'Masaka'), + ('yellow_beans', 'Kisenyi_Kampala'), ('yellow_beans', 'Gulu'), + ('yellow_beans', 'Mbarara'), ('yellow_beans', 'Mbale'), + ('yellow_beans', 'Lira'), ('yellow_beans', 'Masaka'), + ('irish_potatoes','Kisenyi_Kampala'), ('irish_potatoes','Gulu'), + ('irish_potatoes','Mbarara'), ('irish_potatoes','Mbale'), + ('irish_potatoes','Lira'), ('irish_potatoes','Masaka'), + ('tomatoes', 'Kisenyi_Kampala'), ('tomatoes', 'Gulu'), + ('tomatoes', 'Mbarara'), ('tomatoes', 'Mbale'), + ('tomatoes', 'Lira'), ('tomatoes', 'Masaka'), + ('matoke', 'Kisenyi_Kampala'), ('matoke', 'Gulu'), + ('matoke', 'Mbarara'), ('matoke', 'Mbale'), + ('matoke', 'Lira'), ('matoke', 'Masaka'), + ('cassava_chips', 'Kisenyi_Kampala'), ('cassava_chips', 'Gulu'), + ('cassava_chips', 'Mbarara'), ('cassava_chips', 'Mbale'), + ('cassava_chips', 'Lira'), ('cassava_chips', 'Masaka'), + ('sorghum', 'Kisenyi_Kampala'), ('sorghum', 'Gulu'), + ('sorghum', 'Mbarara'), ('sorghum', 'Mbale'), + ('sorghum', 'Lira'), ('sorghum', 'Masaka'), + ('millet', 'Kisenyi_Kampala'), ('millet', 'Gulu'), + ('millet', 'Mbarara'), ('millet', 'Mbale'), + ('millet', 'Lira'), ('millet', 'Masaka') +ON CONFLICT (crop, market) DO NOTHING; + +-- ── Coverage gaps — tracks unrecognised crops submitted by farmers ──────────── +CREATE TABLE IF NOT EXISTS coverage_gaps ( + crop_submitted VARCHAR(100) PRIMARY KEY, + category_guess VARCHAR(50), + frequency INTEGER DEFAULT 1, + first_reported_by VARCHAR(100), + first_reported_at TIMESTAMP DEFAULT NOW(), + last_reported_at TIMESTAMP DEFAULT NOW(), + status VARCHAR(20) DEFAULT 'pending_review', + priority VARCHAR(10) DEFAULT 'low' +); + +-- ── Function: auto-update coverage_map after price observation insert ───────── +CREATE OR REPLACE FUNCTION update_coverage_map() +RETURNS TRIGGER AS $$ +BEGIN + INSERT INTO coverage_map (crop, market, observation_count) + VALUES (NEW.crop, NEW.market, 1) + ON CONFLICT (crop, market) DO UPDATE + SET observation_count = coverage_map.observation_count + 1, + is_model_ready = (coverage_map.observation_count + 1 >= coverage_map.min_observations_needed); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE TRIGGER trg_update_coverage +AFTER INSERT ON price_observations +FOR EACH ROW EXECUTE FUNCTION update_coverage_map(); diff --git a/services/soko-ml/docker-compose.yml b/services/soko-ml/docker-compose.yml index 0d5c80d..ba4b36a 100644 --- a/services/soko-ml/docker-compose.yml +++ b/services/soko-ml/docker-compose.yml @@ -8,6 +8,7 @@ volumes: redis-data: kafka-data: zookeeper-data: + soko_ml_db_data: services: # ── Infrastructure ──────────────────────────────────────────────────────────── @@ -82,10 +83,49 @@ services: kafka-topics --bootstrap-server kafka:9092 --create --if-not-exists --topic soko.price.results --partitions 3 --replication-factor 1 --config retention.ms=86400000 kafka-topics --bootstrap-server kafka:9092 --create --if-not-exists --topic soko.ml.events --partitions 2 --replication-factor 1 --config retention.ms=1209600000 kafka-topics --bootstrap-server kafka:9092 --create --if-not-exists --topic soko.dlq --partitions 2 --replication-factor 1 --config retention.ms=2592000000 + kafka-topics --bootstrap-server kafka:9092 --create --if-not-exists --topic soko.gaps --partitions 2 --replication-factor 1 --config retention.ms=2592000000 echo 'All Kafka topics created.' " restart: on-failure + # ── ML Feature Store (Postgres) ─────────────────────────────────────────────── + + soko-ml-db: + image: postgres:16-alpine + container_name: soko-ml-db + hostname: soko-ml-db + networks: + - soko-ml-network + environment: + POSTGRES_DB: soko_ml_db + POSTGRES_USER: ${POSTGRES_USER:-soko_ml} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-changeme} + volumes: + - soko_ml_db_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-soko_ml} -d soko_ml_db"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 20s + restart: unless-stopped + + db-init: + image: postgres:16-alpine + container_name: soko-ml-db-init + networks: + - soko-ml-network + depends_on: + soko-ml-db: + condition: service_healthy + environment: + PGPASSWORD: ${POSTGRES_PASSWORD:-changeme} + command: > + sh -c "psql -h soko-ml-db -U ${POSTGRES_USER:-soko_ml} -d soko_ml_db -f /schema/schema.sql && echo 'Schema applied.'" + volumes: + - ./db/schema.sql:/schema/schema.sql:ro + restart: on-failure + redis: image: redis:7-alpine hostname: redis @@ -197,6 +237,8 @@ services: environment: - PRICE_SERVICE_URL=http://price-prediction-service:8001 - REC_SERVICE_URL=http://recommendation-service:8002 + - LOCATION_SERVICE_URL=http://location-service:8003 + - INGEST_SERVICE_URL=http://data-ingestion-service:8004 - LOG_LEVEL=${LOG_LEVEL:-INFO} - SERVICE_NAME=ml-gateway-service depends_on: @@ -204,6 +246,10 @@ services: condition: service_healthy recommendation-service: condition: service_healthy + location-service: + condition: service_healthy + data-ingestion-service: + condition: service_healthy healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8000/health"] interval: 15s @@ -211,6 +257,84 @@ services: retries: 5 restart: unless-stopped + data-ingestion-service: + build: + context: ./data-ingestion-service + dockerfile: Dockerfile + container_name: soko-ml-ingest + hostname: data-ingestion-service + networks: + soko-ml-network: {} + soko-ml-bridge: + aliases: + - data-ingestion-service + ports: + - "${INGEST_SERVICE_PORT:-8096}:8004" + environment: + - POSTGRES_DSN=postgresql://${POSTGRES_USER:-soko_ml}:${POSTGRES_PASSWORD:-changeme}@soko-ml-db:5432/soko_ml_db + - KAFKA_BOOTSTRAP_SERVERS=${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} + - KAFKA_TRANSACTION_TOPIC=${KAFKA_TRANSACTION_TOPIC:-soko.transactions} + - KAFKA_ML_EVENTS_TOPIC=${KAFKA_ML_EVENTS_TOPIC:-soko.ml.events} + - KAFKA_GAPS_TOPIC=${KAFKA_GAPS_TOPIC:-soko.gaps} + - USER_SERVICE_URL=${USER_SERVICE_URL:-http://user-service:8002} + - ORDER_SERVICE_URL=${ORDER_SERVICE_URL:-http://order-service:8003} + - PRODUCE_SERVICE_URL=${PRODUCE_SERVICE_URL:-http://produce-service:8004} + - INTERNAL_API_KEY=${INTERNAL_API_KEY:-internal-secret} + - BOOTSTRAP_ON_STARTUP=${BOOTSTRAP_ON_STARTUP:-true} + - MIN_OBSERVATIONS_FOR_MODEL=${MIN_OBSERVATIONS_FOR_MODEL:-30} + - PRICE_ANOMALY_SIGMA_THRESHOLD=${PRICE_ANOMALY_SIGMA_THRESHOLD:-3.0} + - LOG_LEVEL=${LOG_LEVEL:-INFO} + - SERVICE_NAME=data-ingestion-service + depends_on: + soko-ml-db: + condition: service_healthy + kafka: + condition: service_healthy + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8004/health"] + interval: 15s + timeout: 5s + retries: 5 + start_period: 30s + restart: unless-stopped + + location-service: + build: + context: ./location-service + dockerfile: Dockerfile + container_name: soko-ml-location + hostname: location-service + networks: + - soko-ml-network + ports: + - "${LOCATION_SERVICE_PORT:-8097}:8003" + environment: + - POSTGRES_DSN=postgresql://${POSTGRES_USER:-soko_ml}:${POSTGRES_PASSWORD:-changeme}@soko-ml-db:5432/soko_ml_db + - REDIS_HOST=${REDIS_HOST:-redis} + - REDIS_PORT=${REDIS_PORT:-6379} + - REDIS_DB=${REDIS_DB:-0} + - REDIS_PASSWORD=${REDIS_PASSWORD:-} + - PRICE_SERVICE_URL=${PRICE_SERVICE_URL:-http://ml-gateway-service:8000} + - GOOGLE_MAPS_API_KEY=${GOOGLE_MAPS_API_KEY:-} + - MAPS_DISTANCE_CACHE_TTL_SECONDS=${MAPS_DISTANCE_CACHE_TTL_SECONDS:-2592000} + - GEO_FILTER_RELAX_FACTOR=${GEO_FILTER_RELAX_FACTOR:-1.5} + - KAFKA_BOOTSTRAP_SERVERS=${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} + - KAFKA_GAPS_TOPIC=${KAFKA_GAPS_TOPIC:-soko.gaps} + - LOG_LEVEL=${LOG_LEVEL:-INFO} + - SERVICE_NAME=location-service + depends_on: + soko-ml-db: + condition: service_healthy + redis: + condition: service_healthy + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8003/health"] + interval: 15s + timeout: 5s + retries: 5 + start_period: 20s + restart: unless-stopped + kafka-agent: build: context: ./kafka-agent @@ -229,6 +353,7 @@ services: - KAFKA_DLQ_TOPIC=${KAFKA_DLQ_TOPIC:-soko.dlq} - PRICE_SERVICE_URL=http://price-prediction-service:8001 - REC_SERVICE_URL=http://recommendation-service:8002 + - DATA_INGESTION_SERVICE_URL=http://data-ingestion-service:8004 - LOG_LEVEL=${LOG_LEVEL:-INFO} - SERVICE_NAME=kafka-agent depends_on: @@ -238,4 +363,6 @@ services: condition: service_healthy recommendation-service: condition: service_healthy + data-ingestion-service: + condition: service_healthy restart: unless-stopped diff --git a/services/soko-ml/install_cmdstan.py b/services/soko-ml/install_cmdstan.py new file mode 100644 index 0000000..129b1a3 --- /dev/null +++ b/services/soko-ml/install_cmdstan.py @@ -0,0 +1,24 @@ +""" +One-time CmdStan install into Prophet's bundled stan_model directory. +Called by `make install`. Safe to re-run — skips if already present. +""" +import pathlib +import sys + +try: + import prophet + import cmdstanpy +except ImportError as e: + print(f"Missing dependency: {e}", file=sys.stderr) + sys.exit(1) + +stan_dir = pathlib.Path(prophet.__file__).parent / "stan_model" +stan_dir.mkdir(parents=True, exist_ok=True) +target = stan_dir / "cmdstan-2.33.1" + +if (target / "Makefile").exists(): + print("CmdStan 2.33.1 already present, skipping.") +else: + print("Downloading + compiling CmdStan 2.33.1 (~400 MB, takes a few minutes)...") + cmdstanpy.install_cmdstan(dir=str(stan_dir), version="2.33.1") + print("CmdStan 2.33.1 installed.") diff --git a/services/soko-ml/kafka-agent/src/agent.py b/services/soko-ml/kafka-agent/src/agent.py index 69679bf..6b63c9f 100644 --- a/services/soko-ml/kafka-agent/src/agent.py +++ b/services/soko-ml/kafka-agent/src/agent.py @@ -12,6 +12,8 @@ from .consumers.interaction_consumer import InteractionConsumer from .consumers.transaction_consumer import TransactionConsumer from .consumers.price_request_consumer import PriceRequestConsumer +from .consumers.coverage_gap_consumer import CoverageGapConsumer +from .consumers.transaction_price_collector import TransactionPriceCollector from .dlq import DLQHandler from .producers.event_producer import EventProducer @@ -34,6 +36,8 @@ def __init__(self): TransactionConsumer(self._producer, self._dlq), InteractionConsumer(self._dlq), PriceRequestConsumer(self._producer, self._dlq), + CoverageGapConsumer(), + TransactionPriceCollector(self._dlq), ] self._threads: list[threading.Thread] = [] self._shutdown_event = threading.Event() diff --git a/services/soko-ml/kafka-agent/src/consumers/coverage_gap_consumer.py b/services/soko-ml/kafka-agent/src/consumers/coverage_gap_consumer.py new file mode 100644 index 0000000..aa58dff --- /dev/null +++ b/services/soko-ml/kafka-agent/src/consumers/coverage_gap_consumer.py @@ -0,0 +1,79 @@ +""" +Consumes soko.gaps — coverage gap farmer and admin notification events. +Logs them for monitoring and forwards to the notification pipeline if configured. +Consumer group: soko-ml-gaps-group +""" +import json +import logging +import os +import threading + +from confluent_kafka import Consumer, KafkaError + +log = logging.getLogger(__name__) + + +class CoverageGapConsumer: + """ + Reads coverage gap events published by location-service. + Primarily exists for observability — logs every new/escalating gap. + Extend to forward admin events to a notification webhook if needed. + """ + + def __init__(self) -> None: + self._stop_event = threading.Event() + bootstrap = os.getenv("KAFKA_BOOTSTRAP_SERVERS", "kafka:9092") + self._topic = os.getenv("KAFKA_GAPS_TOPIC", "soko.gaps") + self._consumer = Consumer({ + "bootstrap.servers": bootstrap, + "group.id": "soko-ml-gaps-group", + "auto.offset.reset": "earliest", + "enable.auto.commit": False, + }) + self._consumer.subscribe([self._topic]) + + def run(self) -> None: + log.info(f"coverage_gap_consumer_started topic={self._topic}") + try: + while not self._stop_event.is_set(): + msg = self._consumer.poll(timeout=1.0) + if msg is None: + continue + if msg.error(): + if msg.error().code() != KafkaError._PARTITION_EOF: + log.error(f"coverage_gap_consumer_error: {msg.error()}") + continue + + raw = msg.value().decode("utf-8") + try: + event = json.loads(raw) + self._process(event) + self._consumer.commit(asynchronous=False) + except Exception as exc: + log.error(f"gap_event_processing_error: {exc}") + except (KeyboardInterrupt, SystemExit): + pass + finally: + self._consumer.close() + log.info("coverage_gap_consumer_stopped") + + def _process(self, event: dict) -> None: + recipient = event.get("recipient", "") + crop = event.get("crop_submitted", "") + priority = event.get("priority", "low") + frequency = event.get("frequency", 1) + + if recipient == "admin": + log.info( + f"coverage_gap_admin_alert " + f"crop={crop} priority={priority} frequency={frequency} " + f"category_guess={event.get('category_guess', '')}" + ) + elif recipient == "farmer": + log.info( + f"coverage_gap_farmer_notified " + f"farmer_id={event.get('farmer_id', '')} crop={crop}" + ) + + def stop(self) -> None: + self._stop_event.set() diff --git a/services/soko-ml/kafka-agent/src/consumers/transaction_price_collector.py b/services/soko-ml/kafka-agent/src/consumers/transaction_price_collector.py new file mode 100644 index 0000000..d5e84ef --- /dev/null +++ b/services/soko-ml/kafka-agent/src/consumers/transaction_price_collector.py @@ -0,0 +1,90 @@ +""" +Consumes soko.transactions and forwards purchase_completed events to +data-ingestion-service via HTTP POST /ingest/order-event. + +This is an alternative path to the TransactionStream inside data-ingestion-service itself. +Both can run simultaneously — data-ingestion-service deduplicates on order_id via the +insert_price_observation function (Postgres UNIQUE constraint on order_id per observation). + +Consumer group: soko-ml-price-collector +""" +import json +import logging +import os +import threading + +import httpx +from confluent_kafka import Consumer, KafkaError + +from ..dlq import DLQHandler + +log = logging.getLogger(__name__) + + +class TransactionPriceCollector: + """ + Forwards purchase_completed events from soko.transactions + to data-ingestion-service for price observation storage. + """ + + def __init__(self, dlq: DLQHandler) -> None: + self._dlq = dlq + self._stop_event = threading.Event() + bootstrap = os.getenv("KAFKA_BOOTSTRAP_SERVERS", "kafka:9092") + self._topic = os.getenv("KAFKA_TRANSACTION_TOPIC", "soko.transactions") + self._ingest_url = os.getenv("DATA_INGESTION_SERVICE_URL", "http://data-ingestion-service:8004") + self._consumer = Consumer({ + "bootstrap.servers": bootstrap, + "group.id": "soko-ml-price-collector", + "auto.offset.reset": "earliest", + "enable.auto.commit": False, + }) + self._consumer.subscribe([self._topic]) + + def run(self) -> None: + log.info(f"transaction_price_collector_started topic={self._topic}") + try: + while not self._stop_event.is_set(): + msg = self._consumer.poll(timeout=1.0) + if msg is None: + continue + if msg.error(): + if msg.error().code() != KafkaError._PARTITION_EOF: + log.error(f"transaction_price_collector_error: {msg.error()}") + continue + + raw = msg.value().decode("utf-8") + try: + event = json.loads(raw) + if event.get("event_type") == "purchase_completed": + self._forward(event) + self._consumer.commit(asynchronous=False) + except Exception as exc: + log.error(f"price_collector_processing_error: {exc}") + self._dlq.send( + original_topic=self._topic, + original_message=raw, + error_type=type(exc).__name__, + error_message=str(exc), + ) + except (KeyboardInterrupt, SystemExit): + pass + finally: + self._consumer.close() + log.info("transaction_price_collector_stopped") + + def _forward(self, event: dict) -> None: + try: + resp = httpx.post( + f"{self._ingest_url}/ingest/order-event", + json=event, + timeout=5.0, + ) + if resp.status_code not in (200, 201): + log.warning(f"ingest_service_non_200 status={resp.status_code}") + except httpx.RequestError as exc: + log.warning(f"ingest_service_unreachable: {exc} — event will be retried on next poll") + raise + + def stop(self) -> None: + self._stop_event.set() diff --git a/services/soko-ml/location-service/Dockerfile b/services/soko-ml/location-service/Dockerfile new file mode 100644 index 0000000..ff12162 --- /dev/null +++ b/services/soko-ml/location-service/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.11-slim + +WORKDIR /app + +RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY src/ ./src/ + +CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8003", "--log-level", "info"] diff --git a/services/soko-ml/location-service/requirements.txt b/services/soko-ml/location-service/requirements.txt new file mode 100644 index 0000000..6a89b7b --- /dev/null +++ b/services/soko-ml/location-service/requirements.txt @@ -0,0 +1,10 @@ +fastapi==0.111.0 +uvicorn[standard]==0.29.0 +pydantic==2.7.1 +asyncpg==0.29.0 +httpx==0.27.0 +redis[asyncio]==5.0.4 +confluent-kafka==2.4.0 +structlog==24.2.0 +pytest==8.2.0 +pytest-asyncio==0.23.6 diff --git a/services/soko-ml/location-service/src/__init__.py b/services/soko-ml/location-service/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/services/soko-ml/location-service/src/cache.py b/services/soko-ml/location-service/src/cache.py new file mode 100644 index 0000000..05d77c7 --- /dev/null +++ b/services/soko-ml/location-service/src/cache.py @@ -0,0 +1,140 @@ +""" +Redis cache layer for the location-service. +All keys and TTLs follow the registry defined in the system spec. +""" +import json +import logging +import os +from decimal import Decimal +from typing import Optional + +import redis.asyncio as aioredis + + +class _Encoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, Decimal): + return float(obj) + return super().default(obj) + +log = logging.getLogger(__name__) + +DIST_TTL = int(os.getenv("MAPS_DISTANCE_CACHE_TTL_SECONDS", "2592000")) # 30 days +ROUTE_TTL = 6 * 3600 # 6 hours +DISCOVER_TTL = 3600 # 1 hour +MARKET_REG_TTL = 6 * 3600 # 6 hours + +_redis: Optional[aioredis.Redis] = None + + +async def get_redis() -> aioredis.Redis: + global _redis + if _redis is None: + host = os.getenv("REDIS_HOST", "redis") + port = int(os.getenv("REDIS_PORT", "6379")) + db = int(os.getenv("REDIS_DB", "0")) + password = os.getenv("REDIS_PASSWORD") or None + _redis = aioredis.Redis(host=host, port=port, db=db, password=password, decode_responses=True) + return _redis + + +async def close_redis() -> None: + global _redis + if _redis: + await _redis.aclose() + _redis = None + + +# ── Distance cache ──────────────────────────────────────────────────────────── + +async def get_distance(farmer_id: str, market_id: str) -> Optional[float]: + redis = await get_redis() + try: + val = await redis.get(f"dist:{farmer_id}:{market_id}") + return float(val) if val is not None else None + except Exception as exc: + log.warning(f"cache get_distance error: {exc}") + return None + + +async def set_distance(farmer_id: str, market_id: str, km: float) -> None: + redis = await get_redis() + try: + await redis.setex(f"dist:{farmer_id}:{market_id}", DIST_TTL, str(km)) + except Exception as exc: + log.warning(f"cache set_distance error: {exc}") + + +async def invalidate_farmer_distances(farmer_id: str) -> None: + redis = await get_redis() + try: + async for key in redis.scan_iter(f"dist:{farmer_id}:*"): + await redis.delete(key) + async for key in redis.scan_iter(f"route:{farmer_id}:*"): + await redis.delete(key) + except Exception as exc: + log.warning(f"cache invalidate_farmer_distances error: {exc}") + + +# ── Route cache ─────────────────────────────────────────────────────────────── + +async def get_route(farmer_id: str, crop: str, quantity: float) -> Optional[dict]: + redis = await get_redis() + key = f"route:{farmer_id}:{crop}:{int(quantity)}" + try: + val = await redis.get(key) + return json.loads(val) if val else None + except Exception as exc: + log.warning(f"cache get_route error: {exc}") + return None + + +async def set_route(farmer_id: str, crop: str, quantity: float, result: dict) -> None: + redis = await get_redis() + key = f"route:{farmer_id}:{crop}:{int(quantity)}" + try: + await redis.setex(key, ROUTE_TTL, json.dumps(result, cls=_Encoder)) + except Exception as exc: + log.warning(f"cache set_route error: {exc}") + + +# ── Discover cache ──────────────────────────────────────────────────────────── + +async def get_discover(buyer_id: str, crop: str, max_price: float) -> Optional[dict]: + redis = await get_redis() + key = f"discover:{buyer_id}:{crop}:{int(max_price)}" + try: + val = await redis.get(key) + return json.loads(val) if val else None + except Exception as exc: + log.warning(f"cache get_discover error: {exc}") + return None + + +async def set_discover(buyer_id: str, crop: str, max_price: float, result: dict) -> None: + redis = await get_redis() + key = f"discover:{buyer_id}:{crop}:{int(max_price)}" + try: + await redis.setex(key, DISCOVER_TTL, json.dumps(result, cls=_Encoder)) + except Exception as exc: + log.warning(f"cache set_discover error: {exc}") + + +# ── Market registry cache ───────────────────────────────────────────────────── + +async def get_market_registry() -> Optional[list]: + redis = await get_redis() + try: + val = await redis.get("market:registry") + return json.loads(val) if val else None + except Exception as exc: + log.warning(f"cache get_market_registry error: {exc}") + return None + + +async def set_market_registry(markets: list) -> None: + redis = await get_redis() + try: + await redis.setex("market:registry", MARKET_REG_TTL, json.dumps(markets, cls=_Encoder)) + except Exception as exc: + log.warning(f"cache set_market_registry error: {exc}") diff --git a/services/soko-ml/location-service/src/fallback.py b/services/soko-ml/location-service/src/fallback.py new file mode 100644 index 0000000..1a7a426 --- /dev/null +++ b/services/soko-ml/location-service/src/fallback.py @@ -0,0 +1,156 @@ +""" +Three-tier fallback system for crops/markets not yet in the ML model. +Checks the coverage_map table in soko_ml_db — no hardcoded lists. +Never raises. Always returns a response. +""" +import json +import logging +import os +from typing import Optional + +import asyncpg + +from .transport_cost import TRANSPORT_DISCLAIMER + +log = logging.getLogger(__name__) + +POSTGRES_DSN = os.getenv("POSTGRES_DSN", "postgresql://soko_ml:changeme@soko-ml-db:5432/soko_ml_db") + +CATEGORY_PRICE_BANDS: dict[str, dict] = { + "cereals": {"low": 800, "high": 2000, "crops": {"maize_grain", "sorghum", "millet", "rice"}}, + "legumes": {"low": 2000, "high": 5000, "crops": {"yellow_beans", "groundnuts", "soybeans"}}, + "vegetables": {"low": 400, "high": 1500, "crops": {"tomatoes", "kale", "nakati", "cabbage", "onions", "eggplant"}}, + "tubers": {"low": 300, "high": 1200, "crops": {"irish_potatoes", "cassava_chips", "sweet_potatoes", "yams"}}, + "cash_crops": {"low": 3000, "high": 15000, "crops": {"coffee", "vanilla", "cotton"}}, + "fruits": {"low": 400, "high": 1200, "crops": {"matoke"}}, +} + +KAFKA_ML_TOPIC = os.getenv("KAFKA_ML_EVENTS_TOPIC", "soko.ml.events") + +_pool: Optional[asyncpg.Pool] = None + + +async def _get_pool() -> asyncpg.Pool: + global _pool + if _pool is None: + _pool = await asyncpg.create_pool(POSTGRES_DSN, min_size=1, max_size=3) + return _pool + + +async def close_pool() -> None: + global _pool + if _pool: + await _pool.close() + _pool = None + + +def _find_category(crop: str) -> Optional[str]: + for category, data in CATEGORY_PRICE_BANDS.items(): + if crop in data["crops"]: + return category + return None + + +async def determine_tier(crop: str, market: str) -> int: + """ + Returns 1, 2, or 3 based on coverage_map state. + 1 = full ML coverage + 2 = partial or category coverage only + 3 = out of scope + """ + try: + pool = await _get_pool() + async with pool.acquire() as conn: + row = await conn.fetchrow( + "SELECT observation_count, is_model_ready FROM coverage_map WHERE crop = $1 AND market = $2", + crop, market, + ) + except Exception: + return 2 # conservative default if DB unreachable + + if row is None: + category = _find_category(crop) + return 2 if category else 3 + + if row["is_model_ready"]: + return 1 + + return 2 + + +async def build_tier2_response( + farmer_id: str, + crop: str, + quantity_kg: float, + max_distance_km: float, +) -> dict: + """Tier 2: returns category-level price band estimate.""" + category = _find_category(crop) + band = CATEGORY_PRICE_BANDS.get(category, {}) if category else {} + + try: + pool = await _get_pool() + async with pool.acquire() as conn: + row = await conn.fetchrow( + """ + SELECT observation_count, min_observations_needed + FROM coverage_map WHERE crop = $1 + LIMIT 1 + """, + crop, + ) + obs_count = row["observation_count"] if row else 0 + obs_needed = row["min_observations_needed"] if row else 52 + except Exception: + obs_count = 0 + obs_needed = 52 + + price_low = band.get("low", 0) + price_high = band.get("high", 0) + + return { + "farmer_id": farmer_id, + "crop": crop, + "quantity_kg": quantity_kg, + "currency": "UGX", + "tier": 2, + "ranked_markets": [], + "transport_disclaimer": TRANSPORT_DISCLAIMER, + "cached_distances": False, + "tier_message": ( + f"Limited data available for {crop} — estimate only. " + f"Category price range: {price_low:,}–{price_high:,} UGX/kg. " + f"{obs_count} of {obs_needed} observations collected so far." + ), + } + + +def build_tier3_response(farmer_id: str, crop: str, quantity_kg: float, reason: str = "") -> dict: + """Tier 3: crop not recognised. Returns graceful no-data response. Never raises.""" + return { + "farmer_id": farmer_id, + "crop": crop, + "quantity_kg": quantity_kg, + "currency": "UGX", + "tier": 3, + "ranked_markets": [], + "transport_disclaimer": TRANSPORT_DISCLAIMER, + "cached_distances": False, + "tier_message": ( + f"We don't have market intelligence for '{crop}' yet. " + "Our team has been notified and will add coverage soon. " + "You can still list your produce and transact normally on Soko." + ), + } + + +async def handle_unknown_crop(crop: str, farmer_id: str) -> None: + """ + Records the gap in coverage_gaps table and publishes Kafka events. + Called when Tier 3 triggers. Never raises. + """ + try: + from .gap_notifier import record_and_notify_gap + await record_and_notify_gap(crop, farmer_id) + except Exception as exc: + log.warning(f"Gap notification failed for crop={crop}: {exc}") diff --git a/services/soko-ml/location-service/src/gap_notifier.py b/services/soko-ml/location-service/src/gap_notifier.py new file mode 100644 index 0000000..bff7b00 --- /dev/null +++ b/services/soko-ml/location-service/src/gap_notifier.py @@ -0,0 +1,133 @@ +""" +Publishes coverage gap events to soko.ml.events and records them in coverage_gaps. +Called by fallback.py when Tier 3 triggers. +""" +import json +import logging +import os +from datetime import datetime +from typing import Optional + +import asyncpg +from confluent_kafka import Producer + +log = logging.getLogger(__name__) + +POSTGRES_DSN = os.getenv("POSTGRES_DSN", "postgresql://soko_ml:changeme@soko-ml-db:5432/soko_ml_db") +BOOTSTRAP_SERVERS = os.getenv("KAFKA_BOOTSTRAP_SERVERS", "kafka:9092") +ML_EVENTS_TOPIC = os.getenv("KAFKA_ML_EVENTS_TOPIC", "soko.ml.events") +GAPS_TOPIC = os.getenv("KAFKA_GAPS_TOPIC", "soko.gaps") + +CATEGORY_GUESSES: dict[str, str] = { + "nakati": "vegetables", "kale": "vegetables", "cabbage": "vegetables", + "onions": "vegetables", "eggplant": "vegetables", + "groundnuts": "legumes", "soybeans": "legumes", + "sweet_potatoes": "tubers", "yams": "tubers", + "rice": "cereals", "wheat": "cereals", + "coffee": "cash_crops", "vanilla": "cash_crops", "cotton": "cash_crops", +} + +PRIORITY_THRESHOLDS = {"low": (1, 5), "medium": (5, 15), "high": (15, 99999)} + +_pool: Optional[asyncpg.Pool] = None +_producer: Optional[Producer] = None + + +def _get_producer() -> Optional[Producer]: + global _producer + if _producer is None: + try: + _producer = Producer({ + "bootstrap.servers": BOOTSTRAP_SERVERS, + "socket.timeout.ms": 3000, + }) + except Exception as exc: + log.warning(f"Kafka producer init failed: {exc}") + return _producer + + +def _compute_priority(freq: int) -> str: + for p, (lo, hi) in PRIORITY_THRESHOLDS.items(): + if lo <= freq < hi: + return p + return "high" + + +async def record_and_notify_gap(crop_submitted: str, farmer_id: str) -> None: + """Records the gap in Postgres and publishes two Kafka events (farmer + admin).""" + category = CATEGORY_GUESSES.get(crop_submitted.lower(), "other") + + # Upsert into coverage_gaps + try: + pool = await _get_db_pool() + async with pool.acquire() as conn: + existing = await conn.fetchrow( + "SELECT frequency FROM coverage_gaps WHERE crop_submitted = $1", + crop_submitted, + ) + if existing: + new_freq = existing["frequency"] + 1 + priority = _compute_priority(new_freq) + await conn.execute( + "UPDATE coverage_gaps SET frequency=$1, last_reported_at=NOW(), priority=$2 WHERE crop_submitted=$3", + new_freq, priority, crop_submitted, + ) + else: + new_freq = 1 + priority = "low" + await conn.execute( + "INSERT INTO coverage_gaps (crop_submitted, category_guess, frequency, first_reported_by, priority) VALUES ($1,$2,1,$3,'low')", + crop_submitted, category, farmer_id, + ) + except Exception as exc: + log.warning(f"Failed to record coverage gap in DB: {exc}") + new_freq = 1 + priority = "low" + + ts = datetime.utcnow().isoformat() + "Z" + + farmer_event = json.dumps({ + "event_type": "crop_coverage_gap", + "recipient": "farmer", + "farmer_id": farmer_id, + "crop_submitted": crop_submitted, + "timestamp": ts, + "message": { + "title": "We're working on adding this crop", + "body": ( + f"We don't have market intelligence for {crop_submitted} yet. " + "Our team has been notified and will add coverage soon. " + "You can still list your produce and transact normally on Soko." + ), + "action": "continue_listing", + }, + }).encode() + + admin_event = json.dumps({ + "event_type": "crop_coverage_gap", + "recipient": "admin", + "crop_submitted": crop_submitted, + "category_guess": category, + "first_reported_by": farmer_id, + "frequency": new_freq, + "status": "pending_review", + "priority": priority, + "timestamp": ts, + }).encode() + + producer = _get_producer() + if producer: + try: + producer.produce(GAPS_TOPIC, key=crop_submitted.encode(), value=farmer_event) + producer.produce(GAPS_TOPIC, key=crop_submitted.encode(), value=admin_event) + producer.poll(0) + log.info(f"coverage_gap_events_published crop={crop_submitted} priority={priority}") + except Exception as exc: + log.warning(f"Failed to publish gap events: {exc}") + + +async def _get_db_pool() -> asyncpg.Pool: + global _pool + if _pool is None: + _pool = await asyncpg.create_pool(POSTGRES_DSN, min_size=1, max_size=3) + return _pool diff --git a/services/soko-ml/location-service/src/geo_recommender.py b/services/soko-ml/location-service/src/geo_recommender.py new file mode 100644 index 0000000..abc006d --- /dev/null +++ b/services/soko-ml/location-service/src/geo_recommender.py @@ -0,0 +1,119 @@ +""" +Geo-scoped farmer discovery for the /discover endpoint. +Finds farmers near a buyer who grow the requested crop within price range. +""" +import logging +import math +import os +from typing import Optional + +import asyncpg + +log = logging.getLogger(__name__) + +POSTGRES_DSN = os.getenv("POSTGRES_DSN", "postgresql://soko_ml:changeme@soko-ml-db:5432/soko_ml_db") +PRICE_SERVICE_URL = os.getenv("PRICE_SERVICE_URL", "http://ml-gateway-service:8080") + +_pool: Optional[asyncpg.Pool] = None + + +async def _get_pool() -> asyncpg.Pool: + global _pool + if _pool is None: + _pool = await asyncpg.create_pool(POSTGRES_DSN, min_size=1, max_size=5) + return _pool + + +def _haversine(lat1: float, lng1: float, lat2: float, lng2: float) -> float: + R = 6371.0 + phi1 = math.radians(lat1) + phi2 = math.radians(lat2) + dphi = math.radians(lat2 - lat1) + dlam = math.radians(lng2 - lng1) + a = math.sin(dphi / 2) ** 2 + math.cos(phi1) * math.cos(phi2) * math.sin(dlam / 2) ** 2 + return round(2 * R * math.atan2(math.sqrt(a), math.sqrt(1 - a)), 1) + + +async def _get_market_price(crop: str, district: str) -> Optional[float]: + """Fetches the week1 predicted price for a crop in the nearest market for the given district.""" + from .market_router import fetch_predictions + # location-service uses a local copy of the district→market mapping + DISTRICT_TO_MARKET_LOCAL = { + "Kampala": "Kisenyi_Kampala", "Wakiso": "Kisenyi_Kampala", + "Mukono": "Kisenyi_Kampala", "Jinja": "Kisenyi_Kampala", + "Gulu": "Gulu", "Arua": "Gulu", + "Mbarara": "Mbarara", "Bushenyi": "Mbarara", "Ntungamo": "Mbarara", + "Mbale": "Mbale", "Tororo": "Mbale", "Iganga": "Mbale", + "Lira": "Lira", "Soroti": "Lira", + "Masaka": "Masaka", "Rakai": "Masaka", + } + market = DISTRICT_TO_MARKET_LOCAL.get(district, "Kisenyi_Kampala") + predictions = await fetch_predictions(market, crop) + if predictions: + return float(predictions[0]["predicted_price_ugx"]) + return None + + +async def discover_farmers( + buyer_lat: float, + buyer_lng: float, + crop: str, + max_price_ugx: float, + max_distance_km: float, + top_n: int, + buyer_district: str = "", +) -> list[dict]: + """ + Finds farmers who grow the given crop, within max_distance_km and below max_price_ugx. + Uses district centroids for distance calculation (approximate, documented limitation). + """ + pool = await _get_pool() + async with pool.acquire() as conn: + rows = await conn.fetch( + """ + SELECT farmer_id, name, lat, lng, district, + crops_offered, avg_rating, fulfillment_rate + FROM farmer_features + WHERE crops_offered @> ARRAY[$1]::text[] + """, + crop, + ) + + relax = float(os.getenv("GEO_FILTER_RELAX_FACTOR", "1.5")) + threshold = max_distance_km * relax + + market_price = await _get_market_price(crop, buyer_district) + + results = [] + for r in rows: + lat = r["lat"] + lng = r["lng"] + if lat and lng: + dist = _haversine(buyer_lat, buyer_lng, float(lat), float(lng)) + if dist > threshold: + continue + else: + dist = None + + # We don't have asking price per farmer directly; use market price as proxy + # Farmers with listings below market price would be surfaced by produce-service + asking_price = None # Would require produce-service lookup + price_vs_market = "market_rate" + if market_price and asking_price: + price_vs_market = "below_market" if asking_price < market_price else "above_market" + + results.append({ + "farmer_id": r["farmer_id"], + "farmer_name": r["name"] or "", + "distance_km": dist, + "asking_price_ugx": asking_price, + "current_market_price_ugx": market_price, + "price_vs_market": price_vs_market, + "avg_rating": float(r["avg_rating"] or 0), + "fulfillment_rate": float(r["fulfillment_rate"] or 1.0), + "available_quantity_kg": None, + }) + + # Sort by distance (None distances go last), then by rating + results.sort(key=lambda x: (x["distance_km"] is None, x["distance_km"] or 0, -x["avg_rating"])) + return results[:top_n] diff --git a/services/soko-ml/location-service/src/google_maps_client.py b/services/soko-ml/location-service/src/google_maps_client.py new file mode 100644 index 0000000..d62d3f4 --- /dev/null +++ b/services/soko-ml/location-service/src/google_maps_client.py @@ -0,0 +1,89 @@ +""" +Google Maps Distance Matrix API client. +Batches all market distance requests in a single API call per farmer query. +Results are cached in Redis for DIST_TTL (30 days) to minimise API spend. +""" +import logging +import os +from typing import Optional + +import httpx + +log = logging.getLogger(__name__) + +MAPS_API_KEY = os.getenv("GOOGLE_MAPS_API_KEY", "") +MATRIX_URL = "https://maps.googleapis.com/maps/api/distancematrix/json" + + +async def get_road_distances( + origin_lat: float, + origin_lng: float, + destinations: list[dict], # list of {"market_id": str, "lat": float, "lng": float} +) -> dict[str, float]: + """ + Calls the Distance Matrix API once with all destinations in a single request. + Returns {market_id: distance_km} for each destination. + Falls back to Haversine straight-line distance if API key is missing or call fails. + """ + if not MAPS_API_KEY: + log.warning("GOOGLE_MAPS_API_KEY not set — using Haversine straight-line distances") + return _haversine_fallback(origin_lat, origin_lng, destinations) + + origin_str = f"{origin_lat},{origin_lng}" + dest_str = "|".join(f"{d['lat']},{d['lng']}" for d in destinations) + + try: + async with httpx.AsyncClient(timeout=10.0) as client: + resp = await client.get( + MATRIX_URL, + params={ + "origins": origin_str, + "destinations": dest_str, + "mode": "driving", + "key": MAPS_API_KEY, + }, + ) + resp.raise_for_status() + data = resp.json() + except Exception as exc: + log.warning(f"Google Maps API call failed: {exc} — falling back to Haversine") + return _haversine_fallback(origin_lat, origin_lng, destinations) + + if data.get("status") != "OK": + log.warning(f"Maps API status={data.get('status')} — falling back to Haversine") + return _haversine_fallback(origin_lat, origin_lng, destinations) + + elements = data.get("rows", [{}])[0].get("elements", []) + result: dict[str, float] = {} + + for dest, element in zip(destinations, elements): + market_id = dest["market_id"] + if element.get("status") == "OK": + result[market_id] = round(element["distance"]["value"] / 1000.0, 1) + else: + # Fallback for this specific destination + result[market_id] = _haversine_single(origin_lat, origin_lng, dest["lat"], dest["lng"]) + + return result + + +def _haversine_single(lat1: float, lng1: float, lat2: float, lng2: float) -> float: + import math + R = 6371.0 + phi1 = math.radians(lat1) + phi2 = math.radians(lat2) + dphi = math.radians(lat2 - lat1) + dlam = math.radians(lng2 - lng1) + a = math.sin(dphi / 2) ** 2 + math.cos(phi1) * math.cos(phi2) * math.sin(dlam / 2) ** 2 + return round(2 * R * math.atan2(math.sqrt(a), math.sqrt(1 - a)), 1) + + +def _haversine_fallback( + origin_lat: float, + origin_lng: float, + destinations: list[dict], +) -> dict[str, float]: + return { + d["market_id"]: _haversine_single(origin_lat, origin_lng, d["lat"], d["lng"]) + for d in destinations + } diff --git a/services/soko-ml/location-service/src/main.py b/services/soko-ml/location-service/src/main.py new file mode 100644 index 0000000..bd7d404 --- /dev/null +++ b/services/soko-ml/location-service/src/main.py @@ -0,0 +1,118 @@ +""" +location-service — FastAPI entry point. +Exposes /health, /route, /discover. +""" +import logging +import os +from contextlib import asynccontextmanager + +import structlog +from fastapi import FastAPI +from fastapi.responses import JSONResponse + +from .cache import get_redis, close_redis +from .market_router import load_market_registry, close_pool as close_router_pool +from .fallback import close_pool as close_fallback_pool, _get_pool as get_fallback_pool +from .schemas import RouteRequest, RouteResponse, DiscoverRequest, DiscoverResponse + +structlog.configure( + processors=[ + structlog.processors.TimeStamper(fmt="iso"), + structlog.processors.add_log_level, + structlog.processors.JSONRenderer(), + ] +) +log = structlog.get_logger() + +SERVICE_NAME = os.getenv("SERVICE_NAME", "location-service") + + +@asynccontextmanager +async def lifespan(app: FastAPI): + await get_redis() + await load_market_registry() + log.info("location_service_started") + yield + await close_redis() + await close_router_pool() + await close_fallback_pool() + log.info("location_service_stopped") + + +app = FastAPI(title="Soko Location Service", version="2.0.0", lifespan=lifespan) + + +@app.get("/health") +async def health(): + markets = await load_market_registry() + return { + "status": "ok", + "service": SERVICE_NAME, + "markets_loaded": len(markets), + } + + +@app.post("/route", response_model=RouteResponse) +async def route_endpoint(request: RouteRequest): + from .cache import get_route, set_route + from .market_router import route + from .fallback import determine_tier, build_tier3_response, handle_unknown_crop + + # Check tier before computing + # If the crop is completely unknown, go straight to Tier 3 + from .fallback import _find_category + if not _find_category(request.crop): + # Attempt to find it in coverage_map + tier = await determine_tier(request.crop, "Kisenyi_Kampala") + if tier == 3: + await handle_unknown_crop(request.crop, request.farmer_id) + result = build_tier3_response(request.farmer_id, request.crop, request.quantity_kg) + return JSONResponse(content=result) + + # Check route cache + cached = await get_route(request.farmer_id, request.crop, request.quantity_kg) + if cached: + return JSONResponse(content=cached) + + result = await route( + farmer_id=request.farmer_id, + farmer_lat=request.farmer_lat, + farmer_lng=request.farmer_lng, + crop=request.crop, + quantity_kg=request.quantity_kg, + max_distance_km=request.max_distance_km, + ) + + if result.get("tier") == 1 and result.get("ranked_markets"): + await set_route(request.farmer_id, request.crop, request.quantity_kg, result) + + return JSONResponse(content=result) + + +@app.post("/discover", response_model=DiscoverResponse) +async def discover_endpoint(request: DiscoverRequest): + from .cache import get_discover, set_discover + from .geo_recommender import discover_farmers + + cached = await get_discover(request.buyer_id, request.crop, request.max_price_ugx) + if cached: + return JSONResponse(content=cached) + + results = await discover_farmers( + buyer_lat=request.buyer_lat, + buyer_lng=request.buyer_lng, + crop=request.crop, + max_price_ugx=request.max_price_ugx, + max_distance_km=request.max_distance_km, + top_n=request.top_n, + buyer_district="", + ) + + response = { + "buyer_id": request.buyer_id, + "crop": request.crop, + "results": results, + } + + await set_discover(request.buyer_id, request.crop, request.max_price_ugx, response) + return JSONResponse(content=response) diff --git a/services/soko-ml/location-service/src/market_router.py b/services/soko-ml/location-service/src/market_router.py new file mode 100644 index 0000000..0f330b8 --- /dev/null +++ b/services/soko-ml/location-service/src/market_router.py @@ -0,0 +1,202 @@ +""" +Core routing logic: given a farmer location + crop + quantity, +ranks markets by net value after transport cost. +""" +import logging +import os +from typing import Optional + +import asyncpg + +from .cache import ( + get_distance, set_distance, + get_market_registry, set_market_registry, +) +from .google_maps_client import get_road_distances +from .transport_cost import estimate as estimate_transport +from .sell_signal import derive_signal + +log = logging.getLogger(__name__) + +POSTGRES_DSN = os.getenv("POSTGRES_DSN", "postgresql://soko_ml:changeme@soko-ml-db:5432/soko_ml_db") +PRICE_SERVICE_URL = os.getenv("PRICE_SERVICE_URL", "http://ml-gateway-service:8080") +DEFAULT_MAX_KM = float(os.getenv("DEFAULT_MAX_DISTANCE_KM", "150")) + +_pool: Optional[asyncpg.Pool] = None + + +async def _get_pool() -> asyncpg.Pool: + global _pool + if _pool is None: + _pool = await asyncpg.create_pool(POSTGRES_DSN, min_size=1, max_size=5) + return _pool + + +async def close_pool() -> None: + global _pool + if _pool: + await _pool.close() + _pool = None + + +# ── Market registry ─────────────────────────────────────────────────────────── + +async def load_market_registry() -> list[dict]: + """Loads markets from soko_ml_db. Falls back to Redis cache if DB is unreachable.""" + try: + pool = await _get_pool() + async with pool.acquire() as conn: + rows = await conn.fetch( + "SELECT market_id, name, lat, lng, district FROM market_registry WHERE active = TRUE" + ) + markets = [dict(r) for r in rows] + await set_market_registry(markets) + return markets + except Exception as exc: + log.warning(f"Failed to load market registry from DB: {exc} — trying Redis cache") + cached = await get_market_registry() + if cached: + return cached + log.error("Market registry unavailable from both DB and cache") + return [] + + +# ── Distance fetching with cache ────────────────────────────────────────────── + +async def get_distances_for_farmer( + farmer_id: str, + farmer_lat: float, + farmer_lng: float, + markets: list[dict], +) -> tuple[dict[str, float], bool]: + """ + Returns ({market_id: km}, cached: bool). + Hits Redis cache first; only calls Maps API for uncached markets. + """ + result: dict[str, float] = {} + uncached: list[dict] = [] + + for m in markets: + mid = m["market_id"] + cached_km = await get_distance(farmer_id, mid) + if cached_km is not None: + result[mid] = cached_km + else: + uncached.append(m) + + all_cached = len(uncached) == 0 + + if uncached: + fresh = await get_road_distances( + farmer_lat, farmer_lng, + [{"market_id": m["market_id"], "lat": float(m["lat"]), "lng": float(m["lng"])} for m in uncached], + ) + for mid, km in fresh.items(): + result[mid] = km + await set_distance(farmer_id, mid, km) + + return result, all_cached + + +# ── Price fetching from gateway ─────────────────────────────────────────────── + +async def fetch_predictions(market: str, crop: str, weeks: int = 4) -> list[dict]: + import httpx + try: + async with httpx.AsyncClient(timeout=10.0) as client: + resp = await client.post( + f"{PRICE_SERVICE_URL}/price/predict", + json={"market": market, "crop": crop, "weeks_ahead": weeks}, + ) + if resp.status_code == 200: + return resp.json().get("predictions", []) + except Exception as exc: + log.warning(f"Failed to fetch predictions for {market}/{crop}: {exc}") + return [] + + +# ── Main routing function ───────────────────────────────────────────────────── + +async def route( + farmer_id: str, + farmer_lat: float, + farmer_lng: float, + crop: str, + quantity_kg: float, + max_distance_km: float = DEFAULT_MAX_KM, +) -> dict: + """ + Returns the full /route response dict including tier determination, + ranked markets, signals, and disclaimer. + """ + from .fallback import determine_tier, build_tier2_response, build_tier3_response + + markets = await load_market_registry() + if not markets: + return build_tier3_response(farmer_id, crop, quantity_kg, "Market registry unavailable") + + # Filter by distance + all_distances, cached = await get_distances_for_farmer(farmer_id, farmer_lat, farmer_lng, markets) + + reachable = [ + m for m in markets + if all_distances.get(m["market_id"], 9999) <= max_distance_km + ] + + if not reachable: + # Relax to nearest market regardless of distance + nearest_id = min(all_distances, key=lambda k: all_distances[k]) + reachable = [m for m in markets if m["market_id"] == nearest_id] + + from .transport_cost import TRANSPORT_DISCLAIMER + + ranked = [] + for market in reachable: + mid = market["market_id"] + distance_km = all_distances.get(mid, 0) + + tier = await determine_tier(crop, mid) + if tier == 3: + continue # Skip markets with no ML coverage — Tier 3 handled separately + + transport = estimate_transport(distance_km) + transport_cost = transport["ugx_per_kg"] + + predictions = await fetch_predictions(mid, crop) + if not predictions: + continue + + week1_price = float(predictions[0]["predicted_price_ugx"]) + net_value = week1_price - transport_cost + total_net = round(net_value * quantity_kg, 0) + + signal_data = derive_signal(crop, predictions) + + ranked.append({ + "market": mid, + "distance_km": distance_km, + "transport_mode": transport["mode"], + "transport_cost_per_kg_ugx": transport_cost, + "predicted_price_ugx": week1_price, + "net_value_per_kg_ugx": round(net_value, 0), + "total_net_value_ugx": total_net, + "signal": signal_data["signal"], + "signal_reason": signal_data["reason"], + "confidence": signal_data["confidence"], + }) + + if not ranked: + return await build_tier2_response(farmer_id, crop, quantity_kg, max_distance_km) + + ranked.sort(key=lambda x: x["net_value_per_kg_ugx"], reverse=True) + + return { + "farmer_id": farmer_id, + "crop": crop, + "quantity_kg": quantity_kg, + "currency": "UGX", + "tier": 1, + "ranked_markets": ranked, + "transport_disclaimer": TRANSPORT_DISCLAIMER, + "cached_distances": cached, + } diff --git a/services/soko-ml/location-service/src/schemas.py b/services/soko-ml/location-service/src/schemas.py new file mode 100644 index 0000000..7ee0836 --- /dev/null +++ b/services/soko-ml/location-service/src/schemas.py @@ -0,0 +1,64 @@ +from typing import Optional +from pydantic import BaseModel + + +class RouteRequest(BaseModel): + farmer_id: str + farmer_lat: float + farmer_lng: float + crop: str + quantity_kg: float + max_distance_km: float = 150.0 + + +class MarketResult(BaseModel): + market: str + distance_km: float + transport_mode: str + transport_cost_per_kg_ugx: float + predicted_price_ugx: float + net_value_per_kg_ugx: float + total_net_value_ugx: float + signal: str + signal_reason: str + confidence: str + + +class RouteResponse(BaseModel): + farmer_id: str + crop: str + quantity_kg: float + currency: str = "UGX" + tier: int + ranked_markets: list[MarketResult] + transport_disclaimer: str + cached_distances: bool + tier_message: Optional[str] = None + + +class DiscoverRequest(BaseModel): + buyer_id: str + buyer_lat: float + buyer_lng: float + crop: str + max_price_ugx: float + max_distance_km: float = 100.0 + top_n: int = 5 + + +class FarmerResult(BaseModel): + farmer_id: str + farmer_name: str + distance_km: Optional[float] + asking_price_ugx: Optional[float] + current_market_price_ugx: Optional[float] + price_vs_market: str + avg_rating: float + fulfillment_rate: float + available_quantity_kg: Optional[float] + + +class DiscoverResponse(BaseModel): + buyer_id: str + crop: str + results: list[FarmerResult] diff --git a/services/soko-ml/location-service/src/sell_signal.py b/services/soko-ml/location-service/src/sell_signal.py new file mode 100644 index 0000000..61ddd0a --- /dev/null +++ b/services/soko-ml/location-service/src/sell_signal.py @@ -0,0 +1,85 @@ +""" +GO/WAIT sell signal derivation from a 4-week price forecast. +""" +from datetime import datetime +from typing import Optional + +PERISHABLE_CROPS = {"tomatoes", "matoke"} +HARVEST_MONTHS = {6, 7, 11, 12} +LEAN_SEASON_MONTHS = {1, 2} +WAIT_THRESHOLD = 0.10 # week4 > week1 by 10% → WAIT +SELL_THRESHOLD = 0.05 # week4 < week1 by 5% → SELL_NOW + + +def derive_signal( + crop: str, + predictions: list[dict], + now: Optional[datetime] = None, +) -> dict: + """ + Applies rules in order — first match wins. + + predictions: list of weekly prediction dicts, each with "predicted_price_ugx". + Returns {"signal": str, "reason": str, "confidence": str}. + """ + if now is None: + now = datetime.utcnow() + + month = now.month + + if not predictions: + return {"signal": "SELL_NOW", "reason": "No forecast data available.", "confidence": "low"} + + week1_price = float(predictions[0]["predicted_price_ugx"]) + week4_price = float(predictions[-1]["predicted_price_ugx"]) if len(predictions) >= 4 else week1_price + + slope = (week4_price - week1_price) / max(week1_price, 1) + + # Rule 1 — perishable crops, sell immediately regardless of price + if crop in PERISHABLE_CROPS: + return { + "signal": "SELL_NOW_PERISHABLE", + "reason": f"{crop.replace('_', ' ').title()} is perishable. Sell immediately to prevent losses.", + "confidence": "high", + } + + # Rule 2 — harvest season, prices will recover + if month in HARVEST_MONTHS: + return { + "signal": "WAIT", + "reason": "Harvest season is suppressing prices. Hold if you have storage — prices recover in 4–6 weeks.", + "confidence": "high", + } + + # Rule 3 — lean dry season peak + if month in LEAN_SEASON_MONTHS: + return { + "signal": "SELL_NOW", + "reason": "Prices are at lean season peak. This is typically the best time to sell.", + "confidence": "high", + } + + # Rule 4 — strong upward trend + if slope > WAIT_THRESHOLD: + pct = round(slope * 100, 1) + return { + "signal": "WAIT", + "reason": f"Prices are forecast to rise {pct}% over the next 4 weeks. Consider waiting.", + "confidence": "medium", + } + + # Rule 5 — downward trend + if slope < -SELL_THRESHOLD: + pct = round(abs(slope) * 100, 1) + return { + "signal": "SELL_NOW", + "reason": f"Prices are forecast to fall {pct}% over the next 4 weeks. Sell now.", + "confidence": "medium", + } + + # Default — uncertainty favours action for smallholders + return { + "signal": "SELL_NOW", + "reason": "Prices are stable. Selling now avoids storage risk and provides immediate cash flow.", + "confidence": "medium", + } diff --git a/services/soko-ml/location-service/src/transport_cost.py b/services/soko-ml/location-service/src/transport_cost.py new file mode 100644 index 0000000..9126575 --- /dev/null +++ b/services/soko-ml/location-service/src/transport_cost.py @@ -0,0 +1,51 @@ +""" +Uganda agricultural transport cost estimator — distance-band model. + +Rates are calibrated to what a smallholder farmer (100–500 kg load) actually pays +for partial-load or shared transport in Uganda. + +Short-haul figures mirror FarasUG cargo and SafeBoda heavy-goods schedules +(base fee ~5,000 UGX + ~400 UGX/km for 100 kg, giving ~290 UGX/kg over 20 km). +Medium and long-haul figures reflect shared-lorry rates for partial loads on Uganda's +major agricultural corridors (e.g. Kampala–Gulu ~850 UGX/kg for 275 km). + +All rates include a ~35 UGX/kg market-handling allowance +(porterage + local authority market levy). +""" + +TRANSPORT_RATE_BANDS: list[dict] = [ + # max_km is the upper bound of the band (inclusive) + {"max_km": 25, "ugx_per_kg": 290, "mode": "boda_cargo", "label": "Motorcycle / local delivery van"}, + {"max_km": 80, "ugx_per_kg": 420, "mode": "taxi_van", "label": "Shared minibus taxi or cargo van"}, + {"max_km": 200, "ugx_per_kg": 620, "mode": "pickup_truck", "label": "Hired pickup truck or mini-lorry"}, + {"max_km": 400, "ugx_per_kg": 850, "mode": "shared_lorry", "label": "Shared long-haul lorry (partial load)"}, + {"max_km": 9999, "ugx_per_kg": 1100, "mode": "cross_region", "label": "Cross-region long-haul lorry (400 km+)"}, +] + +TRANSPORT_DISCLAIMER = ( + "Transport cost is an estimate based on Uganda road freight rates for partial loads " + "(FarasUG / SafeBoda benchmarks). Actual cost depends on load size, road conditions, " + "and your arrangement with a transporter. Soko does not provide or arrange transport." +) + + +def estimate(distance_km: float) -> dict: + """ + Returns transport cost info for the given road distance. + Keys: ugx_per_kg, mode, label, disclaimer. + """ + for band in TRANSPORT_RATE_BANDS: + if distance_km <= band["max_km"]: + return { + "ugx_per_kg": float(band["ugx_per_kg"]), + "mode": band["mode"], + "label": band["label"], + "disclaimer": TRANSPORT_DISCLAIMER, + } + last = TRANSPORT_RATE_BANDS[-1] + return { + "ugx_per_kg": float(last["ugx_per_kg"]), + "mode": last["mode"], + "label": last["label"], + "disclaimer": TRANSPORT_DISCLAIMER, + } diff --git a/services/soko-ml/location-service/tests/__init__.py b/services/soko-ml/location-service/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/services/soko-ml/location-service/tests/test_market_router.py b/services/soko-ml/location-service/tests/test_market_router.py new file mode 100644 index 0000000..a45d288 --- /dev/null +++ b/services/soko-ml/location-service/tests/test_market_router.py @@ -0,0 +1,182 @@ +""" +Unit + lightweight integration tests for the location-service. +DB/Redis-dependent tests are skipped when infrastructure is unavailable. +""" +import math +import os +import pytest +import pytest_asyncio + +pytestmark = pytest.mark.asyncio + + +# ── Haversine (pure function) ───────────────────────────────────────────────── + +from src.geo_recommender import _haversine + + +class TestHaversine: + def test_same_point_is_zero(self): + assert _haversine(0.0, 0.0, 0.0, 0.0) == 0.0 + + def test_kampala_to_gulu_approx(self): + # Kampala (0.3476, 32.5825) → Gulu (2.7747, 32.2990) ≈ 272 km by road + # Haversine (straight-line) is ~270 km + dist = _haversine(0.3476, 32.5825, 2.7747, 32.2990) + assert 250 < dist < 300, f"Expected ~270 km, got {dist}" + + def test_kampala_to_mbarara_approx(self): + # ~265 km straight line + dist = _haversine(0.3476, 32.5825, -0.6072, 30.6545) + assert 230 < dist < 300, f"Expected ~265 km, got {dist}" + + def test_symmetry(self): + d1 = _haversine(0.0, 30.0, 1.0, 31.0) + d2 = _haversine(1.0, 31.0, 0.0, 30.0) + assert abs(d1 - d2) < 0.01 + + +# ── Sell signal derivation (pure function) ──────────────────────────────────── + +from src.sell_signal import derive_signal + + +class TestSellSignal: + # derive_signal(crop, predictions, now=None) — actual function signature + + STABLE_PREDICTIONS = [ + {"predicted_price_ugx": 1000.0}, + {"predicted_price_ugx": 1010.0}, + {"predicted_price_ugx": 1005.0}, + {"predicted_price_ugx": 1008.0}, + ] + RISING_PREDICTIONS = [ + {"predicted_price_ugx": 1000.0}, + {"predicted_price_ugx": 1040.0}, + {"predicted_price_ugx": 1080.0}, + {"predicted_price_ugx": 1120.0}, + ] + FALLING_PREDICTIONS = [ + {"predicted_price_ugx": 1000.0}, + {"predicted_price_ugx": 970.0}, + {"predicted_price_ugx": 940.0}, + {"predicted_price_ugx": 910.0}, + ] + + def test_perishable_crop_always_sell_now(self): + signal = derive_signal("tomatoes", self.STABLE_PREDICTIONS) + assert signal["signal"] == "SELL_NOW_PERISHABLE" + assert "perishable" in signal["reason"].lower() + + def test_rising_trend_yields_wait(self): + # 12% rise over 4 weeks exceeds WAIT_THRESHOLD (10%) + signal = derive_signal("maize_grain", self.RISING_PREDICTIONS) + assert signal["signal"] == "WAIT" + + def test_falling_trend_yields_sell_now(self): + # 9% fall exceeds SELL_THRESHOLD (5%) + signal = derive_signal("maize_grain", self.FALLING_PREDICTIONS) + assert signal["signal"] == "SELL_NOW" + + def test_empty_predictions_returns_sell_now(self): + signal = derive_signal("maize_grain", []) + assert signal["signal"] == "SELL_NOW" + assert signal["confidence"] == "low" + + def test_signal_has_required_fields(self): + signal = derive_signal("sorghum", self.STABLE_PREDICTIONS) + assert "signal" in signal + assert "reason" in signal + assert "confidence" in signal + assert signal["signal"] in ("SELL_NOW", "WAIT", "SELL_NOW_PERISHABLE") + + +# ── Transport cost estimation (pure function) ───────────────────────────────── + +from src.transport_cost import estimate as estimate_transport + + +class TestTransportCost: + def test_short_haul_is_cheaper_than_long_haul(self): + short = estimate_transport(20.0) + long_ = estimate_transport(300.0) + assert short["ugx_per_kg"] < long_["ugx_per_kg"] + + def test_local_band_is_boda_cargo(self): + result = estimate_transport(10.0) + assert result["mode"] == "boda_cargo" + assert result["ugx_per_kg"] == 290.0 + + def test_medium_band_is_pickup(self): + result = estimate_transport(150.0) + assert result["mode"] == "pickup_truck" + assert result["ugx_per_kg"] == 620.0 + + def test_long_haul_band_is_shared_lorry(self): + result = estimate_transport(350.0) + assert result["mode"] == "shared_lorry" + assert result["ugx_per_kg"] == 850.0 + + def test_cross_region_above_400km(self): + result = estimate_transport(600.0) + assert result["mode"] == "cross_region" + assert result["ugx_per_kg"] == 1100.0 + + def test_rates_are_positive_ugx(self): + for km in [5, 50, 150, 350, 600]: + result = estimate_transport(float(km)) + assert result["ugx_per_kg"] > 0, f"Rate at {km}km should be positive UGX" + + def test_result_has_disclaimer(self): + result = estimate_transport(100.0) + assert "disclaimer" in result + assert "UGX" not in result["disclaimer"] or True # disclaimer is a plain string + + +# ── Market router integration (skipped if DB unavailable) ──────────────────── + +async def _db_available() -> bool: + try: + import asyncpg + dsn = os.getenv("POSTGRES_DSN", "postgresql://soko_ml:changeme@localhost:5432/soko_ml_db") + conn = await asyncpg.connect(dsn, timeout=2) + await conn.close() + return True + except Exception: + return False + + +@pytest.mark.asyncio +async def test_load_market_registry_returns_markets(): + if not await _db_available(): + pytest.skip("Postgres unreachable") + + from src.market_router import load_market_registry + markets = await load_market_registry() + assert isinstance(markets, list) + assert len(markets) >= 1 + for m in markets: + assert "market_id" in m + assert "lat" in m + assert "lng" in m + + +@pytest.mark.asyncio +async def test_route_returns_ranked_list_for_known_crop(): + if not await _db_available(): + pytest.skip("Postgres unreachable") + + from src.market_router import route + + result = await route( + farmer_id="F0001", + farmer_lat=0.3476, + farmer_lng=32.5825, + crop="maize_grain", + quantity_kg=500.0, + harvest_month=8, + ) + assert isinstance(result, dict) + assert "tier" in result + assert "ranked_markets" in result + assert isinstance(result["ranked_markets"], list) diff --git a/services/soko-ml/ml-gateway-service/src/main.py b/services/soko-ml/ml-gateway-service/src/main.py index 3c21106..4d2b8cf 100644 --- a/services/soko-ml/ml-gateway-service/src/main.py +++ b/services/soko-ml/ml-gateway-service/src/main.py @@ -7,7 +7,11 @@ from fastapi.responses import JSONResponse from .logger import RequestLoggingMiddleware -from .proxy import proxy_request, get_service_breaker_status, PRICE_SERVICE_URL, REC_SERVICE_URL +from .proxy import ( + proxy_request, get_service_breaker_status, + PRICE_SERVICE_URL, REC_SERVICE_URL, + LOCATION_SERVICE_URL, INGEST_SERVICE_URL, +) structlog.configure( processors=[ @@ -45,15 +49,21 @@ async def check_service(base_url: str) -> str: except Exception: return "unreachable" - price_status = await check_service(PRICE_SERVICE_URL) - rec_status = await check_service(REC_SERVICE_URL) - overall = "ok" if price_status == "ok" and rec_status == "ok" else "degraded" + price_status = await check_service(PRICE_SERVICE_URL) + rec_status = await check_service(REC_SERVICE_URL) + location_status = await check_service(LOCATION_SERVICE_URL) + ingest_status = await check_service(INGEST_SERVICE_URL) + + core_ok = price_status == "ok" and rec_status == "ok" + overall = "ok" if core_ok else "degraded" return { "gateway": overall, "services": { - "price-prediction": price_status, - "recommendation": rec_status, + "price-prediction": price_status, + "recommendation": rec_status, + "location": location_status, + "data-ingestion": ingest_status, }, "circuit_breakers": get_service_breaker_status(), } @@ -95,3 +105,51 @@ async def recommend_buyers(farmer_id: str, top_n: int = Query(default=5, ge=1, l url = f"{REC_SERVICE_URL}/recommend/buyers-for-farmer/{farmer_id}" result, status = await proxy_request(client, "GET", url, params={"top_n": top_n}) return JSONResponse(content=result, status_code=status) + + +# ── Location service routes ─────────────────────────────────────────────────── + +@app.post("/location/route") +async def location_route(request: Request): + body = await request.json() + client: httpx.AsyncClient = app.state.http_client + result, status = await proxy_request(client, "POST", f"{LOCATION_SERVICE_URL}/route", json_body=body) + return JSONResponse(content=result, status_code=status) + + +@app.post("/location/discover") +async def location_discover(request: Request): + body = await request.json() + client: httpx.AsyncClient = app.state.http_client + result, status = await proxy_request(client, "POST", f"{LOCATION_SERVICE_URL}/discover", json_body=body) + return JSONResponse(content=result, status_code=status) + + +# ── Data ingestion / gaps routes ────────────────────────────────────────────── + +@app.get("/gaps/summary") +async def gaps_summary(): + client: httpx.AsyncClient = app.state.http_client + result, status = await proxy_request(client, "GET", f"{INGEST_SERVICE_URL}/gaps/summary") + return JSONResponse(content=result, status_code=status) + + +@app.get("/coverage") +async def coverage(): + client: httpx.AsyncClient = app.state.http_client + result, status = await proxy_request(client, "GET", f"{INGEST_SERVICE_URL}/coverage") + return JSONResponse(content=result, status_code=status) + + +@app.post("/ingest/bootstrap") +async def ingest_bootstrap(request: Request): + client: httpx.AsyncClient = app.state.http_client + result, status = await proxy_request(client, "POST", f"{INGEST_SERVICE_URL}/bootstrap") + return JSONResponse(content=result, status_code=status) + + +@app.get("/ingest/status") +async def ingest_status(): + client: httpx.AsyncClient = app.state.http_client + result, status = await proxy_request(client, "GET", f"{INGEST_SERVICE_URL}/bootstrap/status") + return JSONResponse(content=result, status_code=status) diff --git a/services/soko-ml/ml-gateway-service/src/proxy.py b/services/soko-ml/ml-gateway-service/src/proxy.py index 5139fe7..e6b213b 100644 --- a/services/soko-ml/ml-gateway-service/src/proxy.py +++ b/services/soko-ml/ml-gateway-service/src/proxy.py @@ -9,8 +9,10 @@ log = structlog.get_logger() -PRICE_SERVICE_URL = os.getenv("PRICE_SERVICE_URL", "http://price-prediction-service:8001") -REC_SERVICE_URL = os.getenv("REC_SERVICE_URL", "http://recommendation-service:8002") +PRICE_SERVICE_URL = os.getenv("PRICE_SERVICE_URL", "http://price-prediction-service:8001") +REC_SERVICE_URL = os.getenv("REC_SERVICE_URL", "http://recommendation-service:8002") +LOCATION_SERVICE_URL = os.getenv("LOCATION_SERVICE_URL", "http://location-service:8003") +INGEST_SERVICE_URL = os.getenv("INGEST_SERVICE_URL", "http://data-ingestion-service:8004") MAX_RETRIES = 3 RETRY_BACKOFF_SECONDS = 0.5 @@ -75,13 +77,19 @@ def is_open(self) -> bool: # One breaker per downstream service, keyed by service name _breakers: dict[str, CircuitBreaker] = { "price-prediction": CircuitBreaker("price-prediction"), - "recommendation": CircuitBreaker("recommendation"), + "recommendation": CircuitBreaker("recommendation"), + "location": CircuitBreaker("location"), + "ingestion": CircuitBreaker("ingestion"), } def _get_breaker(url: str) -> CircuitBreaker: if PRICE_SERVICE_URL in url: return _breakers["price-prediction"] + if LOCATION_SERVICE_URL in url: + return _breakers["location"] + if INGEST_SERVICE_URL in url: + return _breakers["ingestion"] return _breakers["recommendation"] @@ -134,17 +142,29 @@ async def proxy_request( def _fallback_response(url: str) -> dict: if "price" in url: return { - "error": "price-prediction-service unavailable", - "message": "Price prediction is temporarily unavailable. Please try again later.", - "cached": False, + "error": "price-prediction-service unavailable", + "message": "Price prediction is temporarily unavailable. Please try again later.", + "cached": False, "predictions": [], } + if "location" in url or "route" in url or "discover" in url: + return { + "error": "location-service unavailable", + "message": "Market routing is temporarily unavailable. Please try again later.", + "tier": 0, + "ranked_markets": [], + } + if "ingest" in url or "gaps" in url or "coverage" in url or "bootstrap" in url: + return { + "error": "data-ingestion-service unavailable", + "message": "Data ingestion service is temporarily unavailable.", + } return { - "error": "recommendation-service unavailable", - "message": "Recommendations are temporarily unavailable. Please try again later.", - "cached": False, + "error": "recommendation-service unavailable", + "message": "Recommendations are temporarily unavailable. Please try again later.", + "cached": False, "recommended_farmers": [], - "recommended_buyers": [], + "recommended_buyers": [], } diff --git a/services/soko-ml/price-prediction-service/src/feature_store_client.py b/services/soko-ml/price-prediction-service/src/feature_store_client.py new file mode 100644 index 0000000..83d6cbc --- /dev/null +++ b/services/soko-ml/price-prediction-service/src/feature_store_client.py @@ -0,0 +1,77 @@ +""" +asyncpg client for reading price_observations from soko_ml_db. +Used by the training pipeline to load real transaction data instead of CSV files. +""" +import logging +import os +from typing import Optional + +import asyncpg +import pandas as pd + +log = logging.getLogger(__name__) + +POSTGRES_DSN = os.getenv("POSTGRES_DSN", "postgresql://soko_ml:changeme@soko-ml-db:5432/soko_ml_db") +MIN_OBSERVATIONS = int(os.getenv("MIN_OBSERVATIONS_FOR_MODEL", "30")) + +_pool: Optional[asyncpg.Pool] = None + + +async def get_pool() -> asyncpg.Pool: + global _pool + if _pool is None: + _pool = await asyncpg.create_pool(POSTGRES_DSN, min_size=1, max_size=5) + return _pool + + +async def close_pool() -> None: + global _pool + if _pool: + await _pool.close() + _pool = None + + +async def fetch_training_data(market: str, crop: str) -> tuple[pd.DataFrame, str]: + """ + Returns (DataFrame with columns [ds, y], source_label). + source_label is 'soko_order' when using real data, 'farmgain_seed' when falling back. + Falls back to seed data if fewer than MIN_OBSERVATIONS real records exist. + """ + pool = await get_pool() + async with pool.acquire() as conn: + rows = await conn.fetch( + """ + SELECT observed_at AS ds, price_per_kg AS y + FROM price_observations + WHERE market = $1 AND crop = $2 + AND source = 'soko_order' + ORDER BY observed_at ASC + """, + market, crop, + ) + + if len(rows) >= MIN_OBSERVATIONS: + df = pd.DataFrame([{"ds": r["ds"], "y": float(r["y"])} for r in rows]) + log.info(f"Training data loaded from feature store: {market}/{crop} ({len(df)} rows)") + return df, "soko_order" + + if len(rows) > 0: + log.warning( + f"Insufficient real data for {market}/{crop}, using seed fallback " + f"({len(rows)} observations — need {MIN_OBSERVATIONS})" + ) + else: + log.warning(f"No real data for {market}/{crop}, using seed fallback") + + return pd.DataFrame(), "farmgain_seed" + + +async def get_coverage_status(market: str, crop: str) -> Optional[dict]: + """Returns coverage_map row for this pair, or None if not tracked.""" + pool = await get_pool() + async with pool.acquire() as conn: + row = await conn.fetchrow( + "SELECT observation_count, is_model_ready FROM coverage_map WHERE crop = $1 AND market = $2", + crop, market, + ) + return dict(row) if row else None diff --git a/services/soko-ml/price-prediction-service/src/predictor.py b/services/soko-ml/price-prediction-service/src/predictor.py index 6ed7fb3..678e3d2 100644 --- a/services/soko-ml/price-prediction-service/src/predictor.py +++ b/services/soko-ml/price-prediction-service/src/predictor.py @@ -124,9 +124,60 @@ def _fallback_predict(self, base_price: float, weeks_ahead: int) -> list[dict]: return results -def train_all_models() -> None: - """Train one Prophet model per market–crop pair and save as .pkl. +async def train_all_models_from_feature_store() -> None: + """ + Train one Prophet model per market–crop pair using soko_ml_db as the data source. + Falls back to synthetic seed data for pairs with insufficient real observations. + Replaces the CSV-based train_all_models() — call this from the retrain handler. + """ + try: + from prophet import Prophet + except ImportError: + log.error("prophet_not_installed") + return + + from .feature_store_client import fetch_training_data + + model_dir = Path(os.getenv("MODEL_DIR", "models")) + model_dir.mkdir(parents=True, exist_ok=True) + + for market in SUPPORTED_MARKETS: + for crop in SUPPORTED_CROPS: + key = f"{market}__{crop}" + pkl_path = model_dir / f"{key}.pkl" + + df, source = await fetch_training_data(market, crop) + + if source == "farmgain_seed" or df.empty: + df = _generate_pair_data(market, crop).rename( + columns={"date": "ds", "price_ugx": "y"} + )[["ds", "y"]] + + df["ds"] = pd.to_datetime(df["ds"]) + df = df.dropna() + if len(df) < 10: + log.warning("insufficient_data_skip", key=key, rows=len(df)) + continue + m = Prophet( + yearly_seasonality=True, + weekly_seasonality=False, + daily_seasonality=False, + seasonality_mode="multiplicative", + interval_width=0.80, + ) + m.add_seasonality(name="uganda_bimodal", period=26, fourier_order=5) + m.fit(df) + + with open(pkl_path, "wb") as f: + pickle.dump(m, f) + log.info("model_trained", key=key, rows=len(df), source=source, path=str(pkl_path)) + + +def train_all_models() -> None: + """ + Legacy CSV-based trainer — retained for cold-start bootstrap only. + In normal operation, use train_all_models_from_feature_store(). Reads crop_prices_raw.csv if available; otherwise generates inline data. """ try: @@ -138,14 +189,13 @@ def train_all_models() -> None: model_dir = Path(os.getenv("MODEL_DIR", "models")) model_dir.mkdir(parents=True, exist_ok=True) - # DATA_DIR can be set to an absolute path; default looks sibling to this service - data_dir = Path(os.getenv("DATA_DIR", "../recommendation-service/data/raw")) + data_dir = Path(os.getenv("DATA_DIR", "../recommendation-service/data/raw")) data_path = data_dir / "crop_prices_raw.csv" - df_all = pd.read_csv(data_path) if data_path.exists() else _generate_training_data() + df_all = pd.read_csv(data_path) if data_path.exists() else _generate_training_data() for market in SUPPORTED_MARKETS: for crop in SUPPORTED_CROPS: - key = f"{market}__{crop}" + key = f"{market}__{crop}" pkl_path = model_dir / f"{key}.pkl" if pkl_path.exists(): log.info("model_exists_skipping", key=key) @@ -166,7 +216,6 @@ def train_all_models() -> None: seasonality_mode="multiplicative", interval_width=0.80, ) - # Uganda bimodal seasonality — 26-week (half-year) cycle m.add_seasonality(name="uganda_bimodal", period=26, fourier_order=5) m.fit(df) diff --git a/services/soko-ml/recommendation-service/requirements.txt b/services/soko-ml/recommendation-service/requirements.txt index 733e166..f4ac724 100644 --- a/services/soko-ml/recommendation-service/requirements.txt +++ b/services/soko-ml/recommendation-service/requirements.txt @@ -2,6 +2,7 @@ fastapi==0.111.0 uvicorn[standard]==0.29.0 pydantic==2.7.1 pydantic-settings==2.2.1 +asyncpg==0.29.0 redis[asyncio]==5.0.4 confluent-kafka==2.4.0 pandas==2.2.2 diff --git a/services/soko-ml/recommendation-service/src/feature_store_client.py b/services/soko-ml/recommendation-service/src/feature_store_client.py new file mode 100644 index 0000000..b68953a --- /dev/null +++ b/services/soko-ml/recommendation-service/src/feature_store_client.py @@ -0,0 +1,122 @@ +""" +asyncpg client for reading farmer_features and buyer_features from soko_ml_db. +Replaces CSV file loading in the recommendation-service. +""" +import logging +import os +from typing import Optional + +import asyncpg +import pandas as pd + +log = logging.getLogger(__name__) + +POSTGRES_DSN = os.getenv("POSTGRES_DSN", "postgresql://soko_ml:changeme@soko-ml-db:5432/soko_ml_db") + +_pool: Optional[asyncpg.Pool] = None + + +async def get_pool() -> asyncpg.Pool: + global _pool + if _pool is None: + _pool = await asyncpg.create_pool(POSTGRES_DSN, min_size=2, max_size=8) + return _pool + + +async def close_pool() -> None: + global _pool + if _pool: + await _pool.close() + _pool = None + + +async def load_farmers() -> pd.DataFrame: + """ + Loads all farmer_features rows into a DataFrame. + Returns empty DataFrame (with correct columns) if the table is empty. + """ + pool = await get_pool() + async with pool.acquire() as conn: + rows = await conn.fetch( + """ + SELECT + farmer_id AS id, + name AS full_name, + district, + lat, lng, + crops_offered AS specialties, + markets_served, + avg_rating AS average_rating, + fulfillment_rate, + avg_response_time_hrs, + total_orders_completed AS total_sales, + total_orders_cancelled, + total_listings, + last_active_at + FROM farmer_features + """ + ) + + if not rows: + log.warning("farmer_features table is empty — recommendations will be empty until bootstrap runs") + return pd.DataFrame(columns=[ + "id", "full_name", "district", "lat", "lng", + "specialties", "markets_served", "average_rating", + "fulfillment_rate", "avg_response_time_hrs", + "total_sales", "total_listings", + ]) + + records = [] + for r in rows: + rec = dict(r) + # asyncpg returns TEXT[] as list already + rec["specialties"] = rec.get("specialties") or [] + rec["markets_served"] = rec.get("markets_served") or [] + records.append(rec) + + df = pd.DataFrame(records) + log.info(f"farmers_loaded_from_feature_store count={len(df)}") + return df + + +async def load_buyers() -> pd.DataFrame: + """ + Loads all buyer_features rows into a DataFrame. + Returns empty DataFrame if the table is empty. + """ + pool = await get_pool() + async with pool.acquire() as conn: + rows = await conn.fetch( + """ + SELECT + buyer_id AS id, + name AS full_name, + district, + lat, lng, + preferred_crops AS interests, + preferred_markets, + avg_spend_per_order AS total_spent, + payment_reliability, + total_purchases AS total_orders, + last_active_at + FROM buyer_features + """ + ) + + if not rows: + log.warning("buyer_features table is empty — buyer recommendations will be empty") + return pd.DataFrame(columns=[ + "id", "full_name", "district", "lat", "lng", + "interests", "total_spent", "payment_reliability", "total_orders", + ]) + + records = [] + for r in rows: + rec = dict(r) + rec["interests"] = rec.get("interests") or [] + rec["preferred_markets"] = rec.get("preferred_markets") or [] + records.append(rec) + + df = pd.DataFrame(records) + log.info(f"buyers_loaded_from_feature_store count={len(df)}") + return df diff --git a/services/soko-ml/recommendation-service/src/geo_filter.py b/services/soko-ml/recommendation-service/src/geo_filter.py new file mode 100644 index 0000000..4878c6c --- /dev/null +++ b/services/soko-ml/recommendation-service/src/geo_filter.py @@ -0,0 +1,86 @@ +""" +Haversine pre-filter for the recommendation-service. +Eliminates farmers/buyers that are too far away before the scoring step, +reducing the scoring set to a manageable size when profile counts grow large. +""" +import math +from typing import Optional + +import pandas as pd + +EARTH_RADIUS_KM = 6371.0 + + +def haversine_km(lat1: float, lng1: float, lat2: float, lng2: float) -> float: + """Returns the great-circle distance in km between two GPS coordinates.""" + phi1, phi2 = math.radians(lat1), math.radians(lat2) + dphi = math.radians(lat2 - lat1) + dlambda = math.radians(lng2 - lng1) + a = math.sin(dphi / 2) ** 2 + math.cos(phi1) * math.cos(phi2) * math.sin(dlambda / 2) ** 2 + return 2 * EARTH_RADIUS_KM * math.atan2(math.sqrt(a), math.sqrt(1 - a)) + + +def filter_by_distance( + df: pd.DataFrame, + origin_lat: float, + origin_lng: float, + max_km: float, + lat_col: str = "lat", + lng_col: str = "lng", + relax_factor: float = 1.5, +) -> pd.DataFrame: + """ + Returns a filtered DataFrame containing only rows within max_km * relax_factor + of the origin coordinates. + + relax_factor is applied to avoid cutting off profiles at the exact boundary + when coordinate precision is low (district centroids can be ~30km off). + + Rows with NULL lat/lng are always included — they won't be excluded because + their location is unknown, not because they're far away. + """ + if df.empty: + return df + + threshold = max_km * relax_factor + + def _within(row) -> bool: + lat = row.get(lat_col) + lng = row.get(lng_col) + if lat is None or lng is None or (lat == 0 and lng == 0): + return True # unknown location — include by default + try: + return haversine_km(origin_lat, origin_lng, float(lat), float(lng)) <= threshold + except Exception: + return True + + mask = df.apply(_within, axis=1) + return df[mask].copy() + + +def add_distance_column( + df: pd.DataFrame, + origin_lat: float, + origin_lng: float, + lat_col: str = "lat", + lng_col: str = "lng", + distance_col: str = "distance_km", +) -> pd.DataFrame: + """Adds a distance_km column to the DataFrame. Rows with NULL coords get NaN.""" + if df.empty: + df[distance_col] = pd.Series(dtype=float) + return df + + def _dist(row) -> Optional[float]: + lat = row.get(lat_col) + lng = row.get(lng_col) + if lat is None or lng is None: + return None + try: + return round(haversine_km(origin_lat, origin_lng, float(lat), float(lng)), 1) + except Exception: + return None + + df = df.copy() + df[distance_col] = df.apply(_dist, axis=1) + return df diff --git a/services/soko-ml/recommendation-service/src/main.py b/services/soko-ml/recommendation-service/src/main.py index 229b9ca..3ca79da 100644 --- a/services/soko-ml/recommendation-service/src/main.py +++ b/services/soko-ml/recommendation-service/src/main.py @@ -1,3 +1,4 @@ +import asyncio import os from contextlib import asynccontextmanager @@ -9,6 +10,7 @@ get_cached_farmers, set_cached_farmers, get_cached_buyers, set_cached_buyers, ) +from .feature_store_client import close_pool from .interaction_store import InteractionStore from .kafka_consumer import InteractionConsumer from .recommender import ProfileStore, Recommender @@ -26,34 +28,53 @@ ) log = structlog.get_logger() -FARMERS_PATH = os.getenv("FARMERS_DATA_PATH", "/app/data/raw/farmers.csv") -BUYERS_PATH = os.getenv("BUYERS_DATA_PATH", "/app/data/raw/buyers.csv") -DEFAULT_TOP_N = int(os.getenv("DEFAULT_TOP_N", "5")) -SERVICE_NAME = os.getenv("SERVICE_NAME", "recommendation-service") +DEFAULT_TOP_N = int(os.getenv("DEFAULT_TOP_N", "5")) +SERVICE_NAME = os.getenv("SERVICE_NAME", "recommendation-service") +PROFILE_REFRESH_INTERVAL = int(os.getenv("PROFILE_REFRESH_INTERVAL_SECONDS", "900")) @asynccontextmanager async def lifespan(app: FastAPI): - profile_store = ProfileStore(FARMERS_PATH, BUYERS_PATH) - n_farmers, n_buyers = profile_store.load() + profile_store = ProfileStore() + + # Initial load — exit if Postgres is unreachable (service cannot function without profiles) + try: + n_farmers, n_buyers = await profile_store.reload() + except Exception as exc: + log.critical(f"Cannot load profiles from feature store at startup: {exc}") + raise SystemExit(1) interaction_store = InteractionStore() - redis_client = await get_redis_client() - recommender = Recommender(profile_store, interaction_store) + redis_client = await get_redis_client() + recommender = Recommender(profile_store, interaction_store) consumer = InteractionConsumer(interaction_store) consumer.start() - app.state.recommender = recommender - app.state.redis = redis_client + app.state.recommender = recommender + app.state.redis = redis_client app.state.profile_store = profile_store - app.state.consumer = consumer + app.state.consumer = consumer log.info("recommendation_service_started", farmers=n_farmers, buyers=n_buyers) + + # Periodic reload task + async def _reload_loop(): + while True: + await asyncio.sleep(PROFILE_REFRESH_INTERVAL) + try: + await profile_store.reload() + except Exception as exc: + log.warning(f"Periodic profile reload failed: {exc}") + + reload_task = asyncio.create_task(_reload_loop()) + yield + reload_task.cancel() consumer.stop() await redis_client.aclose() + await close_pool() log.info("recommendation_service_stopped") diff --git a/services/soko-ml/recommendation-service/src/recommender.py b/services/soko-ml/recommendation-service/src/recommender.py index 8ee9b8c..c2ce0dc 100644 --- a/services/soko-ml/recommendation-service/src/recommender.py +++ b/services/soko-ml/recommendation-service/src/recommender.py @@ -1,5 +1,3 @@ -import os -from pathlib import Path from typing import Optional import pandas as pd @@ -10,13 +8,12 @@ log = structlog.get_logger() -def _parse_list_field(value) -> list[str]: - """Parse a comma-separated string or pass-through a list.""" - if pd.isna(value) or value == "": - return [] +def _parse_list_field(value) -> list: if isinstance(value, list): return value - return [v.strip() for v in str(value).split(",") if v.strip()] + if not value: + return [] + return [item.strip() for item in str(value).split(",") if item.strip()] def _safe_int(value, default: int = 0) -> int: @@ -35,34 +32,29 @@ def _safe_float(value, default: float = 0.0) -> float: class ProfileStore: """ - Loads farmer and buyer profile DataFrames from CSV at startup. - CSV columns mirror UserProfile / FarmerStats / BuyerStats field names - so real user data can be swapped in without transformation. + Holds farmer and buyer DataFrames loaded from the ML feature store (Postgres). + Call reload() to refresh from the database without restarting the service. """ - def __init__(self, farmers_path: str, buyers_path: str): - self._farmers_path = farmers_path - self._buyers_path = buyers_path + def __init__(self) -> None: self.farmers: pd.DataFrame = pd.DataFrame() - self.buyers: pd.DataFrame = pd.DataFrame() + self.buyers: pd.DataFrame = pd.DataFrame() - def load(self) -> tuple[int, int]: + async def reload(self) -> tuple[int, int]: + """Load or refresh profiles from soko_ml_db. Never raises.""" + from .feature_store_client import load_farmers, load_buyers try: - self.farmers = pd.read_csv(self._farmers_path) - # specialties: comma-sep crops grown (UserProfile.specialties) - self.farmers["specialties"] = self.farmers["specialties"].apply(_parse_list_field) - log.info("farmers_loaded", count=len(self.farmers)) - except FileNotFoundError: - log.warning("farmers_file_not_found", path=self._farmers_path) + self.farmers = await load_farmers() + except Exception as exc: + log.error(f"Failed to load farmers from feature store: {exc}") + # Keep existing data rather than going empty try: - self.buyers = pd.read_csv(self._buyers_path) - # interests: comma-sep preferred crops (UserProfile.interests) - self.buyers["interests"] = self.buyers["interests"].apply(_parse_list_field) - log.info("buyers_loaded", count=len(self.buyers)) - except FileNotFoundError: - log.warning("buyers_file_not_found", path=self._buyers_path) + self.buyers = await load_buyers() + except Exception as exc: + log.error(f"Failed to load buyers from feature store: {exc}") + log.info("profiles_reloaded", farmers=len(self.farmers), buyers=len(self.buyers)) return len(self.farmers), len(self.buyers) def get_farmer(self, farmer_id: str) -> Optional[pd.Series]: diff --git a/services/soko-ml/shared/events.py b/services/soko-ml/shared/events.py index 362e451..b08126c 100644 --- a/services/soko-ml/shared/events.py +++ b/services/soko-ml/shared/events.py @@ -153,6 +153,97 @@ def from_dict(cls, data: dict) -> "MLEvent": ) +@dataclass +class SokoTransactionEvent: + """ + Published by order-service to soko.transactions. + Extends TransactionEvent with product_name for crop-level normalisation. + """ + event_type: str # purchase_completed | purchase_cancelled + order_id: str + buyer_id: str + farmer_id: str + crop: str # raw category value from listing (e.g. "Grains") + product_name: str # specific product name (e.g. "Maize") — use for normalisation + market: str # delivery district (e.g. "Kampala") + quantity_kg: float + price_per_kg_ugx: float + total_ugx: float + timestamp: str = field(default_factory=lambda: datetime.utcnow().isoformat() + "Z") + + @classmethod + def from_dict(cls, data: dict) -> "SokoTransactionEvent": + return cls( + event_type=data["event_type"], + order_id=data.get("order_id", ""), + buyer_id=data["buyer_id"], + farmer_id=data["farmer_id"], + crop=data.get("crop", ""), + product_name=data.get("product_name", ""), + market=data.get("market", ""), + quantity_kg=float(data.get("quantity_kg", 0)), + price_per_kg_ugx=float(data.get("price_per_kg_ugx", 0)), + total_ugx=float(data.get("total_ugx", 0)), + timestamp=data.get("timestamp", datetime.utcnow().isoformat() + "Z"), + ) + + +@dataclass +class CoverageGapEvent: + """Published to soko.ml.events when a Tier 3 crop is requested.""" + event_type: str = "crop_coverage_gap" + crop_submitted: str = "" + category_guess: str = "" + farmer_id: str = "" + frequency: int = 1 + priority: str = "low" + timestamp: str = field(default_factory=lambda: datetime.utcnow().isoformat() + "Z") + + def to_json(self) -> str: + return json.dumps({ + "event_type": self.event_type, + "crop_submitted": self.crop_submitted, + "category_guess": self.category_guess, + "farmer_id": self.farmer_id, + "frequency": self.frequency, + "priority": self.priority, + "timestamp": self.timestamp, + }) + + @classmethod + def from_dict(cls, data: dict) -> "CoverageGapEvent": + return cls( + event_type=data.get("event_type", "crop_coverage_gap"), + crop_submitted=data.get("crop_submitted", ""), + category_guess=data.get("category_guess", ""), + farmer_id=data.get("farmer_id", ""), + frequency=data.get("frequency", 1), + priority=data.get("priority", "low"), + timestamp=data.get("timestamp", datetime.utcnow().isoformat() + "Z"), + ) + + +@dataclass +class RetrainRequestedEvent: + """Published to soko.ml.events when a market-crop pair reaches 52 real observations.""" + event_type: str = "retrain_requested" + market: str = "" + crop: str = "" + reason: str = "" + data_source: str = "soko_order" + timestamp: str = field(default_factory=lambda: datetime.utcnow().isoformat() + "Z") + + def to_json(self) -> str: + return json.dumps({ + "event_type": self.event_type, + "market": self.market, + "crop": self.crop, + "reason": self.reason, + "data_source": self.data_source, + "timestamp": self.timestamp, + }) + + @dataclass class DLQEvent: original_topic: str diff --git a/services/user/app/routers/profile.py b/services/user/app/routers/profile.py index eb1ae0f..f43885c 100644 --- a/services/user/app/routers/profile.py +++ b/services/user/app/routers/profile.py @@ -90,6 +90,29 @@ def get_farmers( for farmer in farmers ] +@router.get("/buyers", response_model=list[AuthenticatedUser]) +def get_buyers( + district: Optional[str] = Query(default=None), + page: int = Query(default=1, ge=1), + limit: int = Query(default=100, le=500), + db: Session = Depends(get_db) +): + """ + Returns all users with role buyer or both. + Used by data-ingestion-service bootstrap to populate buyer_features table. + """ + from app.helpers.builders import build_authenticated_user + q = db.query(UserProfile).filter( + UserProfile.role.in_([UserRole.buyer, UserRole.both]) + ) + if district: + q = q.filter(UserProfile.district == district) + + buyers = q.order_by(UserProfile.created_at.desc()) \ + .offset((page - 1) * limit).limit(limit).all() + return [build_authenticated_user(b) for b in buyers] + + @router.get("/{user_id}", response_model=FarmerProfile) def get_farmer_profile(user_id: str, db: Session = Depends(get_db)): try: From 329d508c3f2850e6f2d67ab57d93b09c2f9e9d9c Mon Sep 17 00:00:00 2001 From: the-icemann Date: Fri, 15 May 2026 02:47:59 +0300 Subject: [PATCH 02/24] Ideally the final build for all integrated changes in conjungtion with the last commit --- CONTRACTS.md | 207 ---- CONTRIBUTING.md | 73 -- Makefile | 14 +- NGINX_502_FIX.md | 73 -- PHASE2_ANALYSIS.md | 424 ------- README.md | 1084 ++++++++++------- execution.md | 237 ---- nginx/nginx.conf | 11 + scripts/seed.py | 709 +++++++++++ scripts/smoke_test.py | 138 +++ services/soko-ml/.env.example | 4 +- .../src/clients/order_client.py | 2 +- .../src/clients/user_client.py | 2 +- .../data-ingestion-service/src/main.py | 25 +- services/soko-ml/docker-compose.yml | 16 +- .../soko-ml/ml-gateway-service/src/main.py | 28 +- .../soko-ml/ml-gateway-service/src/proxy.py | 5 +- .../recommendation-service/src/main.py | 51 +- 18 files changed, 1595 insertions(+), 1508 deletions(-) delete mode 100644 CONTRACTS.md delete mode 100644 CONTRIBUTING.md delete mode 100644 NGINX_502_FIX.md delete mode 100644 PHASE2_ANALYSIS.md delete mode 100644 execution.md create mode 100644 scripts/seed.py create mode 100644 scripts/smoke_test.py diff --git a/CONTRACTS.md b/CONTRACTS.md deleted file mode 100644 index eee3330..0000000 --- a/CONTRACTS.md +++ /dev/null @@ -1,207 +0,0 @@ -# Soko – Service Contracts - -Defines the interfaces between services — HTTP and RabbitMQ. -Update this file when adding or changing any cross-service interface. - ---- - -## RabbitMQ Event Payloads - -All messages follow the envelope `{ "event": "", "data": { ... } }`. -All queues are declared **durable**. - ---- - -### `farmer.registered` -Published by: **Farmer service** after profile creation. - -```json -{ - "event": "farmer.registered", - "data": { - "farmer_id": 1, - "user_id": 1, - "district": "Kampala" - } -} -``` - ---- - -### `farm.created` -Published by: **Farmer service** after a farm is added. - -```json -{ - "event": "farm.created", - "data": { - "farm_id": 1, - "farmer_id": 1, - "location": "Mukono, Central Uganda" - } -} -``` - ---- - -### `produce.listed` -Published by: **Produce service** after a listing is created. -Consumed by: **Recommendation service** (stores `ProduceSummary`). - -```json -{ - "event": "produce.listed", - "data": { - "produce_id": 3, - "farmer_id": 1, - "name": "Maize", - "category": "grains", - "district": "Kampala", - "price_per_unit": 1200.0, - "unit": "kg" - } -} -``` - ---- - -### `order.placed` -Published by: **Buyer service** after an order is committed to the DB. -Consumed by: **Recommendation service** (stores `OrderEvent`, used to exclude already-ordered produce). - -> **Important:** `buyer_id` is the **auth user ID** (JWT `sub`), not the buyer profile ID. -> `farmer_id` is the **auth user ID** of the farmer, not the farmer profile ID. - -```json -{ - "event": "order.placed", - "data": { - "order_id": 1, - "produce_id": 3, - "farmer_id": 1, - "buyer_id": 2, - "quantity_kg": 5.0, - "total_price": 6000.0 - } -} -``` - ---- - -### `order.completed` -Published by: **Buyer service** when a farmer marks an order as `completed`. -Consumed by: — (reserved for notification service). - -```json -{ - "event": "order.completed", - "data": { - "order_id": 1, - "produce_id": 3, - "farmer_id": 1, - "quantity_kg": 5.0, - "total_price": 6000.0 - } -} -``` - ---- - -### `quality.scored` -Published by: **Buyer service** when a buyer submits a review. -Consumed by: -- **Produce service** — updates `avg_rating` and `review_count` on the listing -- **Recommendation service** — stores `QualityScore` - -> **Note:** `buyer_id` here is the buyer **profile ID** (not auth user ID). - -```json -{ - "event": "quality.scored", - "data": { - "order_id": 1, - "produce_id": 3, - "farmer_id": 1, - "buyer_id": 1, - "stars": 4, - "comment": "Great tomatoes" - } -} -``` - ---- - -## HTTP Service-to-Service Calls - -### Buyer → Produce -Used during order placement to verify stock and reserve it. - -| Method | Endpoint | Purpose | -|---|---|---| -| GET | `/produce/{id}` | Check listing exists, price, available quantity | -| PATCH | `/produce/{id}/reduce-stock` | Deduct ordered quantity | - -`reduce-stock` payload: -```json -{ "quantity": 5.0 } -``` - -### Produce → Farmer -Used at listing creation to denormalise the farmer's display name. - -| Method | Endpoint | Purpose | -|---|---|---| -| GET | `/farmers/by-user/{user_id}` | Retrieve farmer name for storage on listing | - ---- - -## Produce Listing Response Shape - -Returned by `GET /produce/` and `GET /produce/{id}`. Frontend-relevant fields: - -```json -{ - "id": 3, - "farmer_id": 1, - "farmer_name": "John Mukasa", - "name": "Maize", - "description": "Fresh dry maize", - "category": "grains", - "unit": "kg", - "quantity": 95.0, - "price_per_unit": 1200.0, - "district": "Kampala", - "is_available": true, - "avg_rating": 4.2, - "review_count": 5, - "created_at": "2026-03-23T08:00:00Z", - "updated_at": null -} -``` - ---- - -## Order Status Transitions - -| From | To | Actor | -|---|---|---| -| `pending` | `confirmed` | Farmer | -| `pending` | `rejected` | Farmer | -| `pending` | `cancelled` | Buyer | -| `confirmed` | `completed` | Farmer | - -Only `completed` orders can receive a review. - ---- - -## Recommendation Scoring - -Scores are computed at request time (not stored) from `ProduceSummary` and `QualityScore` tables: - -| Signal | Weight | -|---|---| -| Matching category (based on order history) | 0.4 | -| Matching district | 0.3 | -| Average star rating (normalised 0–1) | 0.3 | - -Produce the buyer has already ordered is excluded from results. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index ad2389a..0000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,73 +0,0 @@ -# Contributing to Soko - -## Before you start - -- All changes must go through a pull request — direct pushes to `main` are blocked. -- Your PR must pass both CI checks (`lint` and `integration-tests`) and receive at least one approval before it can be merged. - -## Services overview - -| Service | Port | Responsibility | -|---|---|---| -| auth | 8001 | Registration, login, JWT | -| farmer | 8002 | Farmer profiles | -| buyer | 8003 | Buyer profiles, orders, reviews | -| produce | 8004 | Produce listings | -| recommendation | 8005 | Personalised recommendations | - -Each service lives in `services//` and has its own database. Services communicate via RabbitMQ events and authenticate requests using the shared JWT secret. - -## Running locally - -**Requirements:** Docker Desktop (with Compose v2), Python 3.11+ - -```bash -# Start everything -docker compose up --build -d - -# Verify all services are up -curl http://localhost:8001/health -curl http://localhost:8004/health -``` - -To rebuild a single service after editing its code: - -```bash -docker compose up --build -d -# e.g. docker compose up --build -d auth_service -``` - -## Running the tests - -```bash -pip install -r tests/integration/requirements.txt -python -m pytest tests/integration/ -v --tb=short -``` - -Tests expect all services to be running locally on their default ports. The full suite takes around 2–3 minutes because some tests poll for async RabbitMQ events. - -## Linting - -We use [ruff](https://docs.astral.sh/ruff/). Run it before pushing: - -```bash -pip install ruff -ruff check services/auth/app services/farmer/app services/buyer/app services/produce/app services/recommendation/app -``` - -CI will fail on any lint error. - -## Pull request checklist - -- [ ] `ruff check` passes with no errors -- [ ] All integration tests pass locally (`pytest tests/integration/`) -- [ ] New endpoints have a corresponding integration test -- [ ] Changes to the auth service (JWT shape, user fields) are coordinated with all other services — they all decode the same token - -## User ID convention - -All inter-service user identifiers are UUIDs (strings), sourced from the auth service. Never store or accept bare integer user IDs across service boundaries. - -## Event contracts - -RabbitMQ event schemas are documented in [CONTRACTS.md](CONTRACTS.md). If you change a published event's shape, update that file and notify the team — consumers in other services will break silently otherwise. diff --git a/Makefile b/Makefile index d59fc38..9646f5d 100644 --- a/Makefile +++ b/Makefile @@ -507,16 +507,12 @@ health: @echo "=== Data Ingestion Service ===" && \ curl -sf http://localhost:8096/health | python3 -m json.tool || echo "UNREACHABLE" +seed: + @echo "Seeding Ugandan dummy data into all service databases..." + @python3 scripts/seed.py + smoke-test: - @echo "=== Smoke: Price Prediction ===" - @curl -sf -X POST http://localhost:8080/price/predict \ - -H 'Content-Type: application/json' \ - -d '{"market":"Kisenyi_Kampala","crop":"maize_grain","weeks_ahead":4}' \ - | python3 -m json.tool - @echo "=== Smoke: Farmers for Buyer ===" - @curl -sf "http://localhost:8080/recommend/farmers-for-buyer/B0001?top_n=3" | python3 -m json.tool - @echo "=== Smoke: Buyers for Farmer ===" - @curl -sf "http://localhost:8080/recommend/buyers-for-farmer/F0001?top_n=3" | python3 -m json.tool + @python3 scripts/smoke_test.py smoke-route: @echo "=== Smoke: Market Route (farmer sell signal) ===" diff --git a/NGINX_502_FIX.md b/NGINX_502_FIX.md deleted file mode 100644 index 4a9b1f3..0000000 --- a/NGINX_502_FIX.md +++ /dev/null @@ -1,73 +0,0 @@ -# nginx 502 Bad Gateway — Root Cause & Fix - -## What's happening - -When `make core-up` rebuilds and restarts backend containers (e.g. `user_service`, `produce_service`), those containers can come up with different Docker-assigned IPs than they had before. nginx resolves static `upstream` block hostnames **once at startup** and caches the IPs — it does not re-resolve them when upstream containers restart. - -Result: nginx sends `user_service` requests to produce_service's old IP and vice versa → `Connection refused (111)` → **502 Bad Gateway**. - -Confirmed from nginx error log: -``` -connect() failed (111) upstream: "http://172.20.0.12:8002/docs" ← /users/ route -connect() failed (111) upstream: "http://172.20.0.15:8003/docs" ← /listings/ route -``` -But docker network shows `produce_service=172.20.0.12` and `user_service=172.20.0.15` — the ports are crossed because nginx cached the pre-rebuild IPs. - ---- - -## Quick fix (immediate) - -Restart nginx so it re-resolves all upstream hostnames: - -```bash -docker restart api_gateway -``` - -Also add this to `make core-up` so it always reloads nginx after rebuilding services: - -```makefile -core-up: - $(COMPOSE_CORE) up --build -d - @docker restart api_gateway # re-resolve upstream IPs after container rebuilds - ... -``` - ---- - -## Proper fix (nginx config) - -Convert all static `upstream` blocks in `nginx/nginx.conf` to variable-based resolution, the same pattern already used for the ML gateway. Variable-based `proxy_pass` forces nginx to re-query the Docker DNS resolver (`127.0.0.11`) on every request. - -**Current (broken on rebuild):** -```nginx -upstream user_service { server user_service:8002; } - -location /users/ { - proxy_pass http://user_service/; -} -``` - -**Fixed:** -```nginx -# Remove the upstream block entirely, use a variable instead: - -location /users/ { - set $user_svc "user_service:8002"; - proxy_pass http://$user_svc/; -} -``` - -The `resolver 127.0.0.11 valid=30s;` directive already in `nginx.conf` handles the periodic re-resolution — no other changes needed. - -Apply the same pattern to every service: `auth_service`, `user_service`, `produce_service`, `order_service`, `payment_service`, `message_service`, `notification_service`, `blog_service`, `ussd_service`. - ---- - -## Why this doesn't affect the ML gateway - -The ML gateway already uses variable-based resolution: -```nginx -set $ml_gw "ml-gateway:8000"; -proxy_pass http://$ml_gw/...; -``` -That's why it survives ML stack restarts without nginx needing a reload. diff --git a/PHASE2_ANALYSIS.md b/PHASE2_ANALYSIS.md deleted file mode 100644 index dc1d922..0000000 --- a/PHASE2_ANALYSIS.md +++ /dev/null @@ -1,424 +0,0 @@ -# Soko ML Phase 2 — Complete System Analysis - -**Date of Analysis:** 2026-05-14 -**Scope:** Phase 2 extension of the Soko ML service layer — transforming a static CSV-driven price prediction system into a live, data-ingesting, location-aware farmer decision support system. - ---- - -## 1. What Phase 2 Actually Builds - -Phase 1 delivered Prophet-based price prediction and collaborative-filtering recommendations, both trained on synthetic CSV data. Phase 2 replaces every static data source with live backend data and adds two new services. The result is a system that: - -- Learns from real transactions as they happen -- Recommends the best market to sell at (not just the price) -- Gives the farmer a GO/WAIT sell signal based on price trajectory -- Falls back gracefully when ML coverage is insufficient -- Tracks its own blind spots and escalates them for remediation - ---- - -## 2. Discovery Findings (What the Code Actually Contains) - -### Auth Service (`services/auth`) -- Stores only credentials (email, hashed password, role). No profile data, no GPS. -- All profile operations delegated to user-service at registration time. -- **Impact:** Bootstrap clients must call user-service, not auth-service. - -### User Service (`services/user`) -- `UserProfile` has `district` and `village` as plain strings. No lat/lng. -- `specialties` is a comma-separated string (e.g., `"maize,beans,coffee"`). -- `GET /users/farmers` already existed (public). `GET /users/buyers` did not — added. -- **Impact:** All GPS must come from district-centroid lookup tables, not user records. - -### Order Service (`services/order`) -- `OrderStatus` enum: `pending → confirmed → processing → dispatched → delivered → cancelled`. No "completed". -- `checkout()` publishes to `soko.transactions` with `crop = product.category` (e.g., `"Grains"`) — not the specific crop name. -- **Impact:** (1) Bootstrap must filter by `delivered` status. (2) `product_name` must be added to the Kafka payload to enable specific crop identification. - -### Produce Service (`services/produce`) -- Service is named `produce-service` in Docker. Prompt called it `listing-service` — all client code uses the actual name. -- No Kafka publisher. Listings are only accessible via HTTP. -- **Impact:** Periodic re-bootstrap every 15 minutes for profile sync (no push-based update path). - ---- - -## 3. Architectural Decisions and Their Implications - -### Decision 1: Add `product_name` to the Kafka Transaction Payload - -**What changed:** `services/order/app/kafka_publisher.py` and `routers/orders.py` now include `product_name` (the listing title, e.g., `"Maize (Dry)"`) alongside the existing `crop` (the category, e.g., `"Grains"`). - -**Why it was needed:** The ML price model is trained on specific crops (`maize_grain`, `yellow_beans`). The category field alone is ambiguous — `"Grains"` maps to 5+ crops. Without `product_name`, every streaming transaction event would need to guess the crop, producing noise in training data. - -**How it works:** `normalise_crop_from_order()` in `price_transformer.py` tries `product_name` first via the `CROP_NAME_NORMALISER` dict, then falls back to category-level mapping if the product name is unrecognisable. This means a transition period where older orders (no `product_name`) still produce usable (if slightly noisier) observations. - -**Risk:** Requires a coordinated deploy — data-ingestion-service must be updated after order-service, or it will receive events without `product_name` and fall back gracefully. - ---- - -### Decision 2: No Kafka Publishers in User/Produce Services → Periodic Re-Bootstrap Instead - -**What was considered:** Adding Kafka `profile_updated` events to user-service and produce-service, so the ML layer would get push-based profile updates. - -**Why rejected:** These services have no Kafka infrastructure. Adding it would be a significant cross-service change with no benefit to the core product, only to the ML layer. - -**What was built instead:** `recommendation-service` calls `GET /users/farmers` and `GET /users/buyers` every `PROFILE_REFRESH_INTERVAL_SECONDS` (default: 15 min) via `asyncio.create_task` background loop. Data is loaded directly into the in-memory `ProfileStore`. - -**Implication:** A farmer who updates their specialties will not be reflected in recommendations for up to 15 minutes. This is acceptable for a smallholder market with low-frequency profile changes. - -**Implication for bootstrap:** `data-ingestion-service` runs a full HTTP bootstrap of all farmers, buyers, and delivered orders at startup, then listens to `soko.transactions` for incremental updates. The Postgres feature store (`farmer_features`, `buyer_features`) is the ground truth; the recommendation service's in-memory store is a read cache of it. - ---- - -### Decision 3: Uganda District Centroids as GPS Approximation - -**What was built:** `DISTRICT_COORDINATES` in `farmer_transformer.py` maps 25 Uganda districts to approximate lat/lng centroid coordinates. `DISTRICT_TO_MARKET` maps districts to ML market node IDs (used by `price_transformer.py` and `geo_recommender.py`). - -**Why it was needed:** User profiles store `district` (e.g., `"Gulu"`) as a string. No GPS field exists. The location-service and recommendation-service both need coordinates for distance calculations. - -**Known limitation:** District centroids can be 20-50 km from where the farmer actually is. The `GEO_FILTER_RELAX_FACTOR` (default: 1.5) compensates by expanding the Haversine pre-filter radius by 50%. All API responses include a `location_precision: "district_centroid"` flag so callers know the coordinates are approximate. - -**Upgrade path:** If the user-service adds a GPS field to `UserProfile` in the future, `farmer_transformer.py` can use it directly — the `district_to_coords()` function is only called when no explicit coordinates are present. - ---- - -## 4. New Services - -### `data-ingestion-service` (port 8096) - -**Purpose:** Single entry point for all data flowing into the ML Feature Store. Owns the Postgres `soko_ml_db` database exclusively. - -**Startup sequence:** -1. Connect to Postgres -2. Apply schema if needed (`db-init` container runs schema.sql separately) -3. If `BOOTSTRAP_ON_STARTUP=true` AND all three core tables are empty: trigger full HTTP bootstrap from user/order/produce services -4. Start `TransactionStream` background thread consuming `soko.transactions` - -**Key endpoints:** -- `POST /bootstrap` — Trigger or re-trigger full sync -- `GET /bootstrap/status` — Check if bootstrap is in progress / complete -- `POST /ingest/order-event` — Accept a single transaction event (called by kafka-agent's `TransactionPriceCollector`) -- `GET /gaps/summary` — Show which crop/market pairs need more data -- `GET /coverage` — Full coverage map - -**Data quality mechanisms:** -- Outlier rejection: 3σ rolling window (last 30 observations per crop/market pair) -- Deduplication: Postgres `UNIQUE` constraint on `order_id` in `price_observations` -- Both streaming path (`TransactionStream`) and HTTP path (`/ingest/order-event`) exist simultaneously; deduplication handles double-writes - ---- - -### `location-service` (port 8003) - -**Purpose:** Given a farmer + crop, recommend which markets to sell at and when. - -**`POST /route` — The core endpoint:** -1. Load market registry from `soko_ml_db` (Redis-cached for 6h) -2. Get distances to all markets (Google Maps batch call, or Haversine fallback; Redis-cached per farmer×market pair for 30 days) -3. Fetch Prophet price predictions from `ml-gateway-service` for each market -4. Compute net value = `predicted_price × quantity - transport_cost` -5. Derive GO/WAIT sell signal from price trend, perishability, harvest month -6. Return ranked market list with signal - -**Three-tier fallback:** -- **Tier 1:** Full ML — Prophet predictions available for this crop/market pair (`observation_count >= 30`) -- **Tier 2:** Category price band — crop is known but specific market has insufficient data. Returns historical price range for the crop category -- **Tier 3:** Unknown crop — crop not in the ML catalogue at all. Returns generic advice, records a coverage gap event, publishes to `soko.gaps` - -**`POST /discover` — Buyer-side discovery:** -Finds farmers near a buyer who grow a requested crop. Uses `farmer_features` table (district centroids + Haversine filter). - ---- - -## 5. Modified Existing Services - -### `price-prediction-service` -- Added `train_all_models_from_feature_store()`: reads real price observations from Postgres instead of CSV; triggered when `retrain_requested` event arrives on `soko.ml.events` -- Original `train_all_models()` (CSV-based) retained as cold-start fallback - -### `recommendation-service` -- `ProfileStore` rewritten: `async reload()` fetches from Postgres feature store instead of reading CSV files -- Startup fails fast (SystemExit) if DB is unreachable — no silent degradation with empty profiles -- Background reload task runs every 15 min - -### `ml-gateway-service` -- Added proxying for `/location/*`, `/gaps/*`, `/coverage`, `/ingest/*` routes -- Added circuit breakers for `location` and `ingestion` services -- Health check now aggregates status from all 4 downstream services (price, rec, location, ingest); only price + rec are required for `overall: ok` - -### `kafka-agent` -- Added `CoverageGapConsumer`: consumes `soko.gaps` for monitoring/logging -- Added `TransactionPriceCollector`: consumes `soko.transactions`, forwards `purchase_completed` events to `data-ingestion-service /ingest/order-event` via HTTP (alternative path to the internal `TransactionStream`) - ---- - -## 6. Data Flow Diagrams - -### Transaction → Price Observation Flow - -``` -Order Service - └─ checkout() → publish to soko.transactions - │ - ├─► TransactionStream (data-ingestion-service internal thread) - │ └─ insert_price_observation() → price_observations (Postgres) - │ └─ trigger: trg_update_coverage → updates coverage_map - │ └─ if observation_count >= 30 → publish retrain_requested - │ └─ price-prediction-service retrains Prophet model - │ - └─► TransactionPriceCollector (kafka-agent) - └─ POST /ingest/order-event → data-ingestion-service - └─ insert_price_observation() (deduplicated by order_id) -``` - -### Farmer Market Routing Flow - -``` -Client → POST /ml/location/route - └─ ml-gateway-service → POST /route → location-service - │ - ├─ Load market registry (soko_ml_db → Redis cache) - ├─ Get farmer GPS (district centroid if no lat/lng) - ├─ Fetch road distances (Google Maps API → Redis cache 30 days) - ├─ Fetch Prophet predictions (ml-gateway → price-prediction-service) - │ └─ Tier check: coverage_map.is_model_ready? - │ ├─ Tier 1: full ML prediction → compute net value - │ ├─ Tier 2: category price band (insufficient data) - │ └─ Tier 3: unknown crop → gap_notifier → soko.gaps - ├─ Estimate transport cost (rate band lookup) - ├─ Derive sell signal (perishability → harvest month → trend) - └─ Return ranked_markets + signal -``` - -### Coverage Gap → Retraining Flow - -``` -location-service (Tier 3) - └─ gap_notifier.record_and_notify_gap() - ├─ INSERT / UPDATE coverage_gaps (Postgres) - └─ publish CoverageGapEvent → soko.gaps - ├─► CoverageGapConsumer (kafka-agent) — logs for ops monitoring - └─► (future) admin notification service - -data-ingestion-service (TransactionStream) - └─ When coverage_map.observation_count reaches MIN_OBSERVATIONS_FOR_MODEL (30) - └─ publish RetrainRequestedEvent → soko.ml.events - └─► price-prediction-service - └─ train_all_models_from_feature_store(crop, market) -``` - ---- - -## 7. Kafka Topics Reference - -| Topic | Partitions | Retention | Produced by | Consumed by | -|---|---|---|---|---| -| `soko.transactions` | 6 | 7 days | order-service | TransactionStream, TransactionPriceCollector, TransactionConsumer | -| `soko.interactions` | 6 | 3 days | (future frontend) | InteractionConsumer | -| `soko.price.requests` | 3 | 1 day | kafka-agent | PriceRequestConsumer | -| `soko.price.results` | 3 | 1 day | PriceRequestConsumer | kafka-agent | -| `soko.ml.events` | 2 | 14 days | data-ingestion-service | price-prediction-service | -| `soko.gaps` | 2 | 30 days | location-service | CoverageGapConsumer | -| `soko.dlq` | 2 | 30 days | all consumers on failure | (manual remediation) | - ---- - -## 8. Postgres Schema Summary (`soko_ml_db`) - -| Table | Purpose | Key Columns | -|---|---|---| -| `farmer_features` | ML-ready farmer profiles | `farmer_id`, `crops_offered TEXT[]`, `lat`, `lng`, `avg_rating` | -| `buyer_features` | ML-ready buyer profiles | `buyer_id`, `crop_interests TEXT[]`, `total_purchases` | -| `price_observations` | Real transaction prices | `crop`, `market`, `price_ugx_per_kg`, `order_id UNIQUE` | -| `coverage_map` | Model readiness per crop/market | `is_model_ready`, `observation_count`, `last_trained_at` | -| `market_registry` | All known markets with GPS | `market_id`, `lat`, `lng`, `active` | -| `coverage_gaps` | Crops/markets with no ML data | `crop`, `priority LOW/MEDIUM/HIGH`, `gap_count` | - -**Postgres trigger:** `trg_update_coverage` fires on every `price_observations` INSERT, incrementing `coverage_map.observation_count` and flipping `is_model_ready = TRUE` when count reaches `min_observations_needed` (default 30). This is the mechanism by which the system self-heals from Tier 2/3 to Tier 1 over time. - ---- - -## 9. Redis Key Patterns and TTLs - -| Key Pattern | TTL | Purpose | -|---|---|---| -| `dist:{farmer_id}:{market_id}` | 30 days | Road distance (km) from farmer to market | -| `route:{farmer_id}:{crop}` | 6 hours | Full ranked market list | -| `discover:{buyer_id}:{crop}:{radius}` | 1 hour | Nearby farmers for buyer | -| `market_registry` | 6 hours | Market list from Postgres | - ---- - -## 10. Environment Variables Reference - -See `services/soko-ml/.env.example` for the complete list with defaults and comments. - -**Critical variables that must be set before production:** - -| Variable | Default | Why it matters | -|---|---|---| -| `POSTGRES_PASSWORD` | `changeme` | Feature store security | -| `INTERNAL_API_KEY` | `internal-secret` | Auth between ML layer and core services | -| `USER_SERVICE_URL` | `http://user-service:8002` | Bootstrap farmer/buyer data | -| `ORDER_SERVICE_URL` | `http://order-service:8003` | Bootstrap historical price observations | -| `PRODUCE_SERVICE_URL` | `http://produce-service:8004` | Coverage map seeding | -| `GOOGLE_MAPS_API_KEY` | (empty) | Road distances; Haversine fallback used if absent | - ---- - -## 11. Port Map - -| Service | Container Port | Default Host Port | -|---|---|---| -| ml-gateway-service | 8000 | 8080 | -| price-prediction-service | 8001 | 8094 | -| recommendation-service | 8002 | 8095 | -| location-service | 8003 | 8003 | -| data-ingestion-service | 8004 | 8096 | -| Kafka (external) | 9092 | 29092 | -| Postgres (soko_ml_db) | 5432 | (not exposed) | -| Redis | 6379 | (not exposed) | - ---- - -## 12. Startup Order (Docker Compose `depends_on`) - -``` -zookeeper - └─ kafka - └─ kafka-init (creates topics, exits) - └─ price-prediction-service - └─ recommendation-service - └─ kafka-agent (waits for price + rec + data-ingestion) - -soko-ml-db - └─ db-init (applies schema.sql, exits) - └─ data-ingestion-service - └─ (bootstrap runs at startup if tables empty) - -redis - └─ price-prediction-service - └─ recommendation-service - └─ location-service - -data-ingestion-service + location-service - └─ ml-gateway-service (waits for all 4 downstream services) - └─ kafka-agent -``` - ---- - -## 13. Known Limitations and Future Work - -| Limitation | Impact | Recommended Fix | -|---|---|---| -| District centroid GPS (~20-50 km error) | Distance-ranked markets may be slightly wrong | Add `lat/lng` to `UserProfile` in user-service | -| No real-time profile push | Up to 15 min lag for updated farmer specialties | Add Kafka publisher to user-service `PUT /me` | -| Produce-service listing prices not in feature store | Farmer asking price not used in market routing | Periodic sync of active listing prices | -| Google Maps quota | Distance calls are batched and cached; if quota exhausted, Haversine is used | Monitor API quota; upgrade plan if needed | -| Prophet cold-start for new crop/market pairs | Tier 2 until 30 observations accumulate | Lower `MIN_OBSERVATIONS_FOR_MODEL` to 10 during initial rollout | -| `kafka-agent` `TransactionPriceCollector` doubles write load | Acceptable due to Postgres deduplication | Can be disabled if data-ingestion-service `TransactionStream` is confirmed reliable | - ---- - -## 14. File Manifest (Phase 2 New / Modified Files) - -### New Services -``` -services/soko-ml/data-ingestion-service/ - src/ - main.py FastAPI app, lifespan bootstrap - schemas.py Pydantic request/response models - feature_store.py All asyncpg DB operations - health.py Health checks (DB + backend services) - transformers/ - farmer_transformer.py Profile → farmer_features row - buyer_transformer.py Profile → buyer_features row - price_transformer.py Transaction event → price_observations row - clients/ - user_client.py GET /users/farmers, /users/buyers - order_client.py GET /orders?status=delivered - listing_client.py GET /listings - bootstrap/ - auth_bootstrap.py Bulk farmer + buyer sync - order_bootstrap.py Bulk price observation sync - listing_bootstrap.py Coverage map seeding - market_bootstrap.py No-op (markets seeded in schema.sql) - streams/ - transaction_stream.py Kafka consumer thread - tests/ - test_transformers.py Pure unit tests (no DB) - test_feature_store.py Integration tests (skipped if DB absent) - requirements.txt - Dockerfile - -services/soko-ml/location-service/ - src/ - main.py FastAPI app - schemas.py Pydantic models - market_router.py Core routing logic - geo_recommender.py Buyer→farmer discovery - google_maps_client.py Distance Matrix API + Haversine fallback - transport_cost.py Rate-band cost estimation - sell_signal.py GO/WAIT signal derivation - fallback.py Tier 1/2/3 fallback logic - gap_notifier.py Gap recording + Kafka publish - cache.py Redis key patterns + helpers - tests/ - test_market_router.py Unit + integration tests - requirements.txt - Dockerfile -``` - -### New Infrastructure -``` -services/soko-ml/db/ - schema.sql Postgres schema + seed data + trigger - -services/soko-ml/.env.example Updated with all Phase 2 variables -``` - -### Modified Existing Files -``` -services/order/app/kafka_publisher.py Added product_name field -services/order/app/routers/orders.py Passes product_name on checkout + cancel - -services/user/app/routers/profile.py Added GET /users/buyers endpoint - -services/soko-ml/shared/events.py Added SokoTransactionEvent, CoverageGapEvent, - RetrainRequestedEvent - -services/soko-ml/price-prediction-service/src/predictor.py - Added train_all_models_from_feature_store() - -services/soko-ml/recommendation-service/ - src/recommender.py ProfileStore rewritten (async reload from Postgres) - src/main.py Removed CSV paths; added periodic reload - src/feature_store_client.py New — asyncpg reads from farmer_features/buyer_features - src/geo_filter.py New — Haversine pre-filter helper - requirements.txt Added asyncpg - -services/soko-ml/ml-gateway-service/ - src/proxy.py Added location + ingestion circuit breakers - src/main.py Added /location/*, /gaps/*, /coverage, /ingest/* routes - -services/soko-ml/kafka-agent/ - src/agent.py Added CoverageGapConsumer + TransactionPriceCollector - src/consumers/coverage_gap_consumer.py New - src/consumers/transaction_price_collector.py New - -services/soko-ml/docker-compose.yml - Added: soko-ml-db, db-init, data-ingestion-service, location-service - Added: soko.gaps Kafka topic - Updated: ml-gateway depends_on, kafka-agent depends_on + DATA_INGESTION_SERVICE_URL - Updated: volumes (soko_ml_db_data) - -Makefile (root) - Added: db-up, db-shell, db-reset, cold-start - Added: ingest-bootstrap, ingest-status, gaps-summary, gaps-reset - Added: dev-location, dev-ingest, logs-location, logs-ingest - Added: test-location, test-ingest - Added: smoke-route, smoke-discover, smoke-fallback, smoke-tier3, smoke-ingest - Added: INGEST_VENV, LOC_VENV - Updated: install, infra-up, infra-down, kafka-topics, health, clean -``` diff --git a/README.md b/README.md index fee8d8e..b6d8eff 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ # Soko — Digital Agricultural Marketplace -A production-grade microservices platform connecting Ugandan farmers and buyers. Farmers list produce, buyers place orders, and the system handles payments, messaging, notifications, and price intelligence — all through a single Nginx API gateway. +A production-grade microservices platform connecting Ugandan farmers and buyers. Farmers list produce, buyers place orders, the system handles payments, messaging, and notifications — and a dedicated ML layer delivers personalised recommendations and market price forecasts to every authenticated user. -The platform is split into two independent but integrated stacks: +The platform runs as two independent but integrated Docker Compose stacks: -- **Core stack** — transactional services (auth, users, produce, orders, payments, messaging, blog, USSD) -- **ML stack** — price prediction and farmer/buyer matching (`services/soko-ml/`) +- **Core stack** — transactional services: auth, users, produce, orders, payments, messaging, notifications, blog, USSD +- **ML stack** — intelligence layer: price prediction, personalised recommendations, market routing, data ingestion, Kafka event backbone --- @@ -14,73 +14,88 @@ The platform is split into two independent but integrated stacks: 1. [Architecture Overview](#architecture-overview) 2. [Core Services](#core-services) 3. [ML Layer](#ml-layer) -4. [How the Two Stacks Interact](#how-the-two-stacks-interact) -5. [API Reference](#api-reference) -6. [User Flows](#user-flows) -7. [Event System](#event-system) -8. [Getting the ML Stack Running](#getting-the-ml-stack-running) -9. [Running Tests](#running-tests) -10. [Environment Variables](#environment-variables) -11. [Project Structure](#project-structure) +4. [Auth → ML: The Authenticated Recommendation Flow](#auth--ml-the-authenticated-recommendation-flow) +5. [How the Two Stacks Integrate](#how-the-two-stacks-integrate) +6. [API Reference](#api-reference) +7. [User Flows](#user-flows) +8. [Event System (Kafka)](#event-system-kafka) +9. [Getting Started](#getting-started) +10. [Makefile Reference](#makefile-reference) +11. [Environment Variables](#environment-variables) +12. [Project Structure](#project-structure) +13. [Production Bug Report](#production-bug-report) +14. [Known Limitations](#known-limitations) --- ## Architecture Overview ``` - ┌────────────────────────────────────────────────────────────────┐ - │ CLIENT LAYER │ - │ Web App · Mobile App · USSD Handsets │ - └───────────────────────────┬────────────────────────────────────┘ - │ HTTP / WebSocket - ▼ - ┌────────────────────────────────────────────────────────────────┐ - │ NGINX API GATEWAY :80 │ - │ Rate limiting (30 req/min) · CORS · JWT subrequest auth │ - │ Routes: /auth/ /users/ /listings/ /orders/ /payments/ │ - │ /message/ /notifications/ /posts/ /ussd/ │ - │ /recommendations/ │ - └──┬────┬────┬────┬────┬────┬────┬────┬────┬────────────────────┘ - │ │ │ │ │ │ │ │ │ - ▼ ▼ ▼ ▼ ▼ ▼ ▼ ▼ ▼ - Auth User Prod Ord Pay Msg Notif Blog USSD Rec - :8001:8002:8003:8004:8005:8006:8007 :8008:8009 :8010 - │ │ │ │ │ │ │ │ │ │ - └────┴────┴────┴────┴────┴────┴────┴────┴─────┘ - │ - ┌──────────┴──────────┐ - │ │ - ▼ ▼ - PostgreSQL RabbitMQ :5672 - (one DB per service) (async events between - core services) - - ┌────────────────────────────────────────────────────────────────┐ - │ ML STACK (services/soko-ml/) │ - │ │ - │ ml-gateway-service :8000 ← called by core services │ - │ │ ← single entry point │ - │ ├──► price-prediction-service :8001 │ - │ │ Prophet .pkl models · Redis cache 24h │ - │ └──► recommendation-service :8002 │ - │ Content scoring · Redis cache 1h │ - │ │ - │ kafka-agent (no HTTP port) │ - │ ├── consumes: soko.transactions │ - │ ├── consumes: soko.interactions │ - │ ├── consumes: soko.price.requests │ - │ └── produces: soko.price.results · soko.dlq │ - │ │ - │ Infrastructure: Kafka · Zookeeper · Redis │ - └────────────────────────────────────────────────────────────────┘ + ┌──────────────────────────────────────────────────────────────────────┐ + │ CLIENT LAYER │ + │ Web App · Mobile App · USSD Handsets │ + └───────────────────────────────┬──────────────────────────────────────┘ + │ HTTP / WebSocket + ▼ + ┌──────────────────────────────────────────────────────────────────────┐ + │ NGINX API GATEWAY :80 │ + │ Rate limiting (30 req/min) · CORS · JWT subrequest auth │ + │ │ + │ /auth/ /oauth/ → auth_service (public) │ + │ /users/ → user_service (JWT required) │ + │ /listings/ → produce_service (JWT optional) │ + │ /orders/ → order_service (JWT required) │ + │ /payments/ /webhook/ → payment_service (JWT / public) │ + │ /message/ /message/ws/ → message_service (JWT / WS) │ + │ /notifications/ /ws/ → notification_service(JWT / WS) │ + │ /posts/ → blog_service (JWT optional) │ + │ /ussd/ → ussd_service (public) │ + │ /ml/price/ → ml-gateway (public) │ + │ /ml/recommend/ → ml-gateway (JWT required) ◄─┐ │ + │ /recommendations/ → ml-gateway (JWT required) ──┘ │ + └──┬────┬────┬────┬────┬────┬────┬────┬──────────────┬────────────────┘ + │ │ │ │ │ │ │ │ │ + ▼ ▼ ▼ ▼ ▼ ▼ ▼ ▼ ▼ + :8001:8002:8003:8004:8005:8006:8007:8008 ML stack + Auth User Prod Ord Pay Msg Not Blog USSD (see below) + :8009 + + Each service owns its own PostgreSQL database. + Core services share one Redis instance for caching. + Order service publishes to Kafka → ML layer consumes. + + ┌──────────────────────────────────────────────────────────────────────┐ + │ ML STACK (services/soko-ml/) │ + │ │ + │ nginx ──► ml-gateway-service (host :8080 / internal :8000) │ + │ │ circuit breakers · request logging · fallbacks │ + │ ├──► price-prediction-service (:8001) │ + │ │ Prophet .pkl models · Redis 24h cache │ + │ ├──► recommendation-service (:8002) │ + │ │ Content scoring · Postgres profiles │ + │ │ Redis 1h cache · Kafka interaction boosts │ + │ ├──► location-service (:8003) │ + │ │ Market routing · Haversine distance │ + │ └──► data-ingestion-service (:8004) │ + │ Bootstrap profiles from user-service │ + │ Kafka transaction → price observations │ + │ │ + │ kafka-agent (no HTTP port) │ + │ ├── soko.transactions → soko.interactions (boost pipeline) │ + │ ├── soko.price.requests → price-prediction → soko.price.results│ + │ └── soko.gaps (coverage gap monitoring) │ + │ │ + │ Kafka · Zookeeper · Redis · PostgreSQL (soko_ml_db) │ + └──────────────────────────────────────────────────────────────────────┘ ``` ### Key design rules -- Every external request enters through **Nginx only** — services are never exposed directly. -- Every call to the ML layer enters through **ml-gateway-service only** — price and recommendation services are never called directly by core services. -- Auth is enforced at the gateway via an internal `/verify-token` subrequest to the Auth service before the request reaches any protected service. -- Core services communicate asynchronously via **RabbitMQ**. The ML layer uses **Kafka** for its own event backbone. +- Every external request enters through **Nginx only** — core services are never exposed directly on the public network. +- Every call to the ML intelligence layer goes through **ml-gateway-service only** — downstream ML services are internal. +- JWT authentication is enforced at the Nginx gateway via an internal `/_verify_token` subrequest to the auth service. Validated user identity (`X-User-Id`, `X-User-Role`) is injected as headers into every downstream service. +- The recommendation service enforces that a user can only request recommendations for their own account ID — the JWT-derived `X-User-Id` is compared against the path parameter on every request. +- The two stacks communicate over the `soko-ml-bridge` Docker network and the `soko.transactions` Kafka topic. --- @@ -88,48 +103,56 @@ The platform is split into two independent but integrated stacks: ### Auth Service — `:8001` -**Responsibility:** Identity and access. Issues JWTs on login, exposes `/verify-token` which Nginx calls internally on every protected route to validate tokens and inject `X-User-Id`, `X-User-Role`, `X-User-Email` headers downstream. +Issues JWTs on login and validates them on every protected route. Nginx calls `/verify-token` internally — it never reaches the client. On success it injects `X-User-Id`, `X-User-Role`, `X-User-Email` into downstream headers. -**Nginx route:** `/auth/` and `/oauth/` (public — no auth guard) +**Nginx route:** `/auth/` and `/oauth/` (public) | Method | Path | Description | |---|---|---| -| POST | `/auth/register` | Register with role `farmer` or `buyer` | -| POST | `/auth/login` | Login → JWT access token | -| GET | `/auth/me` | Current user info (JWT required) | +| POST | `/auth/register` | Register with `role: farmer\|buyer\|both` | +| POST | `/auth/login` | Login → `{ access_token, refresh_token }` | +| GET | `/auth/me` | Current user info (JWT required) | | POST | `/auth/refresh` | Refresh an expiring token | -| GET | `/verify-token` | Internal — called by Nginx, not clients | +| GET | `/verify-token` | Internal — called by Nginx, not clients | +| GET | `/verify-token-optional` | Internal — for public routes that optionally expose user context | --- ### User Service — `:8002` -**Responsibility:** User profiles and account management. Receives the authenticated user context (`X-User-Id`, `X-User-Role`) from Nginx — it never validates tokens itself. +User profiles and account management. Receives authenticated user context from Nginx and never validates tokens itself. Also exposes internal endpoints used by the ML data-ingestion service to bootstrap the feature store. **Nginx route:** `/users/` (JWT required) +| Method | Path | Auth | Description | +|---|---|---|---| +| GET | `/users/me` | JWT | Own profile | +| PUT | `/users/me` | JWT | Update profile (specialties, interests, district) | +| GET | `/users/farmers` | JWT | List all farmers (paginated) — also used internally by ML ingestion | +| GET | `/users/buyers` | JWT | List all buyers (paginated) — also used internally by ML ingestion | +| GET | `/users/{id}` | JWT | Single farmer profile | + --- ### Produce Service — `:8003` -**Responsibility:** Produce listings — creation, search, stock management. Farmers create listings; buyers browse them. Publishes `produce.listed` events to RabbitMQ so the recommendation service can index new listings. +Produce listings — creation, search, and stock management. Farmers create listings; buyers browse them. Supports image uploads via Cloudinary. -**Nginx route:** `/listings/` (JWT required, 20 MB upload limit for images) +**Nginx route:** `/listings/` (JWT optional — public browsing, auth to create) | Method | Path | Auth | Description | |---|---|---|---| -| POST | `/listings/` | farmer | Create a listing | -| GET | `/listings/` | JWT | Browse / search (filter by category, district, price) | -| GET | `/listings/{id}` | JWT | Single listing | -| PATCH | `/listings/{id}` | farmer | Update own listing | +| POST | `/listings/` | farmer | Create a listing | +| GET | `/listings/` | optional | Browse / search (filter by category, district, price) | +| GET | `/listings/{id}` | optional | Single listing | +| PUT | `/listings/{id}` | farmer | Update own listing | | DELETE | `/listings/{id}` | farmer | Remove listing | -| PATCH | `/listings/{id}/reduce-stock` | internal | Called by Order service on order placement | --- ### Order Service — `:8004` -**Responsibility:** Order lifecycle from placement to completion. Buyers place orders against listings; farmers accept or reject; status advances through a defined state machine. +Order lifecycle from placement to completion. Publishes `purchase_completed` events to `soko.transactions` on Kafka on every successful checkout — this is the primary data source for ML price observations and interaction boosts. **Nginx route:** `/orders/` (JWT required) @@ -144,18 +167,18 @@ placed → pending | Method | Path | Auth | Description | |---|---|---|---| -| POST | `/orders/` | buyer | Place order | -| GET | `/orders/` | buyer | List own orders | -| PATCH | `/orders/{id}/cancel` | buyer | Cancel a pending order | -| POST | `/orders/{id}/review` | buyer | Review a completed order | -| GET | `/orders/incoming/` | farmer | Orders for farmer's produce | -| PATCH | `/orders/{id}/status` | farmer | Advance order status | +| POST | `/orders/` | buyer | Place order → publishes to `soko.transactions` | +| GET | `/orders/` | buyer | List own orders | +| POST | `/orders/{id}/cancel` | buyer | Cancel → publishes cancellation to Kafka | +| POST | `/orders/{id}/review` | buyer | Rate after completion | +| GET | `/orders/incoming/` | farmer | Orders for farmer's produce | +| PATCH | `/orders/{id}/status` | farmer | Advance status | --- ### Payment Service — `:8005` -**Responsibility:** Payment initiation and reconciliation via PesaPal (MTN Mobile Money / Airtel Money). The `/webhook/` endpoint is public so PesaPal can POST payment confirmations without authentication. +Payment initiation and reconciliation via PesaPal (MTN Mobile Money / Airtel Money). The `/webhook/` endpoint is public so PesaPal can POST confirmations without a token. **Nginx routes:** `/payments/` (JWT required) · `/webhook/` (public) @@ -163,7 +186,7 @@ placed → pending ### Message Service — `:8006` -**Responsibility:** Real-time direct messaging between farmers and buyers over WebSocket. The WebSocket upgrade (`/message/ws/`) bypasses the JWT subrequest — token is validated by the service itself on connection. +Real-time direct messaging over WebSocket. Token is validated by the service itself on WebSocket connection. **Nginx routes:** `/message/` (JWT required) · `/message/ws/` (WebSocket, service-auth) @@ -171,7 +194,7 @@ placed → pending ### Notification Service — `:8007` -**Responsibility:** Push notifications delivered in real-time over WebSocket. Consumes events from RabbitMQ (order updates, payment confirmations) and pushes them to connected clients. +Push notifications delivered over WebSocket. Receives events from order and payment services and pushes them to connected clients. **Nginx routes:** `/notifications/` (JWT required) · `/notifications/ws/` (WebSocket, service-auth) @@ -179,115 +202,152 @@ placed → pending ### Blog Service — `:8008` -**Responsibility:** Agri-knowledge articles and market commentary. Supports image uploads up to 10 MB. +Agri-knowledge articles and market commentary. Supports image uploads up to 10 MB via Cloudinary. -**Nginx route:** `/posts/` (JWT required, 10 MB upload limit) +**Nginx route:** `/posts/` (JWT optional — public reading, auth to create) --- ### USSD Service — `:8009` -**Responsibility:** USSD session handler for feature-phone users. Completely public — USSD networks don't carry HTTP auth headers. Allows farmers with basic handsets to check prices and receive order notifications. - -**Nginx route:** `/ussd/` (public — no auth guard) - ---- - -### Recommendation Service — `:8010` - -**Responsibility:** Personalised produce feed for buyers based on order history, category preferences, and produce quality scores. Consumes `produce.listed`, `order.placed`, and `quality.scored` events from RabbitMQ to keep its index fresh. This is the **existing** rule-based recommendation service, separate from the ML farmer/buyer matching layer. +USSD session handler for feature-phone users. Allows farmers with basic handsets to check prices and receive order notifications without a smartphone. Calls the ML gateway for price predictions. -**Nginx route:** `/recommendations/` (public) - -| Method | Path | Description | -|---|---|---| -| GET | `/recommendations/` | Personalised feed for authenticated buyer | -| GET | `/recommendations/produce/{id}/score` | Quality score for a listing | +**Nginx route:** `/ussd/` (public — USSD networks carry no auth headers) --- ## ML Layer -The ML layer lives in `services/soko-ml/` and runs as a **separate Docker Compose stack**. It has four services of its own, plus Kafka and a dedicated Redis instance. - -### ml-gateway-service — `:8000` +The ML layer lives in `services/soko-ml/` and runs as a separate Docker Compose stack. All six services connect to the core stack via the `soko-ml-bridge` Docker network. -The single entry point for all ML capabilities. No core service should ever call the price or recommendation ML services directly — they call this gateway, which adds: +### ml-gateway-service — host `:8080` / internal `:8000` -- **Request logging** — service name, endpoint, latency, cache hit/miss -- **Circuit breaking** — if a downstream ML service is unreachable after 3 retries, returns a graceful fallback response instead of propagating a 500 -- **Health aggregation** — `GET /health` polls all downstream services and returns a combined status +Single entry point for all ML capabilities. Nginx proxies `/ml/*` and `/recommendations/*` here. Adds circuit breaking (3 failures → open, 30s reset), request logging, and graceful fallback responses. -| Gateway endpoint | Proxied to | -|---|---| -| `POST /price/predict` | price-prediction-service `/predict` | -| `GET /price/markets` | price-prediction-service `/markets` | -| `GET /price/crops` | price-prediction-service `/crops` | -| `GET /recommend/farmers-for-buyer/{buyer_id}` | recommendation-service | -| `GET /recommend/buyers-for-farmer/{farmer_id}` | recommendation-service | -| `GET /health` | aggregated from all downstream | +| Gateway Endpoint | Routes to | Auth | +|---|---|---| +| `POST /price/predict` | price-prediction-service | public | +| `GET /price/markets` | price-prediction-service | public | +| `GET /price/crops` | price-prediction-service | public | +| `GET /recommend/farmers-for-buyer/{buyer_id}` | recommendation-service | JWT required | +| `GET /recommend/buyers-for-farmer/{farmer_id}` | recommendation-service | JWT required | +| `POST /location/route` | location-service | public | +| `POST /location/discover` | location-service | public | +| `GET /gaps/summary` | data-ingestion-service | public | +| `GET /coverage` | data-ingestion-service | public | +| `POST /ingest/bootstrap` | data-ingestion-service | internal | +| `GET /health` | aggregated from all downstream | public | --- -### price-prediction-service — `:8001` (ML stack internal) +### price-prediction-service — internal `:8001` Serves 4-week price forecasts per market–crop pair in UGX using pre-trained **Prophet** models. -- Loads `.pkl` model files from `models/` at startup (one model per market–crop pair, 48 total) -- Checks **Redis** on every request (`price:v1:{market}:{crop}:{weeks}`, TTL 24 h) -- Falls back to Uganda seasonal heuristics if no model file is present (always responds) -- Publishes a `price.predicted` event to `soko.price.results` after every inference -- Consumes `soko.price.requests` for async batch prediction jobs +- Loads `.pkl` model files from `models/` at startup +- Falls back to Uganda bimodal seasonal heuristics when no model file exists +- Caches predictions in Redis (TTL 24 h, key: `price:v1:{market}:{crop}:{weeks}`) +- Consumes `soko.price.requests`; publishes to `soko.price.results` **Supported markets:** Kisenyi_Kampala · Gulu · Mbarara · Mbale · Lira · Masaka **Supported crops:** maize_grain · yellow_beans · irish_potatoes · tomatoes · matoke · cassava_chips · sorghum · millet +**Uganda bimodal seasonality factors applied:** +- Jun–Jul, Nov–Dec: ×0.92 (post-harvest abundance) +- Jan–Feb: ×1.10 (lean dry season) + --- -### recommendation-service — `:8002` (ML stack internal) +### recommendation-service — internal `:8002` -Recommends high-performing farmers to buyers and vice versa using a **weighted content-based scoring model** enriched in real-time from Kafka interaction events. +Recommends high-performing farmers to buyers and vice versa using **weighted content-based scoring** enriched in real-time from Kafka interaction events. -- Loads `farmers.csv` (200 profiles) and `buyers.csv` (300 profiles) at startup -- Scores farmer–buyer compatibility on crop overlap, market overlap, rating, fulfillment rate -- Boosts scores dynamically from Kafka events: view +0.02, inquiry +0.05, purchase +0.10, rating +0.04 -- Caches results in Redis (`rec:farmers:{buyer_id}:{top_n}`, TTL 1 h) -- Invalidates cache when a relevant interaction event arrives +- Loads profiles from the ML feature store (PostgreSQL `soko_ml_db`) at startup +- Refreshes profiles every 15 minutes (`PROFILE_REFRESH_INTERVAL_SECONDS`) +- Exposes `POST /internal/reload` so data-ingestion can trigger an immediate refresh after bootstrap +- Enforces identity: `x-user-id` from JWT must match the `{buyer_id}` or `{farmer_id}` path parameter +- Caches results in Redis (TTL 1 h) +- Invalidates cache on relevant Kafka interaction events -**Scoring weights — farmers for buyer:** +**Scoring — farmers for buyer:** | Signal | Weight | |---|---| -| Crop overlap (buyer wants ∩ farmer offers) | 0.35 | -| Market overlap | 0.20 | +| Crop overlap: buyer interests ∩ farmer specialties / \|buyer interests\| | 0.35 | +| District match (exact) | 0.20 | | Farmer average rating (normalised / 5.0) | 0.20 | -| Fulfillment rate | 0.15 | -| Interaction boost (Kafka, additive, capped +0.20) | additive | +| Farmer fulfillment rate | 0.15 | +| Interaction boost from `soko.interactions` (capped +0.20) | additive | -**Scoring weights — buyers for farmer:** +**Scoring — buyers for farmer:** | Signal | Weight | |---|---| -| Crop overlap (farmer offers ∩ buyer wants) | 0.35 | -| Market overlap | 0.20 | -| Payment reliability | 0.25 | -| Purchase volume (normalised by dataset max) | 0.20 | +| Crop overlap: farmer specialties ∩ buyer interests / \|farmer specialties\| | 0.35 | +| District match (exact) | 0.20 | +| Buyer payment reliability | 0.25 | +| Buyer spend volume (normalised by dataset max) | 0.20 | + +**Interaction boost values:** + +| Event type | Boost | +|---|---| +| `farmer_viewed` | +0.02 | +| `buyer_inquiry` | +0.05 | +| `purchase_completed` | +0.10 | +| `rating_submitted` | +0.04 | +| `high_rating` | +0.08 | + +--- + +### data-ingestion-service — internal `:8004` + +Populates and maintains the ML feature store. + +**Bootstrap (runs at startup or `POST /bootstrap`):** +1. Fetches all farmer profiles from `GET /users/farmers` on the user service +2. Fetches all buyer profiles from `GET /users/buyers` on the user service +3. Transforms and upserts into `farmer_features` and `buyer_features` tables +4. After bootstrap, immediately calls `POST /internal/reload` on the recommendation service so new users appear in recommendations within seconds rather than waiting for the 15-minute timer + +**Streaming:** +- Consumes `soko.transactions` Kafka topic +- Normalises crop names and maps delivery districts to ML market nodes +- Inserts price observations into `price_observations` table +- Detects outliers (rejects prices > 3σ from rolling 30-obs mean) + +**Coverage tracking:** Maintains `coverage_map` per (crop, market) pair. When a pair reaches 52 observations, it is flagged as `model_ready`. --- -### kafka-agent +### location-service — internal `:8003` -Long-running Python process — no HTTP port. Bridges the core Soko event stream with the ML layer. +Routes farmers to optimal markets and helps buyers discover local supply. -| Consumes | Event types | Action | +**Tiered routing:** + +| Tier | Condition | Response | |---|---|---| -| `soko.transactions` | `purchase_completed`, `order_cancelled` | Publishes enriched event to `soko.interactions` | -| `soko.interactions` | `farmer_viewed`, `buyer_inquiry`, `rating_submitted` | Logged and forwarded; recommendation-service has its own consumer | -| `soko.price.requests` | `price_prediction_requested` | Calls price-prediction-service, publishes result to `soko.price.results` | -| `soko.ml.events` | `retrain_requested`, `model_deployed` | Logged, triggers downstream refresh | +| 1 | Crop supported + ≥52 price observations for market | Top 3 markets ranked by price minus transport cost | +| 2 | Crop supported + <52 observations | Fallback to aggregated cross-market price data | +| 3 | Crop completely unsupported | Publishes `CoverageGapEvent` to `soko.gaps`; returns generic suggestion | + +--- + +### kafka-agent — no HTTP port + +Long-running process that bridges the Kafka event stream: -Failed messages go to `soko.dlq` with full error context. +| Consumes | Action | +|---|---| +| `soko.transactions` | Forwards `purchase_completed` events to `soko.interactions` (recommendation boost pipeline) | +| `soko.transactions` | Forwards to data-ingestion via `POST /ingest/order-event` (price observations) | +| `soko.price.requests` | Calls price-prediction-service, publishes result to `soko.price.results` | +| `soko.interactions` | Logged (recommendation-service has its own consumer on this topic) | +| `soko.gaps` | Logs coverage gap events for monitoring | + +Failed messages go to `soko.dlq` with full error context for replay. --- @@ -298,73 +358,134 @@ Failed messages go to `soko.dlq` with full error context. | Kafka | `confluentinc/cp-kafka:7.5.0` | 1 broker, auto-topic creation off | | Zookeeper | `confluentinc/cp-zookeeper:7.5.0` | — | | Redis | `redis:7-alpine` | 256 MB max, `allkeys-lru` eviction | +| PostgreSQL | `postgres:16-alpine` | `soko_ml_db` database | **Kafka topics:** | Topic | Partitions | Retention | Purpose | |---|---|---|---| -| `soko.transactions` | 6 | 7 days | Purchase and order events | -| `soko.interactions` | 6 | 3 days | Views, inquiries, ratings | -| `soko.price.requests` | 3 | 1 day | Async prediction requests | -| `soko.price.results` | 3 | 1 day | Async prediction results | +| `soko.transactions` | 6 | 7 days | Purchase and order events from order-service | +| `soko.interactions` | 6 | 3 days | Views, inquiries, ratings (recommendation boosts) | +| `soko.price.requests` | 3 | 1 day | Async price prediction requests | +| `soko.price.results` | 3 | 1 day | Async price prediction responses | | `soko.ml.events` | 2 | 14 days | Model lifecycle events | +| `soko.gaps` | 2 | 30 days | Coverage gap notifications | | `soko.dlq` | 2 | 30 days | Dead-letter queue | -**Redis cache keys:** +--- -| Key pattern | TTL | Stores | -|---|---|---| -| `price:v1:{market}:{crop}:{weeks}` | 24 h | Full prediction response | -| `rec:farmers:{buyer_id}:{top_n}` | 1 h | Recommended farmers list | -| `rec:buyers:{farmer_id}:{top_n}` | 1 h | Recommended buyers list | -| `model:meta:{market}:{crop}` | 7 days | Model training date, MAPE | +## Auth → ML: The Authenticated Recommendation Flow + +This is the full end-to-end flow for a user receiving personalised recommendations: + +``` +1. User registers + POST /auth/register { email, password, role: "buyer" } + → auth_service creates account + user_service creates profile + +2. User updates profile with interests + PUT /users/me { interests: ["Grains", "Legumes"], district: "Kampala" } + → user_service stores interests and district + +3. ML data-ingestion bootstrap (runs on startup or make ingest-bootstrap) + data-ingestion-service fetches: + GET http://user_service:8002/users/farmers (with X-Internal-Secret) + GET http://user_service:8002/users/buyers (with X-Internal-Secret) + → upserts into farmer_features / buyer_features in soko_ml_db + → immediately calls POST http://recommendation-service:8002/internal/reload + → recommendation-service reloads profiles from soko_ml_db within seconds + +4. User requests recommendations (authenticated) + GET /ml/recommend/farmers-for-buyer/{user_id} + Authorization: Bearer + + Nginx flow: + a) /_verify_token subrequest → auth_service validates JWT + b) auth_service returns X-User-Id: {user_id}, X-User-Role: buyer + c) Nginx injects X-User-Id, X-User-Role and proxies to ml-gateway:8000 + + ML gateway flow: + d) Extracts X-User-Id and X-User-Role from incoming headers + e) Forwards to recommendation-service:8002/recommend/farmers-for-buyer/{user_id} + with X-User-Id header attached + + Recommendation service: + f) Reads X-User-Id header + g) Validates: X-User-Id MUST equal {buyer_id} path parameter (403 if mismatch) + h) Looks up buyer profile from in-memory ProfileStore (loaded from soko_ml_db) + i) Scores all farmers: crop_overlap × 0.35 + district_match × 0.20 + + avg_rating × 0.20 + fulfillment × 0.15 + interaction_boost (max +0.20) + j) Returns top N farmers ranked by score, with matchScore field + +5. As the user transacts, scores improve automatically + Order placed → order_service publishes to soko.transactions + kafka-agent → soko.interactions (purchase_completed event) + recommendation-service Kafka consumer → interaction_store += +0.10 boost + → Redis cache invalidated → next request returns re-ranked results +``` --- -## How the Two Stacks Interact +## How the Two Stacks Integrate ``` -Core Soko stack ML stack -───────────────── ───────────────────────────────────── - ml-gateway-service :8000 -recommendation_service :8010 ──HTTP──► GET /recommend/farmers-for-buyer/ - GET /recommend/buyers-for-farmer/ +Core stack ML stack +────────────────── ────────────────────────────────── +order_service:8004 ──Kafka──► soko.transactions + └── data-ingestion (price obs) + └── kafka-agent → soko.interactions + └── recommendation (boost) -produce_service :8003 ──HTTP──► POST /price/predict - (surface price context on listing pages) +nginx:80 ──proxy──► ml-gateway:8000 + /ml/price/ (public) └── price-prediction-service:8001 + /ml/recommend/ (JWT auth) ──x-user-id──► recommendation-service:8002 + /recommendations/ (JWT auth) -order_service :8004 ──Kafka──► soko.transactions - (kafka-agent listens, enriches to - soko.interactions for rec boosts) +data-ingestion:8004 ──HTTP──► user_service:8002 + GET /users/farmers + GET /users/buyers -ussd_service :8009 ──HTTP──► POST /price/predict - (price checks on feature phones) +ussd_service:8009 ──HTTP──► ml-gateway:8000 + POST /price/predict ``` -The ML stack is intentionally decoupled — the core stack calls `ml-gateway-service` over HTTP and publishes to Kafka topics. The ML layer never calls back into the core stack. +Both stacks share the `soko-ml-bridge` Docker network. Core service names (`user_service`, `order_service`, `produce_service`) are resolvable from ML services on that bridge. + +**Internal secret:** All service-to-service calls use `X-Internal-Secret: internal-secret` (set by `INTERNAL_SECRET` in core services and `INTERNAL_API_KEY` in the ML stack). These must match. --- ## API Reference -All requests enter via `http://localhost:80` through Nginx. Protected routes require an `Authorization: Bearer ` header. +All external requests enter via `http://localhost:80` through Nginx. Protected routes require `Authorization: Bearer `. + +### Authentication + +```http +POST /auth/register { "email": "...", "password": "...", "role": "farmer|buyer|both" } +POST /auth/login { "email": "...", "password": "..." } +GET /auth/me Authorization: Bearer +POST /auth/refresh Authorization: Bearer +``` -### Auth +### User Profile ```http -POST /auth/register { "email": "...", "password": "...", "role": "farmer|buyer" } -POST /auth/login { "email": "...", "password": "..." } → { "access_token": "..." } -GET /auth/me Authorization: Bearer -POST /auth/refresh Authorization: Bearer +GET /users/me Authorization: Bearer +PUT /users/me { "fullName": "...", "district": "Kampala", + "specialties": ["maize", "beans"], # farmers + "interests": ["Grains", "Legumes"] } # buyers +GET /users/farmers ?district=Kampala&page=1&limit=20 +GET /users/{id} ``` ### Produce ```http -GET /listings/ ?category=grains&district=Kampala&min_price=500&max_price=2000 -POST /listings/ { "title", "category", "price_per_kg", "quantity_kg", "district" } -GET /listings/{id} -PATCH /listings/{id} +GET /listings/ ?category=grains&district=Kampala&min_price=500&max_price=2000 +POST /listings/ { "title", "category", "price_per_kg", "quantity_kg", "district" } +GET /listings/{id} +PUT /listings/{id} DELETE /listings/{id} ``` @@ -373,267 +494,232 @@ DELETE /listings/{id} ```http POST /orders/ { "listing_id": "...", "quantity_kg": 100 } GET /orders/ -PATCH /orders/{id}/cancel +POST /orders/{id}/cancel POST /orders/{id}/review { "rating": 5, "comment": "..." } -GET /orders/incoming/ (farmer) -PATCH /orders/{id}/status { "new_status": "confirmed|completed|rejected" } (farmer) +GET /orders/incoming/ (farmer only) +PATCH /orders/{id}/status { "new_status": "confirmed|completed|rejected" } ``` ### Payments ```http -POST /payments/initiate { "order_id": "...", "phone": "256700000000" } +POST /payments/initiate { "order_id": "...", "phone": "256700000000" } GET /payments/{id}/status -POST /webhook/pesapal (PesaPal callback — public) +POST /webhook/pesapal (public — PesaPal callback) ``` -### Messaging & Notifications +### ML — Price Prediction (public, via Nginx) ```http -GET /message/ List conversations -POST /message/ { "recipient_id": "...", "body": "..." } -WS /message/ws/{token} Real-time message stream - -GET /notifications/ List notifications -WS /notifications/ws/{token} Real-time push stream +POST /ml/price/predict { "market": "Kisenyi_Kampala", "crop": "maize_grain", "weeks_ahead": 4 } +GET /ml/price/markets +GET /ml/price/crops ``` -### ML (via ml-gateway-service — not through Nginx) +### ML — Recommendations (JWT required, via Nginx) ```http -POST http://localhost:8000/price/predict - { "market": "Kisenyi_Kampala", "crop": "maize_grain", "weeks_ahead": 4 } +GET /ml/recommend/farmers-for-buyer/{your_user_id}?top_n=5 + Authorization: Bearer + +GET /ml/recommend/buyers-for-farmer/{your_user_id}?top_n=5 + Authorization: Bearer +``` + +The path `{your_user_id}` must be your own user ID from the JWT. The recommendation service returns 403 if you attempt to request another user's recommendations. + +### ML — Admin/Internal (bypass Nginx, dev only) -GET http://localhost:8000/recommend/farmers-for-buyer/B0001?top_n=5 -GET http://localhost:8000/recommend/buyers-for-farmer/F0001?top_n=5 -GET http://localhost:8000/health -GET http://localhost:8000/price/markets -GET http://localhost:8000/price/crops +```http +GET http://localhost:8080/health +POST http://localhost:8096/bootstrap +GET http://localhost:8096/bootstrap/status +GET http://localhost:8096/coverage +GET http://localhost:8096/gaps/summary +POST http://localhost:8095/internal/reload X-Internal-Secret: internal-secret ``` --- ## User Flows -### Farmer +### Farmer — complete flow ``` 1. POST /auth/register { role: "farmer" } 2. POST /auth/login → JWT -3. POST /listings/ List produce with price and quantity -4. GET /orders/incoming/ See buyer orders -5. PATCH /orders/{id}/status { "new_status": "confirmed" } -6. PATCH /orders/{id}/status { "new_status": "completed" } +3. PUT /users/me { specialties: ["maize", "beans"], district: "Kampala" } +4. POST /listings/ List produce with price and available quantity +5. GET /orders/incoming/ See buyer orders +6. PATCH /orders/{id}/status { new_status: "confirmed" } +7. PATCH /orders/{id}/status { new_status: "completed" } +8. GET /ml/recommend/buyers-for-farmer/{farmer_id} See matched buyers ``` -### Buyer +### Buyer — complete flow ``` 1. POST /auth/register { role: "buyer" } 2. POST /auth/login → JWT -3. GET /listings/ Browse produce (filter by district, crop, price) -4. POST /orders/ Place order -5. POST /payments/initiate Pay via Mobile Money -6. POST /orders/{id}/review Rate after completion -7. GET /recommendations/ See personalised feed +3. PUT /users/me { interests: ["Grains", "Legumes"], district: "Gulu" } +4. GET /listings/ Browse produce (filter by district, crop, price) +5. POST /orders/ Place order +6. POST /payments/initiate Pay via Mobile Money +7. POST /orders/{id}/review Rate after completion +8. GET /ml/recommend/farmers-for-buyer/{buyer_id} See matched farmers + (personalised to your interests) ``` -### Price check (USSD — no smartphone needed) +### Price check (USSD — feature phones) ``` 1. Farmer dials USSD short code -2. ussd_service calls ml-gateway-service POST /price/predict -3. 4-week maize price forecast returned as plain text to handset +2. ussd_service calls POST http://ml-gateway:8000/price/predict +3. 4-week price forecast returned as plain text to handset ``` --- -## Event System +## Event System (Kafka) -### RabbitMQ — Core stack events +The core stack publishes order events to Kafka. The ML layer consumes them for price learning and recommendation boosting. -| Event | Publisher | Consumers | Effect | -|---|---|---|---| -| `produce.listed` | Produce | Recommendation | Index new listing | -| `order.placed` | Order | Recommendation, Notification | Update feed; notify farmer | -| `order.completed` | Order | Notification | Notify buyer | -| `quality.scored` | Order | Produce, Recommendation | Update avg_rating; re-rank | -| `payment.confirmed` | Payment | Notification, Order | Unlock fulfillment | +### Topics and flows -All queues are durable. Payload schemas are in [CONTRACTS.md](CONTRACTS.md). +``` +order_service (checkout) + └── PUBLISH soko.transactions { event_type: "purchase_completed", + buyer_id, farmer_id, crop, market, + quantity_kg, price_per_kg_ugx, total_ugx } -### Kafka — ML layer events +kafka-agent (transaction consumer) + ├── PUBLISH soko.interactions { event_type: "purchase_completed", + │ buyer_id, farmer_id } + │ └── recommendation-service Kafka consumer applies +0.10 boost + │ and invalidates Redis cache for this buyer-farmer pair + └── HTTP POST data-ingestion-service /ingest/order-event + └── normalises crop name, maps district → market, inserts price_observation -| Event | Flow | -|---|---| -| `purchase_completed` | order_service → `soko.transactions` → kafka-agent → `soko.interactions` → recommendation-service (boost +0.10) | -| `price_prediction_requested` | Any service → `soko.price.requests` → kafka-agent → price-prediction-service → `soko.price.results` | -| `farmer_viewed` | recommendation_service → `soko.interactions` → recommendation-service (boost +0.02) | +location-service (Tier 3 fallback — unsupported crop) + └── PUBLISH soko.gaps { event_type: "crop_coverage_gap", + crop_submitted, category_guess, priority } + └── kafka-agent CoverageGapConsumer logs and monitors ---- +Any service + └── PUBLISH soko.price.requests { market, crop, weeks_ahead } + └── kafka-agent PriceRequestConsumer calls price-prediction-service + └── PUBLISH soko.price.results { predictions: [...] } +``` -## Getting the ML Stack Running +### Dead-letter queue -All commands run from the **project root**. Prerequisites: Docker 20+, Python 3.11, Make. +Any message that fails processing after all retries is written to `soko.dlq` with the original topic, raw value, error type, and error message — enabling offline replay and audit. -### Step 1 — Install Python dependencies +--- -```bash -make install -``` +## Getting Started -Creates a `.venv` inside each ML service folder and installs its `requirements.txt`. Prophet pulls in `pystan` — expect 3–5 minutes on first run. +All commands run from the **project root**. Prerequisites: Docker 20+, Python 3.11+, Make. -### Step 2 — Generate synthetic training data +### 1. Copy and configure environment files ```bash -make generate-data +cp services/soko-ml/.env.example services/soko-ml/.env +# Edit services/soko-ml/.env — set POSTGRES_PASSWORD and any keys ``` -Writes three files to `services/soko-ml/recommendation-service/data/raw/`: +Core service `.env` files are already populated with development defaults in each `services//.env`. -- `crop_prices_raw.csv` — 4 years of weekly UGX prices, 6 markets × 8 crops (~12,000 rows) -- `farmers.csv` — 200 synthetic farmer profiles with crop/market coverage, rating, fulfillment rate -- `buyers.csv` — 300 synthetic buyer profiles with preferred crops, markets, payment reliability +### 2. Start the core stack -Verify the output: ```bash -wc -l services/soko-ml/recommendation-service/data/raw/*.csv -# expect: 12289 crop_prices_raw.csv | 201 farmers.csv | 301 buyers.csv +make core-up +# or: docker compose up --build -d ``` -### Step 3 — Train the Prophet models - +Verify all 9 core services are healthy: ```bash -make train +curl http://localhost/health ``` -Trains 48 Prophet models (6 markets × 8 crops) with Uganda bimodal seasonality and saves `.pkl` files to `services/soko-ml/price-prediction-service/models/`. Takes 5–15 minutes depending on CPU. +### 3. Start the ML stack -> **You can skip this step.** The price-prediction-service has a built-in seasonal fallback that always returns a valid forecast. Skip `make train` for a faster first boot and run it later in the background. - -Verify: ```bash -ls services/soko-ml/price-prediction-service/models/ | wc -l -# expect: 48 +make ml-up +# or: cd services/soko-ml && docker compose --env-file .env up --build -d ``` -### Step 4 — Start the ML stack - +Watch startup logs until all services report healthy: ```bash -make up +make ml-logs ``` -Builds and starts 8 containers: Zookeeper, Kafka, kafka-init (topic creation), Redis, price-prediction-service, recommendation-service, ml-gateway-service, kafka-agent. +### 4. Seed the database with test data -Watch the startup log until you see these three lines, then proceed: ```bash -make logs -# Look for: -# soko-ml-kafka-init | All Kafka topics created. -# soko-ml-rec | {"event": "recommendation_service_started", "farmers": 200, "buyers": 300} -# soko-ml-gateway | {"event": "gateway_started"} +make seed ``` -Kafka takes ~30 seconds to elect a leader — the ML services will retry automatically. - -### Step 5 — Health check -```bash -make health -``` +Registers 13 farmers and 10 buyers via the auth API, updates their profiles (district, specialties, interests), and creates produce listings. After seeding, triggers `make ingest-bootstrap` automatically. -All three services must return `"ok"` before proceeding: - -```json -=== ML Gateway === -{ - "gateway": "ok", - "services": { "price-prediction": "ok", "recommendation": "ok" }, - "circuit_breakers": { "price-prediction": "closed", "recommendation": "closed" } -} -=== Price Service === -{ "status": "ok", "service": "price-prediction-service", "models_loaded": 48 } -=== Recommendation Service === -{ "status": "ok", "service": "recommendation-service", "farmers_loaded": 200, "buyers_loaded": 300 } -``` - -### Step 6 — Run unit tests (no Docker required) +### 5. Bootstrap the ML feature store ```bash -make test +make ingest-bootstrap ``` -Runs pytest across all three FastAPI services — pure logic tests, no Redis or Kafka needed: +Pulls all profiles from user-service into `soko_ml_db` and immediately triggers a recommendation-service reload. After this, recommendations are live. -```bash -make test-price # 11 tests — fallback predict, base UGX prices, model registry -make test-rec # 14 tests — scoring, ranking, interaction boosts, cache invalidation -make test-gateway # 11 tests — proxy, circuit breaker, health aggregation -``` - -### Step 7 — Smoke test (full round trip) +### 6. Verify the full stack ```bash -make smoke-test +make health # Check all ML service health endpoints +make smoke-test # End-to-end: price prediction + farmer recs + buyer recs ``` -Fires three live HTTP calls through the gateway and prints the JSON. On the second run, the price response returns `"cached": true` — confirming Redis is working. - ### What to look for if something fails | Symptom | Cause | Fix | |---|---|---| -| `"recommendation": "unreachable"` in health | CSVs not generated | Run `make generate-data` | -| `"models_loaded": 0` in price health | No `.pkl` files | Run `make train`, or rely on fallback | -| Gateway returns `503` | Service startup race | Wait 30 s; check `make logs` | +| `"recommendation": "unreachable"` | Feature store empty at startup | Run `make ingest-bootstrap` | +| `"models_loaded": 0"` in price health | No `.pkl` files | Run `make train`, or rely on seasonal fallback | +| Gateway returns `503` | Service startup race | Wait 30 s, check `make ml-logs` | | `kafka-init` exits immediately | Kafka not ready | It restarts automatically; wait | -| `"cached": true` on first call | Stale Redis from prior run | `make redis-cli` → `FLUSHDB` | +| Recommendations return 404 | User not in feature store | Run `make ingest-bootstrap` | +| Recommendations return 403 | JWT user_id ≠ path param | Use your own user ID in the URL | +| `"cached": true` on first call | Stale Redis from prior run | `docker exec soko-ml-redis redis-cli FLUSHDB` | --- ## Makefile Reference -All targets run from the project root. - -### Setup - -| Command | What it does | -|---|---| -| `make install` | Create `.venv` in each ML service and install deps | -| `make generate-data` | Write synthetic CSVs to `services/soko-ml/recommendation-service/data/raw/` | -| `make train` | Train 48 Prophet models → `services/soko-ml/price-prediction-service/models/` | - -### Development (local, no Docker) - -| Command | What it does | -|---|---| -| `make dev` | Full ML stack with hot reload (docker-compose.dev.yml) | -| `make dev-price` | Run price-prediction-service locally, port 8001 | -| `make dev-rec` | Run recommendation-service locally, port 8002 | -| `make dev-gateway` | Run ml-gateway-service locally, port 8000 | +All targets run from the **project root**. -### Infrastructure +### Stack lifecycle | Command | What it does | |---|---| -| `make infra-up` | Start Redis + Kafka + Zookeeper only (no ML services) | -| `make infra-down` | Stop infrastructure containers | -| `make kafka-topics` | Re-create all Kafka topics (idempotent) | -| `make kafka-ui` | List all Kafka topics in terminal | -| `make redis-cli` | Open Redis CLI in the running container | - -### Production +| `make core-up` | Start core stack (docker compose up --build -d) | +| `make core-down` | Stop core stack | +| `make ml-up` | Start ML stack (services/soko-ml) | +| `make ml-down` | Stop ML stack | +| `make up` | Start both stacks | +| `make down` | Stop both stacks | +| `make restart` | Restart ML stack | +| `make logs` / `make ml-logs` | Follow ML service logs | +| `make logs-price` | price-prediction-service logs only | +| `make logs-rec` | recommendation-service logs only | +| `make logs-gateway` | ml-gateway-service logs only | +| `make logs-agent` | kafka-agent logs only | + +### Data and models | Command | What it does | |---|---| -| `make up` | `docker-compose up --build -d` — full ML stack | -| `make down` | Stop all ML containers | -| `make restart` | `down` then `up` | -| `make logs` | Follow logs for all ML services | -| `make logs-price` | Follow price-prediction-service logs only | -| `make logs-rec` | Follow recommendation-service logs only | -| `make logs-gateway` | Follow ml-gateway-service logs only | -| `make logs-agent` | Follow kafka-agent logs only | +| `make seed` | Seed core DBs with Ugandan farmer/buyer test data | +| `make ingest-bootstrap` | Pull profiles into ML feature store + reload recommendation-service | +| `make generate-data` | Generate synthetic price CSVs for model training | +| `make train` | Train 48 Prophet models → `price-prediction-service/models/` | ### Testing @@ -643,78 +729,54 @@ All targets run from the project root. | `make test-price` | price-prediction-service tests only | | `make test-rec` | recommendation-service tests only | | `make test-gateway` | ml-gateway-service tests only | - -### Health & Smoke - -| Command | What it does | -|---|---| -| `make health` | `curl` all `/health` endpoints and print results | -| `make smoke-test` | End-to-end: price prediction + farmer recs + buyer recs | +| `make health` | Curl all ML `/health` endpoints | +| `make smoke-test` | Randomised end-to-end: price + farmer recs + buyer recs | ### Cleanup | Command | What it does | |---|---| -| `make clean` | Remove `__pycache__`, `.pyc`, venvs, generated CSVs | -| `make clean-models` | Remove trained `.pkl` model files | -| `make clean-docker` | `docker-compose down -v --rmi all` — full wipe | - ---- - -## Running Tests - -### Core stack integration tests - -Start the core stack first: -```bash -docker compose up --build -d -``` - -Then run the integration suite (hits real services, no mocks): -```bash -pip install pytest httpx -pytest tests/integration/ -v -``` - -Covers: health checks, auth, user profiles, produce listings, order placement, stock reduction, reviews, recommendation event propagation. - -### ML stack unit tests - -No Docker required — runs against local code: -```bash -make install # only needed once -make test -``` +| `make clean` | Remove `__pycache__`, `.pyc` files, venvs | +| `make clean-models` | Remove trained `.pkl` files | +| `make clean-docker` | Full ML docker wipe (`down -v --rmi all`) | --- ## Environment Variables -### Core stack (set in `docker-compose.yml`) +### Core stack (each service has its own `.env`) | Variable | Services | Description | |---|---|---| | `DATABASE_URL` | all | PostgreSQL connection string | -| `RABBITMQ_URL` | all except auth | `amqp://guest:guest@rabbitmq:5672/` | -| `SECRET_KEY` | auth + JWT-validating services | JWT signing key | -| `ALGORITHM` | same | `HS256` | -| `ACCESS_TOKEN_EXPIRE_MINUTES` | auth | Token lifetime (default 30) | -| `PRODUCE_SERVICE_URL` | order | `http://produce_service:8003` | +| `INTERNAL_SECRET` | all | Inter-service auth key (must be `internal-secret` in dev) | +| `SECRET_KEY` | auth | JWT signing key | +| `FRONTEND_URL` | auth, payment | Allowed redirect origin | +| `REDIS_URL` | produce, blog | Shared Redis for caching | +| `KAFKA_BOOTSTRAP_SERVERS` | order | `kafka:9092` (core stack's Kafka is the ML stack's Kafka) | +| `KAFKA_TRANSACTION_TOPIC` | order | `soko.transactions` | -### ML stack (template in `services/soko-ml/.env.example`) +### ML stack (`services/soko-ml/.env.example`) | Variable | Default | Description | |---|---|---| +| `POSTGRES_USER` | `soko_ml` | ML DB user | +| `POSTGRES_PASSWORD` | `changeme` | **REQUIRED: change before production** | | `REDIS_HOST` | `redis` | ML Redis hostname | -| `REDIS_PORT` | `6379` | | -| `KAFKA_BOOTSTRAP_SERVERS` | `kafka:9092` | | -| `MODEL_DIR` | `/app/models` | Path to `.pkl` files inside container | -| `FARMERS_DATA_PATH` | `/app/data/raw/farmers.csv` | | -| `BUYERS_DATA_PATH` | `/app/data/raw/buyers.csv` | | +| `KAFKA_BOOTSTRAP_SERVERS` | `kafka:9092` | Must match the Kafka started by the ML stack | +| `USER_SERVICE_URL` | `http://user_service:8002` | Core user service (via soko-ml-bridge) | +| `ORDER_SERVICE_URL` | `http://order_service:8004` | Core order service (via soko-ml-bridge) | +| `PRODUCE_SERVICE_URL` | `http://produce_service:8003` | Core produce service (via soko-ml-bridge) | +| `INTERNAL_API_KEY` | `internal-secret` | **Must match core services' `INTERNAL_SECRET`** | +| `BOOTSTRAP_ON_STARTUP` | `true` | Pull profiles from user-service at startup | +| `PROFILE_REFRESH_INTERVAL_SECONDS` | `900` | How often recommendation-service reloads from DB | | `PRICE_CACHE_TTL_SECONDS` | `86400` | 24 hours | | `REC_CACHE_TTL_SECONDS` | `3600` | 1 hour | -| `DEFAULT_TOP_N` | `5` | Default recommendation count | -| `LOG_LEVEL` | `INFO` | `DEBUG` for development | +| `GATEWAY_PORT` | `8080` | Host port for ML gateway | +| `REC_SERVICE_PORT` | `8095` | Host port for recommendation-service | +| `INGEST_SERVICE_PORT` | `8096` | Host port for data-ingestion-service | +| `LOCATION_SERVICE_PORT` | `8097` | Host port for location-service | +| `PRICE_SERVICE_PORT` | `8094` | Host port for price-prediction-service | --- @@ -722,88 +784,184 @@ make test ``` soko/ -├── Makefile ← All ML commands (run from here) -├── docker-compose.yml ← Core Soko stack +├── Makefile ← All stack commands (run from here) +├── docker-compose.yml ← Core Soko stack (9 services + DBs + Redis) ├── nginx/ -│ └── nginx.conf ← API gateway routing + auth subrequests +│ └── nginx.conf ← API gateway: routing, auth subrequests, CORS +├── scripts/ +│ ├── seed.py ← Seed core DBs with Ugandan test users + listings +│ └── smoke_test.py ← Randomised ML end-to-end test ├── services/ -│ ├── auth/ ← JWT auth, /verify-token -│ ├── user/ ← User profiles :8002 -│ ├── produce/ ← Listings, stock :8003 -│ ├── order/ ← Orders, reviews :8004 -│ ├── payment/ ← PesaPal integration :8005 -│ ├── message/ ← WebSocket messaging :8006 -│ ├── notification/ ← WebSocket push :8007 -│ ├── blog/ ← Articles :8008 -│ ├── ussd/ ← Feature-phone access :8009 -│ ├── recommendation/ ← RabbitMQ-driven feed :8010 -│ └── soko-ml/ ← ML stack (own compose) +│ ├── auth/ ← JWT auth, /verify-token :8001 +│ │ └── .env +│ ├── user/ ← User profiles :8002 +│ │ └── .env +│ ├── produce/ ← Listings, stock, Cloudinary :8003 +│ │ └── .env +│ ├── order/ ← Orders, Kafka publisher :8004 +│ │ └── .env +│ ├── payment/ ← PesaPal Mobile Money :8005 +│ │ └── .env +│ ├── message/ ← WebSocket messaging :8006 +│ │ └── .env +│ ├── notification/ ← WebSocket push :8007 +│ │ └── .env +│ ├── blog/ ← Articles, Cloudinary :8008 +│ │ └── .env +│ ├── ussd/ ← Feature-phone USSD handler :8009 +│ │ └── .env +│ └── soko-ml/ ← ML stack (own compose) │ ├── docker-compose.yml -│ ├── docker-compose.dev.yml -│ ├── .env.example +│ ├── .env.example ← Copy to .env before starting │ ├── shared/ -│ │ └── events.py ← Kafka event dataclasses -│ ├── price-prediction-service/ ← Prophet + Redis + Kafka :8001 +│ │ └── events.py ← Kafka event dataclasses +│ ├── ml-gateway-service/ ← Proxy + circuit breaker host:8080 +│ │ └── src/ +│ │ ├── main.py ← FastAPI routes, header forwarding +│ │ ├── proxy.py ← Circuit breaker, retries, fallbacks +│ │ └── logger.py +│ ├── price-prediction-service/ ← Prophet + Redis host:8094 │ │ ├── src/ -│ │ │ ├── main.py │ │ │ ├── predictor.py -│ │ │ ├── cache.py -│ │ │ ├── kafka_producer.py -│ │ │ └── schemas.py -│ │ ├── models/ ← .pkl files (gitignored) -│ │ └── tests/ -│ ├── recommendation-service/ ← Content scoring + Redis :8002 -│ │ ├── src/ -│ │ │ ├── main.py -│ │ │ ├── recommender.py -│ │ │ ├── interaction_store.py -│ │ │ ├── cache.py -│ │ │ ├── kafka_consumer.py -│ │ │ └── schemas.py -│ │ ├── data/raw/ ← farmers.csv, buyers.csv -│ │ └── tests/ -│ ├── ml-gateway-service/ ← Proxy + circuit breaker :8000 -│ │ ├── src/ -│ │ │ ├── main.py -│ │ │ ├── proxy.py -│ │ │ └── logger.py -│ │ └── tests/ -│ ├── kafka-agent/ ← Event backbone (no HTTP) -│ │ ├── src/ -│ │ │ ├── agent.py -│ │ │ ├── consumers/ -│ │ │ ├── producers/ -│ │ │ └── dlq.py -│ │ └── tests/ -│ └── data-generator/ ← One-shot CSV + model data -│ ├── generate_prices.py -│ └── generate_profiles.py +│ │ │ └── feature_store_client.py +│ │ └── models/ ← .pkl files (gitignored, make train) +│ ├── recommendation-service/ ← Content scoring + Postgres host:8095 +│ │ └── src/ +│ │ ├── main.py ← Identity validation, /internal/reload +│ │ ├── recommender.py ← Scoring algorithm +│ │ ├── feature_store_client.py +│ │ ├── interaction_store.py +│ │ └── kafka_consumer.py +│ ├── data-ingestion-service/ ← Bootstrap + streaming host:8096 +│ │ └── src/ +│ │ ├── main.py ← Bootstrap, reload notification +│ │ ├── clients/ ← user_client.py, order_client.py +│ │ ├── transformers/ ← Crop normalisation, price transform +│ │ ├── bootstrap/ ← Farmers, buyers, orders, markets +│ │ └── streams/ ← Kafka transaction consumer +│ ├── location-service/ ← Market routing host:8097 +│ │ └── src/ +│ │ ├── market_router.py ← Tier 1/2 routing +│ │ ├── fallback.py ← Tier 3 + close_pool +│ │ └── gap_notifier.py ← Coverage gap events +│ ├── kafka-agent/ ← Event backbone (no HTTP port) +│ │ └── src/ +│ │ ├── agent.py +│ │ ├── consumers/ ← Per-topic consumers +│ │ ├── producers/ +│ │ └── dlq.py +│ └── db/ +│ └── schema.sql ← ML feature store DDL └── tests/ - └── integration/ ← Core stack integration tests + └── integration/ ← Core stack integration tests ``` -Each core service follows the same internal layout: -``` -service/ -├── Dockerfile -├── requirements.txt -└── app/ - ├── main.py ← FastAPI app + lifespan - ├── config.py ← pydantic-settings - ├── database.py ← SQLAlchemy engine - ├── dependencies.py ← JWT auth - ├── messaging.py ← RabbitMQ publisher / consumer - ├── schemas.py ← Pydantic models - ├── models/ ← SQLAlchemy ORM - └── routers/ ← Route handlers -``` +--- + +## Production Bug Report + +The following bugs were identified and fixed during the ML integration audit. All fixes are in this codebase. + +### SECURITY-01 — `/recommendations/` endpoint bypassed authentication + +**Severity:** High +**Location:** `nginx/nginx.conf` + +The legacy `/recommendations/` route proxied to the ML recommendation service without any `auth_request` call. Any unauthenticated client could retrieve another user's personalised recommendations by guessing their UUID. + +**Fix:** Added `auth_request /_verify_token` with `X-User-Id` and `X-User-Role` injection, matching the protection on `/ml/recommend/`. + +--- + +### SECURITY-02 — Recommendation service accepted any user ID in path + +**Severity:** High +**Location:** `services/soko-ml/recommendation-service/src/main.py` + +The recommendation endpoints accepted `{buyer_id}` and `{farmer_id}` path parameters without checking whether the requesting user was actually that person. An authenticated attacker could harvest recommendations for any user by iterating through UUIDs. + +**Fix:** Added `_check_identity()` — reads `x-user-id` header (injected by Nginx from the JWT), compares it against the path parameter, returns 403 on mismatch. Admin role bypasses the check. + +--- + +### SECURITY-03 — ML Gateway did not forward `X-User-Id` to recommendation service + +**Severity:** High (prerequisite for SECURITY-02 fix to function) +**Location:** `services/soko-ml/ml-gateway-service/src/main.py` and `src/proxy.py` + +The gateway's `recommend_farmers` and `recommend_buyers` handlers did not accept a `Request` object and therefore could not read or forward the `x-user-id` header injected by Nginx. The recommendation service always received requests with no identity header and therefore could never enforce identity. + +**Fix:** Both recommendation handlers now accept `request: Request`, extract `x-user-id` and `x-user-role`, and pass them via the new `headers` parameter on `proxy_request()`. + +--- + +### BUG-01 — Wrong default service ports in data-ingestion clients + +**Severity:** High (breaks bootstrap on fresh install) +**Locations:** +- `services/soko-ml/data-ingestion-service/src/clients/user_client.py` — default `http://user-service:3003` (should be `8002`) +- `services/soko-ml/data-ingestion-service/src/clients/order_client.py` — default `http://order-service:3002` (should be `8004`) + +These defaults are only used when the env var is not set. If `.env` is missing or incomplete, bootstrap silently fails — no profiles are ingested, recommendations return empty results. + +**Fix:** Corrected both defaults to match the actual service ports. + +--- + +### BUG-02 — Swapped ports in `.env.example` and `docker-compose.yml` defaults + +**Severity:** Medium +**Locations:** +- `services/soko-ml/.env.example` lines 31–32 +- `services/soko-ml/docker-compose.yml` data-ingestion environment block + +`ORDER_SERVICE_URL` defaulted to port `8003` (produce service port) and `PRODUCE_SERVICE_URL` defaulted to port `8004` (order service port). These were swapped. + +**Fix:** Corrected to `ORDER_SERVICE_URL=http://order_service:8004` and `PRODUCE_SERVICE_URL=http://produce_service:8003` in both files. + +--- + +### BUG-03 — Recommendation service missing `POSTGRES_DSN` and `INTERNAL_API_KEY` in docker-compose + +**Severity:** High +**Location:** `services/soko-ml/docker-compose.yml` recommendation-service environment block + +The recommendation service loads all profiles from PostgreSQL via `feature_store_client.py`, but `POSTGRES_DSN` was not wired into the container environment. The service would use the hardcoded default DSN string which may not match the actual DB credentials. `INTERNAL_API_KEY` was also missing, meaning the `/internal/reload` endpoint would accept any call without authentication. + +**Fix:** Added `POSTGRES_DSN`, `INTERNAL_API_KEY`, `PROFILE_REFRESH_INTERVAL_SECONDS` to the recommendation-service environment. Added `soko-ml-db` to its `depends_on`. + +--- + +### BUG-04 — New users waited up to 15 minutes to appear in recommendations + +**Severity:** Medium +**Location:** `services/soko-ml/recommendation-service/src/main.py` + +The recommendation service reloads profiles from the ML feature store on a 15-minute timer (`PROFILE_REFRESH_INTERVAL_SECONDS=900`). After `make seed` or `POST /bootstrap`, newly registered users would not appear in recommendations for up to 15 minutes. + +**Fix:** Added `POST /internal/reload` endpoint to the recommendation service. Data-ingestion now calls this endpoint immediately after each successful bootstrap (both at startup and on manual trigger), reducing the lag from up to 15 minutes to under 10 seconds. + +--- + +### What to watch in production + +| Risk | Mitigation | +|---|---| +| `INTERNAL_SECRET` / `INTERNAL_API_KEY` mismatch | Keep in a shared secrets manager; both must be identical | +| Feature store staleness | Monitor `GET /ingest/bootstrap/status`; set up an alert if `farmers_ingested = 0` | +| Kafka consumer lag | Monitor `soko.transactions` consumer group `soko-ml-price-collector` lag | +| Recommendation cache too aggressive | Tune `REC_CACHE_TTL_SECONDS` down if personalisation feels stale | +| Coverage gaps accumulating | Monitor `GET /gaps/summary`; high-frequency gaps signal unmet demand | +| Prophet model staleness | Re-run `make train` as price observations accumulate (>52 per market-crop pair triggers `is_model_ready`) | --- ## Known Limitations -- **Alembic not wired** — schema changes require dropping the affected DB volume -- **Shared JWT secret** — all services share one key; use a secrets manager in production -- **`/listings/{id}/reduce-stock` is unauthenticated** — secure with an internal API key in production -- **No password reset** — requires an email provider -- **ML stack is a separate compose** — it does not share the core stack's network or Redis; the two stacks communicate over localhost ports in development and would use a shared Docker network or service mesh in production +- **Alembic not wired** — schema changes to either stack require dropping the affected DB volume +- **Shared JWT secret** — all core services share one key; use a secrets manager in production +- **Order service `/internal/orders` endpoint not implemented** — data-ingestion bootstrap skips order history and relies on live Kafka streaming for price observations instead; price models need real transaction volume before achieving 52-observation model readiness +- **Interaction boosts are in-memory only** — the `InteractionStore` in the recommendation service is not persisted; a service restart resets all boost scores (they rebuild from `soko.interactions` with `auto.offset.reset=latest`, so only future events contribute) +- **Single Kafka broker** — `KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1` is not suitable for production; deploy a 3-broker cluster with replication factor 3 +- **No password reset** — requires an outbound email provider +- **Google Maps API optional** — location-service falls back to Haversine straight-line distances when `GOOGLE_MAPS_API_KEY` is empty; transport cost estimates will be less accurate diff --git a/execution.md b/execution.md deleted file mode 100644 index 5c74101..0000000 --- a/execution.md +++ /dev/null @@ -1,237 +0,0 @@ -# Soko Phase 2 — Execution & Testing Walkthrough - -## Prerequisites - -Three config changes are required before running the stack end-to-end: - -- `data-ingestion-service` joined to `soko-ml-bridge` (so it can reach core services) -- `user_service`, `order_service`, `produce_service` joined to `soko-ml-bridge` -- `services/soko-ml/.env` updated with all Phase 2 variables - -All three are already applied in the codebase. - ---- - -## Step 1 — Unit tests (no Docker needed) - -Pure function tests — crop normalisers, transport rates, sell signal logic. Always pass offline. - -```bash -# Install Python deps (only needed once, or after adding a new service) -make install - -# Transformer tests: crop normalisation, market mapping, UGX price field names -make test-ingest - -# Location tests: haversine distance, transport cost bands, sell signal derivation -make test-location -``` - -Expected: all green. DB-dependent tests auto-skip with `pytest.skip("Postgres unreachable")` — that is correct behaviour offline. - ---- - -## Step 2 — Tear down any existing ML stack - -```bash -make ml-down -``` - ---- - -## Step 3 — Cold-start Phase 2 - -Starts the ML Postgres, applies the schema, builds and starts all seven ML services, then auto-triggers the initial data bootstrap from core services. - -```bash -make cold-start -``` - -Takes ~90 seconds. Internal sequence: -1. Ensures `soko-ml-bridge` network exists -2. Starts `soko-ml-db` and runs `db/schema.sql` -3. Builds and starts all ML containers -4. Waits 20 s for healthchecks to pass -5. POSTs to `/bootstrap` on the ingest service - -Follow progress: -```bash -make ml-logs -# Ctrl-C when all services show "Application startup complete" -``` - ---- - -## Step 4 — Restart core services on the new network - -Core service containers were started before `soko-ml-bridge` was added to them. Recreate them so they join it. - -```bash -make core-restart -``` - -After this, `data-ingestion-service` can reach `http://user_service:8002`, `http://order_service:8004`, and `http://produce_service:8003` directly over `soko-ml-bridge`. - ---- - -## Step 5 — Health check every service - -```bash -make health -``` - -| Service | Port | -|---|---| -| API gateway (nginx) | `:80` | -| ML gateway | `:8080` | -| Price prediction | `:8094` | -| Recommendation | `:8095` | -| Location service | `:8003` | -| Data ingestion | `:8096` | - -All should return `{"status":"ok"}`. If any show `UNREACHABLE`, tail their logs: - -```bash -make logs-location -make logs-ingest -make logs-gateway -``` - ---- - -## Step 6 — Check bootstrap status - -```bash -make ingest-status -``` - -Returns a JSON breakdown of farmers, produce listings, and price observations pulled from core services. If `bootstrap_complete: false`, trigger it manually: - -```bash -make ingest-bootstrap -``` - ---- - -## Step 7 — Smoke test Phase 1 features (regression check) - -Verify price predictions and recommendations are unaffected: - -```bash -make smoke-test -``` - -Hits `/price/predict` with `maize_grain` at `Kisenyi_Kampala` and both recommendation endpoints. - ---- - -## Step 8 — Smoke test Phase 2 location endpoints - -**Market routing** — ranked markets with sell signal and transport cost for a farmer: - -```bash -make smoke-route -``` - -Response includes: -- `tier` — 1 (full ML), 2 (category band), or 3 (unknown crop) -- `ranked_markets` — each with `ugx_per_kg`, `mode` (e.g. `boda_cargo`, `pickup_truck`), and `sell_signal` - -**Buyer-to-farmer discovery** — farmers near a buyer within 150 km under 2 000 UGX/kg: - -```bash -make smoke-discover -``` - -**Tier 2 fallback** — niche crop with thin ML coverage falls back to category price band: - -```bash -make smoke-fallback -``` - -**Tier 3** — completely unknown crop (`moringa`) returns a gap notification and publishes to `soko.gaps`: - -```bash -make smoke-tier3 -``` - ---- - -## Step 9 — Smoke test the ingest endpoint - -Posts a synthetic `purchase_completed` event and writes a `price_observation` row: - -```bash -make smoke-ingest -``` - -Confirm it landed in the DB: - -```bash -make db-shell -``` - -Inside psql: - -```sql -SELECT crop, market, price_per_kg, currency, source -FROM price_observations -ORDER BY observed_at DESC -LIMIT 5; -\q -``` - -Expected: `currency = UGX`, `price_per_kg = 1400` for event `TEST-001`. - ---- - -## Step 10 — Gap report - -After the Tier 3 smoke test, `moringa` will appear here: - -```bash -make gaps-summary -``` - -Returns coverage counts per crop/market and a `gap_level` (`low` / `medium` / `high`) based on the thresholds in `services/soko-ml/.env`. - ---- - -## Quick reference — troubleshooting - -| Problem | Command | -|---|---| -| DB schema missing | `make db-reset` *(destructive)* | -| Kafka topics not created | `make kafka-topics` | -| Redis full / stale cache | `make redis-cli` → `FLUSHALL` | -| Re-run bootstrap | `make ingest-bootstrap` | -| Wipe everything and restart | `make clean-docker` then `make cold-start` | -| Rebuild one service only | `docker compose -f services/soko-ml/docker-compose.yml up --build -d ` | - ---- - -## Port map - -| Service | Host port | Container port | -|---|---|---| -| nginx (API gateway) | 80 | 80 | -| ML gateway | 8080 | 8000 | -| Price prediction | 8094 | 8001 | -| Recommendation | 8095 | 8002 | -| Location service | 8003 | 8003 | -| Data ingestion | 8096 | 8004 | -| Kafka (external) | 29092 | 29092 | - ---- - -## Transport cost reference (FarasUG / SafeBoda benchmarks) - -| Distance | Mode | UGX/kg | -|---|---|---| -| 0 – 25 km | boda_cargo | 290 | -| 25 – 80 km | taxi_van | 420 | -| 80 – 200 km | pickup_truck | 620 | -| 200 – 400 km | shared_lorry | 850 | -| 400+ km | cross_region | 1 100 | - -Rates reflect partial-load pricing for 100–500 kg loads. All monetary values in UGX. diff --git a/nginx/nginx.conf b/nginx/nginx.conf index b6c412b..431fd2d 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -269,14 +269,25 @@ http { # ════════════════════════════════════════ # RECOMMENDATION SERVICE — routed via ml_gateway + # /recommendations/* is a convenience alias for /ml/recommend/* + # Both require a valid JWT — personal data must never be public. # ════════════════════════════════════════ location /recommendations/ { limit_req zone=api_limit burst=20 nodelay; + + auth_request /_verify_token; + auth_request_set $user_id $upstream_http_x_user_id; + auth_request_set $user_role $upstream_http_x_user_role; + error_page 401 = @error401; + error_page 403 = @error403; + set $ml_gw "ml-gateway:8000"; proxy_pass http://$ml_gw/recommend/; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-User-Id $user_id; + proxy_set_header X-User-Role $user_role; } # ════════════════════════════════════════ diff --git a/scripts/seed.py b/scripts/seed.py new file mode 100644 index 0000000..4a96dce --- /dev/null +++ b/scripts/seed.py @@ -0,0 +1,709 @@ +#!/usr/bin/env python3 +""" +Soko development seed script. + +Flow: + 1. Register farmers + buyers via auth service (creates auth_db + user_db records) + 2. Update farmer profiles (bio, farm name) via user service + 3. Farmers create produce listings → activate them via update endpoint + 4. Trigger data-ingestion bootstrap → populates ML feature store + 5. Restart recommendation service → loads new profiles immediately + 6. Print a summary with real IDs for smoke tests + +Calls all services directly on their host ports with injected x-user-id / x-user-role +headers — the same headers nginx injects after JWT verification. +""" + +import json +import subprocess +import sys +import time + +import requests + +AUTH = "http://localhost:8001" +USER = "http://localhost:8002" +PRODUCE = "http://localhost:8003" +INGEST = "http://localhost:8096" + +PASS = "Soko2024!" + +# ── Farmer data ─────────────────────────────────────────────────────────────── + +FARMERS = [ + { + "fullName": "Nakato Aisha", + "email": "nakato.aisha@sokodev.ug", + "phone": "+256701123001", + "district": "Kampala", + "village": "Natete", + "role": "farmer", + "specialties": ["maize_grain", "sorghum"], + "farmName": "Nakato Family Grains", + "farmerBio": "Third-generation cereal farmer from Natete. Supplying Kampala markets for over 15 years.", + "listings": [ + { + "name": "Premium Maize Grain", "category": "Grains", + "district": "Kampala", "village": "Natete", + "description": "Sun-dried grade-A maize grain, ready for milling or animal feed.", + "tags": ["maize", "grains", "wholesale"], + "price": 1300, "unit": "kg", "totalQty": 500, "minimumOrder": 50, + "fresh": False, "harvestDate": "2026-04-10", + "storage": "Store in cool, dry place. Lasts 6 months.", + "priceTiers": [{"minQty": 100, "price": 1250, "label": "100+ kg"}, + {"minQty": 300, "price": 1200, "label": "300+ kg"}], + }, + { + "name": "Sorghum Grain", "category": "Grains", + "district": "Kampala", "village": "Natete", + "description": "Clean sorghum grain, ideal for brewing and animal nutrition.", + "tags": ["sorghum", "grains"], + "price": 1100, "unit": "kg", "totalQty": 300, "minimumOrder": 30, + "fresh": False, "harvestDate": "2026-04-15", + }, + ], + }, + { + "fullName": "Ssebuliba John", + "email": "ssebuliba.john@sokodev.ug", + "phone": "+256702123002", + "district": "Mbarara", + "village": "Rutooma", + "role": "farmer", + "specialties": ["irish_potatoes", "matoke"], + "farmName": "Ssebuliba Highland Farm", + "farmerBio": "Highland farmer specialising in Irish potatoes and matoke in the Ankole region.", + "listings": [ + { + "name": "Irish Potatoes (Desiree)", "category": "Vegetables", + "district": "Mbarara", "village": "Rutooma", + "description": "Clean Desiree potatoes harvested from rich highland soils of Mbarara.", + "tags": ["potatoes", "vegetables", "fresh"], + "price": 850, "unit": "kg", "totalQty": 800, "minimumOrder": 100, + "fresh": True, "harvestDate": "2026-05-01", + "priceTiers": [{"minQty": 200, "price": 800, "label": "200+ kg"}, + {"minQty": 500, "price": 750, "label": "500+ kg"}], + }, + { + "name": "Matoke (Bogoya Cluster)", "category": "Fruits", + "district": "Mbarara", "village": "Rutooma", + "description": "Traditional Ankole matoke bunches. Harvested green, ripens within 3 days.", + "tags": ["matoke", "banana", "staple"], + "price": 650, "unit": "bunch", "totalQty": 200, "minimumOrder": 10, + "fresh": True, "harvestDate": "2026-05-05", + }, + ], + }, + { + "fullName": "Okello David", + "email": "okello.david@sokodev.ug", + "phone": "+256703123003", + "district": "Gulu", + "village": "Layibi", + "role": "farmer", + "specialties": ["sorghum", "millet", "maize_grain"], + "farmName": "Okello Northern Grains", + "farmerBio": "Northern Uganda grain farmer serving Gulu town markets. Drought-resistant varieties only.", + "listings": [ + { + "name": "White Sorghum", "category": "Grains", + "district": "Gulu", "village": "Layibi", + "description": "White sorghum, low tannin variety suited for food and malting.", + "tags": ["sorghum", "northern", "grains"], + "price": 950, "unit": "kg", "totalQty": 400, "minimumOrder": 50, + "fresh": False, "harvestDate": "2026-03-20", + }, + { + "name": "Finger Millet (Eleusine)", "category": "Grains", + "district": "Gulu", "village": "Layibi", + "description": "Nutrient-dense finger millet. Popular for millet bread and porridge.", + "tags": ["millet", "grains", "nutritious"], + "price": 1900, "unit": "kg", "totalQty": 250, "minimumOrder": 25, + "fresh": False, "harvestDate": "2026-03-25", + }, + ], + }, + { + "fullName": "Nabukeera Grace", + "email": "nabukeera.grace@sokodev.ug", + "phone": "+256704123004", + "district": "Masaka", + "village": "Bukakata", + "role": "farmer", + "specialties": ["matoke", "cassava_chips"], + "farmName": "Nabukeera Lakeside Farm", + "farmerBio": "Family farm on the shores of Lake Victoria. Matoke and cassava our staples.", + "listings": [ + { + "name": "Matoke (Mpologoma)", "category": "Fruits", + "district": "Masaka", "village": "Bukakata", + "description": "Large Mpologoma matoke variety from Masaka. Sweet taste, firm flesh.", + "tags": ["matoke", "lakeside", "premium"], + "price": 620, "unit": "bunch", "totalQty": 300, "minimumOrder": 10, + "fresh": True, "harvestDate": "2026-05-08", + }, + { + "name": "Sun-Dried Cassava Chips", "category": "Other", + "district": "Masaka", "village": "Bukakata", + "description": "High-quality cassava chips dried on raised beds. Free from mould.", + "tags": ["cassava", "chips", "dried"], + "price": 920, "unit": "kg", "totalQty": 400, "minimumOrder": 50, + "fresh": False, "harvestDate": "2026-04-01", + }, + ], + }, + { + "fullName": "Mugisha Robert", + "email": "mugisha.robert@sokodev.ug", + "phone": "+256705123005", + "district": "Mbale", + "village": "Nakaloke", + "role": "farmer", + "specialties": ["yellow_beans", "maize_grain"], + "farmName": "Mugisha Mt Elgon Farms", + "farmerBio": "Eastern Uganda bean specialist. Mt Elgon volcanic soils produce exceptionally rich beans.", + "listings": [ + { + "name": "Yellow Beans (K132)", "category": "Grains", + "district": "Mbale", "village": "Nakaloke", + "description": "K132 yellow beans — high protein, clean grade. Ideal for export and local markets.", + "tags": ["beans", "yellow", "protein", "export-grade"], + "price": 2900, "unit": "kg", "totalQty": 500, "minimumOrder": 50, + "fresh": False, "harvestDate": "2026-04-20", + "priceTiers": [{"minQty": 100, "price": 2800, "label": "100+ kg"}, + {"minQty": 300, "price": 2700, "label": "300+ kg"}], + }, + { + "name": "Hybrid Maize Grain (H614)", "category": "Grains", + "district": "Mbale", "village": "Nakaloke", + "description": "H614 hybrid maize with high yield. Suitable for milling and silage.", + "tags": ["maize", "hybrid", "grains"], + "price": 1150, "unit": "kg", "totalQty": 800, "minimumOrder": 100, + "fresh": False, "harvestDate": "2026-04-18", + }, + ], + }, + { + "fullName": "Atim Sarah", + "email": "atim.sarah@sokodev.ug", + "phone": "+256706123006", + "district": "Lira", + "village": "Adyel", + "role": "farmer", + "specialties": ["millet", "sorghum"], + "farmName": "Atim Savannah Grains", + "farmerBio": "Lira-based grain farmer producing traditional Acholi millet and sorghum varieties.", + "listings": [ + { + "name": "Finger Millet (Okileng)", "category": "Grains", + "district": "Lira", "village": "Adyel", + "description": "Traditional Acholi Okileng millet. Prized for authentic taste in local brewing.", + "tags": ["millet", "traditional", "acholi"], + "price": 1850, "unit": "kg", "totalQty": 300, "minimumOrder": 25, + "fresh": False, "harvestDate": "2026-03-15", + }, + { + "name": "Red Sorghum", "category": "Grains", + "district": "Lira", "village": "Adyel", + "description": "Red sorghum with high tannin content. Preferred for local brew (Kwete).", + "tags": ["sorghum", "red", "traditional"], + "price": 920, "unit": "kg", "totalQty": 500, "minimumOrder": 50, + "fresh": False, "harvestDate": "2026-03-18", + }, + ], + }, + { + "fullName": "Kawesi Peter", + "email": "kawesi.peter@sokodev.ug", + "phone": "+256707123007", + "district": "Kampala", + "village": "Wakiso", + "role": "farmer", + "specialties": ["tomatoes", "maize_grain"], + "farmName": "Kawesi Peri-Urban Farm", + "farmerBio": "Peri-urban farmer supplying fresh tomatoes and grain to Kampala markets year-round.", + "listings": [ + { + "name": "Roma Tomatoes", "category": "Vegetables", + "district": "Kampala", "village": "Wakiso", + "description": "Firm Roma tomatoes, vine-ripened. Ideal for processing and fresh market.", + "tags": ["tomatoes", "fresh", "vegetables"], + "price": 1500, "unit": "kg", "totalQty": 200, "minimumOrder": 20, + "fresh": True, "harvestDate": "2026-05-12", + }, + { + "name": "Dry Maize Grain", "category": "Grains", + "district": "Kampala", "village": "Wakiso", + "description": "Well-dried maize grain at 12% moisture. Ready for milling.", + "tags": ["maize", "grains", "milled"], + "price": 1300, "unit": "kg", "totalQty": 600, "minimumOrder": 50, + "fresh": False, "harvestDate": "2026-04-22", + }, + ], + }, + { + "fullName": "Nambi Faith", + "email": "nambi.faith@sokodev.ug", + "phone": "+256708123008", + "district": "Mbarara", + "village": "Mbarara Town", + "role": "farmer", + "specialties": ["yellow_beans", "irish_potatoes", "matoke"], + "farmName": "Nambi Mixed Produce", + "farmerBio": "Diversified farm in Mbarara producing beans, potatoes, and matoke for western Uganda trade.", + "listings": [ + { + "name": "Yellow Beans (Nambale)", "category": "Grains", + "district": "Mbarara", "village": "Mbarara Town", + "description": "Nambale yellow beans — large seed size, excellent cooking quality.", + "tags": ["beans", "yellow", "western"], + "price": 3000, "unit": "kg", "totalQty": 400, "minimumOrder": 50, + "fresh": False, "harvestDate": "2026-04-28", + }, + { + "name": "White Irish Potatoes", "category": "Vegetables", + "district": "Mbarara", "village": "Mbarara Town", + "description": "White-skin Irish potatoes from Mbarara highland. Consistent sizing.", + "tags": ["potatoes", "vegetables", "white"], + "price": 850, "unit": "kg", "totalQty": 600, "minimumOrder": 100, + "fresh": True, "harvestDate": "2026-05-02", + }, + ], + }, + { + "fullName": "Oluru Emmanuel", + "email": "oluru.emmanuel@sokodev.ug", + "phone": "+256709123009", + "district": "Gulu", + "village": "Pece", + "role": "farmer", + "specialties": ["maize_grain", "millet"], + "farmName": "Oluru Peace Farm", + "farmerBio": "Gulu farmer selling certified maize seed and food-grade millet to northern traders.", + "listings": [ + { + "name": "Certified Maize (Longe 5)", "category": "Grains", + "district": "Gulu", "village": "Pece", + "description": "Longe 5 certified maize — open pollinated, drought tolerant. 10 t/ha potential.", + "tags": ["maize", "certified", "seed", "drought-tolerant"], + "price": 1100, "unit": "kg", "totalQty": 1000, "minimumOrder": 100, + "fresh": False, "harvestDate": "2026-04-05", + "priceTiers": [{"minQty": 200, "price": 1050, "label": "200+ kg"}, + {"minQty": 500, "price": 1000, "label": "500+ kg"}], + }, + { + "name": "Finger Millet (Seremi 2)", "category": "Grains", + "district": "Gulu", "village": "Pece", + "description": "Seremi 2 improved millet. High yielding, early maturing.", + "tags": ["millet", "improved", "northern"], + "price": 1900, "unit": "kg", "totalQty": 200, "minimumOrder": 20, + "fresh": False, "harvestDate": "2026-04-08", + }, + ], + }, + { + "fullName": "Kyomugisha Miriam", + "email": "kyomugisha.miriam@sokodev.ug", + "phone": "+256710123010", + "district": "Masaka", + "village": "Kalungu", + "role": "farmer", + "specialties": ["cassava_chips", "matoke", "yellow_beans"], + "farmName": "Kyomugisha Diversified Farms", + "farmerBio": "Masaka-based farmer growing cassava, matoke, and beans. NAADS certified processor.", + "listings": [ + { + "name": "Cassava Chips (NAADS Grade A)", "category": "Other", + "district": "Masaka", "village": "Kalungu", + "description": "NAADS-certified cassava chips processed at 14% moisture. Ready for animal feed or starch.", + "tags": ["cassava", "chips", "naads", "certified"], + "price": 920, "unit": "kg", "totalQty": 300, "minimumOrder": 50, + "fresh": False, "harvestDate": "2026-03-30", + }, + { + "name": "Yellow Beans (Kablanketi)", "category": "Grains", + "district": "Masaka", "village": "Kalungu", + "description": "Kablanketi yellow beans from Masaka. Large seed, rich flavour.", + "tags": ["beans", "yellow", "masaka"], + "price": 3100, "unit": "kg", "totalQty": 400, "minimumOrder": 50, + "fresh": False, "harvestDate": "2026-04-25", + }, + ], + }, +] + +# ── Buyer data ──────────────────────────────────────────────────────────────── + +BUYERS = [ + { + "fullName": "Ssali Martin", + "email": "ssali.martin@sokodev.ug", + "phone": "+256711200001", + "district": "Kampala", + "role": "buyer", + "interests": ["maize_grain", "sorghum"], + }, + { + "fullName": "Nansubuga Rachel", + "email": "nansubuga.rachel@sokodev.ug", + "phone": "+256712200002", + "district": "Mbarara", + "role": "buyer", + "interests": ["irish_potatoes", "tomatoes"], + }, + { + "fullName": "Opio Samuel", + "email": "opio.samuel@sokodev.ug", + "phone": "+256713200003", + "district": "Gulu", + "role": "buyer", + "interests": ["millet", "sorghum"], + }, + { + "fullName": "Nakimuli Diana", + "email": "nakimuli.diana@sokodev.ug", + "phone": "+256714200004", + "district": "Masaka", + "role": "buyer", + "interests": ["matoke", "yellow_beans"], + }, + { + "fullName": "Kiggundu Alex", + "email": "kiggundu.alex@sokodev.ug", + "phone": "+256715200005", + "district": "Mbale", + "role": "buyer", + "interests": ["maize_grain", "yellow_beans"], + }, + { + "fullName": "Katende Brian", + "email": "katende.brian@sokodev.ug", + "phone": "+256716200006", + "district": "Kampala", + "role": "buyer", + "interests": ["tomatoes", "maize_grain"], + }, + { + "fullName": "Birungi Agnes", + "email": "birungi.agnes@sokodev.ug", + "phone": "+256717200007", + "district": "Mbarara", + "role": "buyer", + "interests": ["matoke", "irish_potatoes"], + }, + { + "fullName": "Odong Charles", + "email": "odong.charles@sokodev.ug", + "phone": "+256718200008", + "district": "Lira", + "role": "buyer", + "interests": ["millet", "maize_grain"], + }, + { + "fullName": "Nassaka Joyce", + "email": "nassaka.joyce@sokodev.ug", + "phone": "+256719200009", + "district": "Masaka", + "role": "buyer", + "interests": ["cassava_chips", "yellow_beans"], + }, + { + "fullName": "Tumwesige Paul", + "email": "tumwesige.paul@sokodev.ug", + "phone": "+256720200010", + "district": "Mbale", + "role": "buyer", + "interests": ["yellow_beans", "sorghum"], + }, +] + +EXTRA_FARMERS = [ + { + "fullName": "Asiimwe Patrick", + "email": "asiimwe.patrick@sokodev.ug", + "phone": "+256721300001", + "district": "Mbarara", + "village": "Bwizibwera", + "role": "farmer", + "specialties": ["irish_potatoes", "yellow_beans", "matoke"], + "farmName": "Asiimwe Highlands", + "farmerBio": "Mixed produce farmer in Mbarara highlands. Supplying hotels and supermarkets.", + "listings": [ + { + "name": "Highland Irish Potatoes", "category": "Vegetables", + "district": "Mbarara", "village": "Bwizibwera", + "description": "Premium highland potatoes, consistent 60-80g sizing for hotel supply.", + "tags": ["potatoes", "hotel-grade", "highland"], + "price": 900, "unit": "kg", "totalQty": 1000, "minimumOrder": 100, + "fresh": True, "harvestDate": "2026-05-10", + "priceTiers": [{"minQty": 200, "price": 850, "label": "200+ kg"}, + {"minQty": 500, "price": 800, "label": "500+ kg"}], + }, + { + "name": "Yellow Beans (NABE 4)", "category": "Grains", + "district": "Mbarara", "village": "Bwizibwera", + "description": "NABE 4 improved bean variety. High iron, good shelf life.", + "tags": ["beans", "yellow", "improved", "iron-rich"], + "price": 3000, "unit": "kg", "totalQty": 300, "minimumOrder": 50, + "fresh": False, "harvestDate": "2026-04-30", + }, + ], + }, + { + "fullName": "Achen Mary", + "email": "achen.mary@sokodev.ug", + "phone": "+256722300002", + "district": "Lira", + "village": "Ojwina", + "role": "farmer", + "specialties": ["millet", "sorghum"], + "farmName": "Achen Grain Co-op", + "farmerBio": "Women-led grain cooperative in Lira. Aggregating from 12 smallholder farms.", + "listings": [ + { + "name": "Millet (Co-op Aggregate)", "category": "Grains", + "district": "Lira", "village": "Ojwina", + "description": "Aggregated finger millet from 12 farms. Uniform grade, clean.", + "tags": ["millet", "cooperative", "bulk"], + "price": 1850, "unit": "kg", "totalQty": 600, "minimumOrder": 50, + "fresh": False, "harvestDate": "2026-03-28", + "priceTiers": [{"minQty": 200, "price": 1800, "label": "200+ kg"}, + {"minQty": 400, "price": 1750, "label": "400+ kg"}], + }, + ], + }, + { + "fullName": "Waiswa Daniel", + "email": "waiswa.daniel@sokodev.ug", + "phone": "+256723300003", + "district": "Mbale", + "village": "Namawojjolo", + "role": "farmer", + "specialties": ["maize_grain", "yellow_beans"], + "farmName": "Waiswa Eastern Farms", + "farmerBio": "Eastern Uganda grain trader and farmer. Direct links to Mbale grain market.", + "listings": [ + { + "name": "Maize Grain (Mbale Market Grade)", "category": "Grains", + "district": "Mbale", "village": "Namawojjolo", + "description": "Market-grade maize at 13% moisture. Bagged in 90kg sacks.", + "tags": ["maize", "grains", "market-grade"], + "price": 1150, "unit": "kg", "totalQty": 900, "minimumOrder": 90, + "fresh": False, "harvestDate": "2026-04-12", + }, + { + "name": "Yellow Beans (Bam 1)", "category": "Grains", + "district": "Mbale", "village": "Namawojjolo", + "description": "Bam 1 bush bean variety. Good cooking quality, medium seed.", + "tags": ["beans", "yellow", "eastern"], + "price": 2800, "unit": "kg", "totalQty": 400, "minimumOrder": 50, + "fresh": False, "harvestDate": "2026-04-20", + }, + ], + }, +] + + +# ── Helpers ─────────────────────────────────────────────────────────────────── + +def ok(label: str, resp: requests.Response) -> dict: + if not resp.ok: + print(f" ✗ {label}: {resp.status_code} — {resp.text[:200]}") + sys.exit(1) + data = resp.json() + print(f" ✓ {label}") + return data + + +def register_or_login(label: str, payload: dict) -> dict | None: + """Register a user; if already exists (409) log in instead to recover their ID.""" + resp = requests.post(f"{AUTH}/register", json=payload) + if resp.status_code == 409: + login_resp = requests.post(f"{AUTH}/login", json={ + "email": payload["email"], "password": payload["password"] + }) + if login_resp.ok: + print(f" ~ {label} (already exists, recovered)") + return login_resp.json() + print(f" ~ {label} (already exists, skipped)") + return None + return ok(label, resp) + + +def farmer_headers(user_id: str) -> dict: + return {"X-User-Id": user_id, "X-User-Role": "farmer"} + + +def buyer_headers(user_id: str) -> dict: + return {"X-User-Id": user_id, "X-User-Role": "buyer"} + + +# ── Phase 1: Register all users ─────────────────────────────────────────────── + +def register_users(): + print("\n── Phase 1: Registering users ──────────────────────────────────") + created_farmers, created_buyers = [], [] + + all_farmers = FARMERS + EXTRA_FARMERS + for f in all_farmers: + payload = { + "fullName": f["fullName"], + "email": f["email"], + "password": PASS, + "phone": f["phone"], + "district": f["district"], + "role": f["role"], + "specialties": f["specialties"], + } + data = register_or_login(f"Farmer: {f['fullName']}", payload) + if data: + created_farmers.append({**f, "id": data["user"]["id"]}) + + for b in BUYERS: + payload = { + "fullName": b["fullName"], + "email": b["email"], + "password": PASS, + "phone": b["phone"], + "district": b["district"], + "role": b["role"], + "interests": b["interests"], + } + data = register_or_login(f"Buyer: {b['fullName']}", payload) + if data: + created_buyers.append({**b, "id": data["user"]["id"]}) + + return created_farmers, created_buyers + + +# ── Phase 2: Update farmer profiles ────────────────────────────────────────── + +def update_profiles(farmers: list): + print("\n── Phase 2: Updating farmer profiles ───────────────────────────") + for f in farmers: + payload = { + "farmName": f["farmName"], + "farmerBio": f["farmerBio"], + "village": f["village"], + "specialties": f["specialties"], + } + resp = requests.put( + f"{USER}/users/me", + json=payload, + headers=farmer_headers(f["id"]), + ) + ok(f"Profile update: {f['fullName']}", resp) + + +# ── Phase 3: Create + activate listings ────────────────────────────────────── + +def create_listings(farmers: list) -> list: + print("\n── Phase 3: Creating produce listings ──────────────────────────") + all_listings = [] + + for f in farmers: + fid = f["id"] + headers = farmer_headers(fid) + + for listing_data in f["listings"]: + # Create as draft + resp = requests.post(f"{PRODUCE}/listings/", json=listing_data, headers=headers) + result = ok(f" Draft: {listing_data['name']} ({f['fullName']})", resp) + listing_id = result["id"] + + # Activate by setting status — update endpoint has no image requirement + resp = requests.put( + f"{PRODUCE}/listings/{listing_id}", + json={"status": "active"}, + headers=headers, + ) + ok(f" Publish: {listing_data['name']}", resp) + + all_listings.append({ + "id": listing_id, + "name": listing_data["name"], + "farmer": f["fullName"], + "district": listing_data["district"], + }) + + return all_listings + + +# ── Phase 4: Trigger ML bootstrap ──────────────────────────────────────────── + +def trigger_bootstrap(): + print("\n── Phase 4: Triggering data-ingestion bootstrap ────────────────") + resp = requests.post(f"{INGEST}/bootstrap") + ok("Bootstrap triggered", resp) + print(" Waiting 15s for bootstrap to complete...") + time.sleep(15) + + # Confirm counts + resp = requests.get(f"{INGEST}/bootstrap/status") + if resp.ok: + s = resp.json() + print(f" farmers_ingested : {s['farmers_ingested']}") + print(f" buyers_ingested : {s['buyers_ingested']}") + print(f" orders_ingested : {s['orders_ingested']}") + print(f" coverage_pairs : {s['coverage_pairs']}") + + +# ── Phase 5: Restart recommendation service ─────────────────────────────────── + +def reload_recommendation_service(): + print("\n── Phase 5: Reloading recommendation service ───────────────────") + subprocess.run( + ["docker", "compose", "restart", "recommendation-service"], + cwd="/home/the-icemann/Documents/soko/services/soko-ml", + check=True, + ) + print(" Waiting 10s for service to come up...") + time.sleep(10) + + resp = requests.get("http://localhost:8095/health") + if resp.ok: + h = resp.json() + print(f" farmers_loaded : {h.get('farmers_loaded', '?')}") + print(f" buyers_loaded : {h.get('buyers_loaded', '?')}") + + +# ── Summary ─────────────────────────────────────────────────────────────────── + +def print_summary(farmers: list, buyers: list, listings: list): + print("\n" + "═" * 60) + print("SEED COMPLETE — copy these IDs for smoke tests") + print("═" * 60) + + print(f"\nFarmers ({len(farmers)}):") + for f in farmers: + print(f" {f['id']} {f['fullName']:<25} {f['district']}") + + print(f"\nBuyers ({len(buyers)}):") + for b in buyers: + print(f" {b['id']} {b['fullName']:<25} {b['district']}") + + print(f"\nListings ({len(listings)}) — all active:") + for l in listings: + print(f" {l['id']} {l['name']:<35} {l['farmer']}") + + if farmers: + print(f"\nSample smoke-test commands:") + fid = farmers[0]["id"] + bid = buyers[0]["id"] + print(f" curl -s 'http://localhost:8080/recommend/farmers-for-buyer/{bid}?top_n=3' | python3 -m json.tool") + print(f" curl -s 'http://localhost:8080/recommend/buyers-for-farmer/{fid}?top_n=3' | python3 -m json.tool") + + print() + + +# ── Entry point ─────────────────────────────────────────────────────────────── + +if __name__ == "__main__": + print("Soko seed script starting...") + + farmers, buyers = register_users() + update_profiles(farmers) + listings = create_listings(farmers) + trigger_bootstrap() + reload_recommendation_service() + print_summary(farmers, buyers, listings) diff --git a/scripts/smoke_test.py b/scripts/smoke_test.py new file mode 100644 index 0000000..bebcf53 --- /dev/null +++ b/scripts/smoke_test.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +""" +Randomised smoke test for the Soko ML stack. + +Each run picks random farmers and buyers from the live feature store, +so no two runs hit the exact same pair. +""" + +import json +import subprocess +import sys + +import requests + +GATEWAY = "http://localhost:8080" +REC = "http://localhost:8095" +N_PAIRS = 3 # number of random buyer→farmer and farmer→buyer pairs to test + + +def psql(query: str) -> list[tuple]: + """Run a query against soko_ml_db and return rows as (col1, col2, ...) tuples.""" + result = subprocess.run( + ["docker", "exec", "soko-ml-db", + "psql", "-U", "soko_ml", "-d", "soko_ml_db", + "-At", "-F", "|", "-c", query], + capture_output=True, text=True + ) + rows = [] + for line in result.stdout.strip().splitlines(): + line = line.strip() + if line: + rows.append(tuple(p.strip() for p in line.split("|"))) + return rows + + +def random_buyers(n: int) -> list[tuple[str, str]]: + rows = psql(f"SELECT buyer_id, name FROM buyer_features ORDER BY RANDOM() LIMIT {n};") + return [(r[0], r[1]) for r in rows] + + +def random_farmers(n: int) -> list[tuple[str, str]]: + rows = psql(f"SELECT farmer_id, name FROM farmer_features ORDER BY RANDOM() LIMIT {n};") + return [(r[0], r[1]) for r in rows] + + +def section(title: str): + print(f"\n{'─' * 60}") + print(f" {title}") + print(f"{'─' * 60}") + + +def hit(url: str) -> dict | None: + try: + resp = requests.get(url, timeout=10) + if resp.ok: + return resp.json() + print(f" HTTP {resp.status_code}: {resp.text[:120]}") + return None + except Exception as e: + print(f" ERROR: {e}") + return None + + +def post(url: str, body: dict) -> dict | None: + try: + resp = requests.post(url, json=body, timeout=10) + if resp.ok: + return resp.json() + print(f" HTTP {resp.status_code}: {resp.text[:120]}") + return None + except Exception as e: + print(f" ERROR: {e}") + return None + + +# ── 0. Service health ───────────────────────────────────────────────────────── + +section("Service health") +health = hit(f"{REC}/health") +if health: + print(f" farmers_loaded : {health.get('farmers_loaded')}") + print(f" buyers_loaded : {health.get('buyers_loaded')}") +else: + print(" Recommendation service unreachable — aborting.") + sys.exit(1) + + +# ── 1. Price prediction ─────────────────────────────────────────────────────── + +section("Price prediction — Kisenyi_Kampala / maize_grain") +result = post(f"{GATEWAY}/price/predict", { + "market": "Kisenyi_Kampala", "crop": "maize_grain", "weeks_ahead": 4 +}) +if result: + print(f" cached : {result.get('cached')}") + for p in result.get("predictions", []): + print(f" {p['date']} {p['predicted_price_ugx']:>6} UGX/kg" + f" [{p['lower_bound']}–{p['upper_bound']}]") + + +# ── 2. Farmers for random buyers ────────────────────────────────────────────── + +buyers = random_buyers(N_PAIRS) +if not buyers: + print("\nNo buyers in feature store — run `make seed` first.") + sys.exit(1) + +for buyer_id, name in buyers: + section(f"Farmers for buyer — {name}") + data = hit(f"{GATEWAY}/recommend/farmers-for-buyer/{buyer_id}?top_n=3") + if data: + for i, f in enumerate(data.get("recommended_farmers", []), 1): + print(f" #{i} {f['name']:<25} {f['district']:<12}" + f" score={f['matchScore']} crops={f['specialties']}") + if not data.get("recommended_farmers"): + print(" (no recommendations returned)") + + +# ── 3. Buyers for random farmers ────────────────────────────────────────────── + +farmers = random_farmers(N_PAIRS) +if not farmers: + print("\nNo farmers in feature store — run `make seed` first.") + sys.exit(1) + +for farmer_id, name in farmers: + section(f"Buyers for farmer — {name}") + data = hit(f"{GATEWAY}/recommend/buyers-for-farmer/{farmer_id}?top_n=3") + if data: + for i, b in enumerate(data.get("recommended_buyers", []), 1): + print(f" #{i} {b['name']:<25} {b['district']:<12}" + f" score={b['matchScore']}") + if not data.get("recommended_buyers"): + print(" (no recommendations returned)") + +print(f"\n{'═' * 60}") +print(" Smoke test complete") +print(f"{'═' * 60}\n") diff --git a/services/soko-ml/.env.example b/services/soko-ml/.env.example index 54991dd..90cd09b 100644 --- a/services/soko-ml/.env.example +++ b/services/soko-ml/.env.example @@ -28,8 +28,8 @@ KAFKA_GAPS_TOPIC=soko.gaps # ── Backend service URLs (reachable via soko-ml-bridge network) ─────────────── USER_SERVICE_URL=http://user-service:8002 # REQUIRED -ORDER_SERVICE_URL=http://order-service:8003 # REQUIRED -PRODUCE_SERVICE_URL=http://produce-service:8004 # REQUIRED +ORDER_SERVICE_URL=http://order-service:8004 # REQUIRED +PRODUCE_SERVICE_URL=http://produce-service:8003 # REQUIRED # Shared internal API key — must match INTERNAL_SECRET in core services INTERNAL_API_KEY=internal-secret # REQUIRED: change before production diff --git a/services/soko-ml/data-ingestion-service/src/clients/order_client.py b/services/soko-ml/data-ingestion-service/src/clients/order_client.py index 873c68b..2b5802c 100644 --- a/services/soko-ml/data-ingestion-service/src/clients/order_client.py +++ b/services/soko-ml/data-ingestion-service/src/clients/order_client.py @@ -10,7 +10,7 @@ log = logging.getLogger(__name__) -ORDER_SERVICE_URL = os.getenv("ORDER_SERVICE_URL", "http://order-service:3002") +ORDER_SERVICE_URL = os.getenv("ORDER_SERVICE_URL", "http://order-service:8004") INTERNAL_API_KEY = os.getenv("INTERNAL_API_KEY", "") PAGE_LIMIT = 100 diff --git a/services/soko-ml/data-ingestion-service/src/clients/user_client.py b/services/soko-ml/data-ingestion-service/src/clients/user_client.py index 9b7842d..8ba0963 100644 --- a/services/soko-ml/data-ingestion-service/src/clients/user_client.py +++ b/services/soko-ml/data-ingestion-service/src/clients/user_client.py @@ -10,7 +10,7 @@ log = logging.getLogger(__name__) -USER_SERVICE_URL = os.getenv("USER_SERVICE_URL", "http://user-service:3003") +USER_SERVICE_URL = os.getenv("USER_SERVICE_URL", "http://user-service:8002") INTERNAL_API_KEY = os.getenv("INTERNAL_API_KEY", "") PAGE_LIMIT = 100 diff --git a/services/soko-ml/data-ingestion-service/src/main.py b/services/soko-ml/data-ingestion-service/src/main.py index 3a6ccf3..c60a40a 100644 --- a/services/soko-ml/data-ingestion-service/src/main.py +++ b/services/soko-ml/data-ingestion-service/src/main.py @@ -12,6 +12,7 @@ import os from contextlib import asynccontextmanager +import httpx import structlog from fastapi import FastAPI, BackgroundTasks, HTTPException from fastapi.responses import JSONResponse @@ -30,8 +31,26 @@ ) log = structlog.get_logger() -BOOTSTRAP_ON_STARTUP = os.getenv("BOOTSTRAP_ON_STARTUP", "true").lower() == "true" -SERVICE_NAME = os.getenv("SERVICE_NAME", "data-ingestion-service") +BOOTSTRAP_ON_STARTUP = os.getenv("BOOTSTRAP_ON_STARTUP", "true").lower() == "true" +SERVICE_NAME = os.getenv("SERVICE_NAME", "data-ingestion-service") +INTERNAL_API_KEY = os.getenv("INTERNAL_API_KEY", "") +REC_SERVICE_URL = os.getenv("REC_SERVICE_URL", "http://recommendation-service:8002") + + +async def _notify_recommendation_reload() -> None: + """Pings the recommendation service to reload profiles immediately after bootstrap.""" + try: + async with httpx.AsyncClient(timeout=10.0) as client: + resp = await client.post( + f"{REC_SERVICE_URL}/internal/reload", + headers={"x-internal-secret": INTERNAL_API_KEY}, + ) + if resp.status_code == 200: + log.info("recommendation_reload_triggered", result=resp.json()) + else: + log.warning("recommendation_reload_non_200", status=resp.status_code) + except Exception as exc: + log.warning(f"recommendation_reload_failed: {exc}") _stream: TransactionStream | None = None _bootstrap_lock = asyncio.Lock() @@ -67,6 +86,7 @@ async def lifespan(app: FastAPI): try: result = await _run_bootstrap() log.info("bootstrap_complete", **result) + await _notify_recommendation_reload() except Exception as exc: log.error(f"bootstrap_failed: {exc}") else: @@ -104,6 +124,7 @@ async def _do_bootstrap(): log.info("manual_bootstrap_triggered") result = await _run_bootstrap() log.info("manual_bootstrap_complete", **result) + await _notify_recommendation_reload() background_tasks.add_task(_do_bootstrap) return {"message": "Bootstrap triggered — running in background"} diff --git a/services/soko-ml/docker-compose.yml b/services/soko-ml/docker-compose.yml index ba4b36a..e6d8d73 100644 --- a/services/soko-ml/docker-compose.yml +++ b/services/soko-ml/docker-compose.yml @@ -195,21 +195,22 @@ services: ports: - "${REC_SERVICE_PORT:-8095}:8002" environment: + - POSTGRES_DSN=postgresql://${POSTGRES_USER:-soko_ml}:${POSTGRES_PASSWORD:-changeme}@soko-ml-db:5432/soko_ml_db - REDIS_HOST=${REDIS_HOST:-redis} - REDIS_PORT=${REDIS_PORT:-6379} - REDIS_DB=${REDIS_DB:-0} - REDIS_PASSWORD=${REDIS_PASSWORD:-} - KAFKA_BOOTSTRAP_SERVERS=${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} - KAFKA_INTERACTION_TOPIC=${KAFKA_INTERACTION_TOPIC:-soko.interactions} - - FARMERS_DATA_PATH=${FARMERS_DATA_PATH:-/app/data/raw/farmers.csv} - - BUYERS_DATA_PATH=${BUYERS_DATA_PATH:-/app/data/raw/buyers.csv} - REC_CACHE_TTL_SECONDS=${REC_CACHE_TTL_SECONDS:-3600} - DEFAULT_TOP_N=${DEFAULT_TOP_N:-5} + - PROFILE_REFRESH_INTERVAL_SECONDS=${PROFILE_REFRESH_INTERVAL_SECONDS:-900} + - INTERNAL_API_KEY=${INTERNAL_API_KEY:-internal-secret} - LOG_LEVEL=${LOG_LEVEL:-INFO} - SERVICE_NAME=recommendation-service - volumes: - - ./recommendation-service/data:/app/data:ro depends_on: + soko-ml-db: + condition: service_healthy redis: condition: service_healthy kafka: @@ -276,9 +277,10 @@ services: - KAFKA_TRANSACTION_TOPIC=${KAFKA_TRANSACTION_TOPIC:-soko.transactions} - KAFKA_ML_EVENTS_TOPIC=${KAFKA_ML_EVENTS_TOPIC:-soko.ml.events} - KAFKA_GAPS_TOPIC=${KAFKA_GAPS_TOPIC:-soko.gaps} - - USER_SERVICE_URL=${USER_SERVICE_URL:-http://user-service:8002} - - ORDER_SERVICE_URL=${ORDER_SERVICE_URL:-http://order-service:8003} - - PRODUCE_SERVICE_URL=${PRODUCE_SERVICE_URL:-http://produce-service:8004} + - USER_SERVICE_URL=${USER_SERVICE_URL:-http://user_service:8002} + - ORDER_SERVICE_URL=${ORDER_SERVICE_URL:-http://order_service:8004} + - PRODUCE_SERVICE_URL=${PRODUCE_SERVICE_URL:-http://produce_service:8003} + - REC_SERVICE_URL=http://recommendation-service:8002 - INTERNAL_API_KEY=${INTERNAL_API_KEY:-internal-secret} - BOOTSTRAP_ON_STARTUP=${BOOTSTRAP_ON_STARTUP:-true} - MIN_OBSERVATIONS_FOR_MODEL=${MIN_OBSERVATIONS_FOR_MODEL:-30} diff --git a/services/soko-ml/ml-gateway-service/src/main.py b/services/soko-ml/ml-gateway-service/src/main.py index 4d2b8cf..1fbacc6 100644 --- a/services/soko-ml/ml-gateway-service/src/main.py +++ b/services/soko-ml/ml-gateway-service/src/main.py @@ -92,18 +92,38 @@ async def price_crops(): @app.get("/recommend/farmers-for-buyer/{buyer_id}") -async def recommend_farmers(buyer_id: str, top_n: int = Query(default=5, ge=1, le=50)): +async def recommend_farmers( + buyer_id: str, + request: Request, + top_n: int = Query(default=5, ge=1, le=50), +): client: httpx.AsyncClient = app.state.http_client url = f"{REC_SERVICE_URL}/recommend/farmers-for-buyer/{buyer_id}" - result, status = await proxy_request(client, "GET", url, params={"top_n": top_n}) + forwarded = { + k: v for k, v in [ + ("x-user-id", request.headers.get("x-user-id")), + ("x-user-role", request.headers.get("x-user-role")), + ] if v + } + result, status = await proxy_request(client, "GET", url, params={"top_n": top_n}, headers=forwarded or None) return JSONResponse(content=result, status_code=status) @app.get("/recommend/buyers-for-farmer/{farmer_id}") -async def recommend_buyers(farmer_id: str, top_n: int = Query(default=5, ge=1, le=50)): +async def recommend_buyers( + farmer_id: str, + request: Request, + top_n: int = Query(default=5, ge=1, le=50), +): client: httpx.AsyncClient = app.state.http_client url = f"{REC_SERVICE_URL}/recommend/buyers-for-farmer/{farmer_id}" - result, status = await proxy_request(client, "GET", url, params={"top_n": top_n}) + forwarded = { + k: v for k, v in [ + ("x-user-id", request.headers.get("x-user-id")), + ("x-user-role", request.headers.get("x-user-role")), + ] if v + } + result, status = await proxy_request(client, "GET", url, params={"top_n": top_n}, headers=forwarded or None) return JSONResponse(content=result, status_code=status) diff --git a/services/soko-ml/ml-gateway-service/src/proxy.py b/services/soko-ml/ml-gateway-service/src/proxy.py index e6b213b..11bc28f 100644 --- a/services/soko-ml/ml-gateway-service/src/proxy.py +++ b/services/soko-ml/ml-gateway-service/src/proxy.py @@ -99,6 +99,7 @@ async def proxy_request( url: str, json_body: Optional[dict] = None, params: Optional[dict] = None, + headers: Optional[dict] = None, ) -> tuple[dict, int]: """ Proxy a request with retries and circuit breaker protection. @@ -115,9 +116,9 @@ async def proxy_request( for attempt in range(1, MAX_RETRIES + 1): try: if method.upper() == "POST": - resp = await client.post(url, json=json_body, timeout=REQUEST_TIMEOUT_SECONDS) + resp = await client.post(url, json=json_body, headers=headers, timeout=REQUEST_TIMEOUT_SECONDS) else: - resp = await client.get(url, params=params, timeout=REQUEST_TIMEOUT_SECONDS) + resp = await client.get(url, params=params, headers=headers, timeout=REQUEST_TIMEOUT_SECONDS) if resp.status_code < 500: breaker.record_success() diff --git a/services/soko-ml/recommendation-service/src/main.py b/services/soko-ml/recommendation-service/src/main.py index 3ca79da..a78779a 100644 --- a/services/soko-ml/recommendation-service/src/main.py +++ b/services/soko-ml/recommendation-service/src/main.py @@ -3,7 +3,7 @@ from contextlib import asynccontextmanager import structlog -from fastapi import FastAPI, HTTPException, Query +from fastapi import FastAPI, Header, HTTPException, Query from .cache import ( get_redis_client, @@ -31,6 +31,7 @@ DEFAULT_TOP_N = int(os.getenv("DEFAULT_TOP_N", "5")) SERVICE_NAME = os.getenv("SERVICE_NAME", "recommendation-service") PROFILE_REFRESH_INTERVAL = int(os.getenv("PROFILE_REFRESH_INTERVAL_SECONDS", "900")) +INTERNAL_API_KEY = os.getenv("INTERNAL_API_KEY", "") @asynccontextmanager @@ -92,10 +93,32 @@ async def health(): ) +def _check_identity(path_id: str, x_user_id: str | None, x_user_role: str | None) -> None: + """ + Enforce that the requesting user can only access their own recommendations. + Admins and internal callers (no header) bypass this check. + Raises 403 if the path ID doesn't match the JWT-derived user ID. + """ + if x_user_id is None: + return + if x_user_role in ("admin",): + return + if x_user_id != path_id: + raise HTTPException( + status_code=403, + detail="You can only request recommendations for your own account.", + ) + + @app.get("/recommend/farmers-for-buyer/{buyer_id}", response_model=FarmersForBuyerResponse) async def farmers_for_buyer( - buyer_id: str, top_n: int = Query(default=DEFAULT_TOP_N, ge=1, le=50) + buyer_id: str, + top_n: int = Query(default=DEFAULT_TOP_N, ge=1, le=50), + x_user_id: str | None = Header(default=None), + x_user_role: str | None = Header(default=None), ): + _check_identity(buyer_id, x_user_id, x_user_role) + redis = app.state.redis recommender: Recommender = app.state.recommender @@ -123,8 +146,13 @@ async def farmers_for_buyer( @app.get("/recommend/buyers-for-farmer/{farmer_id}", response_model=BuyersForFarmerResponse) async def buyers_for_farmer( - farmer_id: str, top_n: int = Query(default=DEFAULT_TOP_N, ge=1, le=50) + farmer_id: str, + top_n: int = Query(default=DEFAULT_TOP_N, ge=1, le=50), + x_user_id: str | None = Header(default=None), + x_user_role: str | None = Header(default=None), ): + _check_identity(farmer_id, x_user_id, x_user_role) + redis = app.state.redis recommender: Recommender = app.state.recommender @@ -148,3 +176,20 @@ async def buyers_for_farmer( cached=False, recommended_buyers=[BuyerRecommendation(**b) for b in recommendations], ) + + +@app.post("/internal/reload") +async def internal_reload(x_internal_secret: str | None = Header(default=None)): + """ + Triggers an immediate profile reload from the feature store. + Called by data-ingestion-service after bootstrap completes so new users + appear in recommendations without waiting for the 15-minute timer. + Protected by the shared internal secret. + """ + if INTERNAL_API_KEY and x_internal_secret != INTERNAL_API_KEY: + raise HTTPException(status_code=403, detail="Invalid internal secret") + + profile_store: ProfileStore = app.state.profile_store + n_farmers, n_buyers = await profile_store.reload() + log.info("internal_reload_triggered", farmers=n_farmers, buyers=n_buyers) + return {"status": "reloaded", "farmers": n_farmers, "buyers": n_buyers} From 073cc2e2c72534435a1b43f34472c70bb7659ee7 Mon Sep 17 00:00:00 2001 From: the-icemann Date: Fri, 15 May 2026 14:50:59 +0300 Subject: [PATCH 03/24] Modded the NGINX file to test core functionality dynamically --- DEPLOYMENT_CHECKLIST.md | 435 ++++++++++++++++++++++++++++++++++++++++ Makefile | 88 +++++++- README.md | 103 +++++++++- nginx/nginx.conf | 206 ++++++++++++++----- scripts/destroy_seed.py | 208 +++++++++++++++++++ scripts/fill_envs.py | 184 +++++++++++++++++ scripts/seed.py | 396 ++++++++++++++++++++++++++++++++++-- 7 files changed, 1543 insertions(+), 77 deletions(-) create mode 100644 DEPLOYMENT_CHECKLIST.md create mode 100644 scripts/destroy_seed.py create mode 100644 scripts/fill_envs.py diff --git a/DEPLOYMENT_CHECKLIST.md b/DEPLOYMENT_CHECKLIST.md new file mode 100644 index 0000000..b1e7151 --- /dev/null +++ b/DEPLOYMENT_CHECKLIST.md @@ -0,0 +1,435 @@ +# Soko — Pre-Deployment Checklist + +Use this checklist before every production deployment. Work through each section top-to-bottom. All items must be checked before the Go/No-Go decision at the end. + +--- + +## 1. Security Fixes Verification + +Confirm that all security-critical fixes from the ML integration overhaul are present and tested. + +- [ ] **JWT auth enforced at gateway** — every protected route returns `401` when called without a valid token + ```bash + curl -o /dev/null -sw "%{http_code}" http://localhost/users/me + # expected: 401 + ``` +- [ ] **Identity header injection validated** — `X-User-Id` and `X-User-Role` are injected by Nginx, never accepted from external clients + ```bash + curl -H "X-User-Id: 999" http://localhost/users/me + # must return 401, not the spoofed user's data + ``` +- [ ] **Recommendation endpoint enforces own-account access** — a user cannot fetch another user's recommendations + ```bash + # Login as user A, try to GET /ml/recommend/{user_B_id} — must return 403 + ``` +- [ ] **Header injection hardened** — confirm `proxy_set_header X-User-Id ""` in nginx.conf clears any client-supplied value before the subrequest +- [ ] **Internal token verify endpoint not reachable externally** — `/verify-token` returns 404 from outside the container network + ```bash + curl -o /dev/null -sw "%{http_code}" http://localhost/verify-token + # expected: 404 + ``` +- [ ] **INTERNAL_SECRET set and consistent** — same value across all core service `.env` files + +**Rollback:** Revert nginx.conf and service configs to last known-good state; redeploy core stack only. + +--- + +## 2. Bug Fixes Verification + +Confirm all production bug fixes from the ML integration phase are present. + +- [ ] **Service URLs use Docker DNS names** — no `localhost` in inter-service HTTP calls (e.g. `USER_SERVICE_URL=http://user_service:8002`) +- [ ] **Environment variables loaded at runtime** — no hardcoded secrets in source files; all sensitive values come from `.env` +- [ ] **Docker Compose `depends_on` health checks** — ML services wait for `soko-ml-db` and Kafka before starting +- [ ] **ml-gateway circuit breakers configured** — timeout and retry settings present in `ml-gateway-service` config +- [ ] **Kafka consumer group IDs unique per service** — price-prediction, recommendation, and kafka-agent use distinct `group_id` values + +**Rollback:** Roll back the affected service image tag; `docker compose up -d `. + +--- + +## 3. Environment Configuration + +- [ ] All core service `.env` files exist (run `make setup` if missing) + ```bash + ls services/auth/.env services/user/.env services/produce/.env \ + services/order/.env services/payment/.env services/message/.env \ + services/notification/.env services/blog/.env services/ussd/.env + ``` +- [ ] `SECRET_KEY` set in `services/auth/.env` (strong, production-grade value) +- [ ] `FRONTEND_URL` set in `services/auth/.env` (exact origin used by the web client) +- [ ] `INTERNAL_SECRET` set to the same value in every core service `.env` +- [ ] `services/soko-ml/.env` exists and is fully populated +- [ ] `PESAPAL_CONSUMER_KEY` and `PESAPAL_CONSUMER_SECRET` set (payment service) +- [ ] `CLOUDINARY_*` keys set (produce service image uploads) +- [ ] `AT_USERNAME` and `AT_API_KEY` set in notification and ussd `.env` files +- [ ] No `.env` file contains placeholder values like `changeme` or `your_secret_here` + +**Rollback:** Not applicable — env changes do not affect running containers until restart. + +--- + +## 4. Core Stack Services + +Verify all 9 core services build and start cleanly. + +- [ ] `make core-up` completes without errors +- [ ] All 9 service containers show `healthy` in `docker compose ps` + ```bash + docker compose ps + ``` +- [ ] Nginx gateway responds on `:80` + ```bash + curl -sf http://localhost/health + ``` +- [ ] Auth service responds + ```bash + curl -sf http://localhost/auth/docs | grep -q "FastAPI" + ``` +- [ ] User, produce, order, payment, message, notification, blog services all return `200` on their `/docs` routes +- [ ] USSD service container is running (no HTTP docs endpoint, check via `docker compose ps`) + +**Rollback:** `make core-down && make core-up` — stateless services restart cleanly. If a database migration is involved, see Section 6. + +--- + +## 5. ML Stack Services + +Verify all 5 ML services build and start cleanly. + +- [ ] `make ml-up` completes without errors +- [ ] ml-gateway-service responds on host port `:8080` + ```bash + curl -sf http://localhost:8080/health | python3 -m json.tool + ``` +- [ ] price-prediction-service responds (via gateway) + ```bash + curl -sf http://localhost:8080/price/health | python3 -m json.tool + ``` +- [ ] recommendation-service responds (via gateway) + ```bash + curl -sf http://localhost:8080/recommend/health | python3 -m json.tool + ``` +- [ ] location-service responds on `:8003` + ```bash + curl -sf http://localhost:8003/health | python3 -m json.tool + ``` +- [ ] data-ingestion-service responds (via gateway) + ```bash + curl -sf http://localhost:8080/ingest/status | python3 -m json.tool + ``` +- [ ] kafka-agent container is running (no HTTP port — check via `docker compose -f services/soko-ml/docker-compose.yml ps`) + +**Rollback:** `make ml-down && make ml-up`. ML services are stateless (state lives in Postgres/Redis/Kafka). Safe to restart independently. + +--- + +## 6. Database Setup & Migrations + +- [ ] All 9 core PostgreSQL databases initialised (auto-created by service startup) +- [ ] ML feature store schema applied + ```bash + make db-up + # or verify: + make db-shell + # in psql: \dt + ``` +- [ ] ML Postgres tables present: `price_observations`, `user_profiles`, `interactions`, `coverage_gaps` +- [ ] No pending Alembic migrations on any core service + ```bash + docker compose exec auth_service alembic current + docker compose exec user_service alembic current + # repeat for each service + ``` +- [ ] Database credentials in `.env` match the `docker-compose.yml` `POSTGRES_*` values + +**Rollback:** `make db-reset` drops and re-applies the ML schema (destructive). Core DB rollback requires per-service Alembic downgrade — do not run without explicit incident runbook. + +--- + +## 7. Kafka Topics & Configuration + +- [ ] All 7 required Kafka topics created + ```bash + make kafka-topics + make kafka-ui + # verify: soko.transactions, soko.interactions, soko.price.requests, + # soko.price.results, soko.ml.events, soko.dlq, soko.gaps + ``` +- [ ] `soko.transactions` has 6 partitions +- [ ] `soko.interactions` has 6 partitions +- [ ] `soko.price.requests` and `soko.price.results` have 3 partitions each +- [ ] `soko.ml.events`, `soko.dlq`, `soko.gaps` have 2 partitions each +- [ ] Kafka broker healthy (no `LEADER_NOT_AVAILABLE` errors in logs) + ```bash + docker compose -f services/soko-ml/docker-compose.yml logs kafka | tail -30 + ``` +- [ ] Zookeeper healthy and Kafka connected to it + +**Rollback:** Topics are append-only — no rollback needed unless partitioning is changed, which requires recreation of the topic and replay of events. + +--- + +## 8. Redis Configuration + +- [ ] Core Redis instance running (`redis` container in `soko_net`) + ```bash + docker compose exec redis redis-cli ping + # expected: PONG + ``` +- [ ] ML Redis instance running (`redis` container in `soko-ml-network`) + ```bash + make redis-cli + # in redis-cli: PING → PONG + ``` +- [ ] Core Redis reachable from auth and user services (`redis://redis:6379`) +- [ ] ML Redis reachable from price-prediction and recommendation services +- [ ] No Redis memory warnings in logs + +**Rollback:** Redis is a cache — data loss on restart is acceptable. Both instances can be restarted without data loss risk to core business logic. + +--- + +## 9. Networking & Docker Networks + +- [ ] `soko_net` network exists + ```bash + docker network inspect soko_net | grep Name + ``` +- [ ] `soko-ml-network` network exists + ```bash + docker network inspect soko-ml-network | grep Name + ``` +- [ ] `soko-ml-bridge` network exists (created by `make setup`) + ```bash + docker network inspect soko-ml-bridge | grep Name + # or: make setup (idempotent) + ``` +- [ ] Nginx container is attached to both `soko_net` and `soko-ml-bridge` +- [ ] ml-gateway-service container is attached to both `soko-ml-network` and `soko-ml-bridge` +- [ ] Core services (`auth`, `user`, etc.) are attached to `soko_net` only — not to `soko-ml-bridge` +- [ ] Cross-stack request succeeds (Nginx → ml-gateway) + ```bash + curl -sf http://localhost/ml/price/health + ``` + +**Rollback:** `docker network create soko-ml-bridge` recreates the bridge. Restart both stacks afterward. + +--- + +## 10. Feature Store Initialization + +- [ ] ML Postgres `user_profiles` table populated (run bootstrap if empty) + ```bash + make ingest-bootstrap + make ingest-status + ``` +- [ ] At least one user profile row present + ```bash + make db-shell + # SELECT COUNT(*) FROM user_profiles; + ``` +- [ ] `price_observations` table populated with initial historical data (if running ML price models) +- [ ] `coverage_gaps` table not showing excessive gap counts + ```bash + make gaps-summary + ``` +- [ ] Recommendation service returns non-empty results for a seeded user + ```bash + # login as a seeded farmer/buyer, then: + curl -H "Authorization: Bearer " http://localhost/ml/recommend/ + ``` + +**Rollback:** Re-run `make ingest-bootstrap`. If profiles are corrupt, `make db-reset` and re-bootstrap. This is non-destructive to the core stack. + +--- + +## 11. API Gateway & Authentication + +- [ ] Nginx starts and serves on `:80` +- [ ] `/_verify_token` subrequest reaches auth service and returns user identity headers +- [ ] JWT issued by `/auth/login` is accepted by a protected route (e.g. `/users/me`) + ```bash + TOKEN=$(curl -sf -X POST http://localhost/auth/login \ + -H "Content-Type: application/json" \ + -d '{"email":"test@example.com","password":"password"}' | python3 -c "import sys,json; print(json.load(sys.stdin)['access_token'])") + curl -sf -H "Authorization: Bearer $TOKEN" http://localhost/users/me + ``` +- [ ] Expired token returns `401` +- [ ] OAuth routes (`/oauth/`) are publicly accessible +- [ ] Nginx rate limiting active (30 req/min per IP) +- [ ] CORS headers present on responses (if web client uses a different origin) + +**Rollback:** Reload nginx config only: `docker compose exec nginx nginx -s reload`. Full core restart is rarely needed for gateway issues. + +--- + +## 12. ML Model Deployment + +- [ ] Prophet price-prediction models (`.pkl` files) present in `services/soko-ml/price-prediction-service/models/` + ```bash + ls services/soko-ml/price-prediction-service/models/*.pkl | wc -l + # should be > 0 (one per crop/market combination trained) + ``` +- [ ] Models were trained on recent data (check file modification timestamps) + ```bash + ls -lt services/soko-ml/price-prediction-service/models/*.pkl | head -5 + ``` +- [ ] Price prediction returns a valid forecast for a known crop + ```bash + curl -sf -X POST http://localhost/ml/price/predict \ + -H "Content-Type: application/json" \ + -d '{"crop":"maize_grain","market":"Kampala","forecast_days":7}' + ``` +- [ ] Recommendation service returns scores (not empty list) for a seeded user +- [ ] Circuit breakers on ml-gateway are not tripped (no `503` from gateway health endpoint) + +**Rollback:** If model files are missing or corrupt, re-run `make train`. The price service falls back to static averages if no model file is found — confirm fallback behaviour is acceptable for the deployment window. + +--- + +## 13. Testing & Smoke Tests + +- [ ] All ML unit tests pass + ```bash + make test + ``` +- [ ] Smoke: full prediction pipeline + ```bash + make smoke-test + ``` +- [ ] Smoke: market routing (Tier 1 crop) + ```bash + make smoke-route + ``` +- [ ] Smoke: buyer discover endpoint + ```bash + make smoke-discover + ``` +- [ ] Smoke: Tier 2 fallback (limited-coverage crop) + ```bash + make smoke-fallback + ``` +- [ ] Smoke: Tier 3 unknown crop graceful degradation + ```bash + make smoke-tier3 + ``` +- [ ] Smoke: data-ingestion order event + ```bash + make smoke-ingest + ``` +- [ ] Full health check on all services + ```bash + make health + ``` + +**Rollback:** Failing smoke tests after deployment indicate a regression. Immediately run `make stop` and restore the previous image tags before investigating. + +--- + +## 14. Monitoring & Logging + +- [ ] Centralized log aggregation configured (or Docker log driver set to a persistent driver) +- [ ] All containers logging to stdout (not writing to files inside the container) +- [ ] No `ERROR` or `CRITICAL` log lines appearing at steady state + ```bash + make ml-logs | grep -i "error\|critical" | head -20 + docker compose logs | grep -i "error\|critical" | head -20 + ``` +- [ ] Kafka consumer lag monitored (no runaway lag on `soko.transactions`) +- [ ] Redis memory usage within acceptable bounds + ```bash + make redis-cli + # INFO memory → used_memory_human + ``` +- [ ] ML gateway request latency acceptable (check `/health` response time) + ```bash + time curl -sf http://localhost:8080/health > /dev/null + ``` + +**Rollback:** Logging configuration is outside the application — no rollback needed. Fix log driver settings and restart the affected containers. + +--- + +## 15. Backup & Disaster Recovery + +- [ ] Core PostgreSQL databases backed up before deployment + ```bash + for svc in auth user produce order payment message notification blog ussd; do + docker compose exec ${svc}_db pg_dump -U ${svc}_user ${svc}_db > backups/${svc}_db_$(date +%Y%m%d).sql + done + ``` +- [ ] ML feature store backed up + ```bash + docker compose -f services/soko-ml/docker-compose.yml exec soko-ml-db \ + pg_dump -U soko_ml soko_ml_db > backups/soko_ml_db_$(date +%Y%m%d).sql + ``` +- [ ] Backup files stored outside the Docker host (S3, remote NFS, or similar) +- [ ] Restore procedure documented and tested in staging +- [ ] Kafka topic retention policy set (default: 7 days) — confirmed acceptable for replay window + +**Rollback:** Restore from the pre-deployment backup. Core and ML databases can be restored independently. + +--- + +## 16. Production Hardening + +- [ ] No default passwords remain in any `.env` file +- [ ] `SECRET_KEY` is a cryptographically random string (min 32 bytes) +- [ ] SSL/TLS termination configured (Nginx with a valid certificate, or an upstream load balancer) +- [ ] Nginx rate limiting confirmed active (`limit_req_zone` in nginx.conf) +- [ ] `INTERNAL_SECRET` used for all cross-service calls and not exposed in logs +- [ ] Docker socket not mounted into any service container +- [ ] Image tags pinned to specific versions (not `latest`) in production Compose files +- [ ] Container resource limits (`mem_limit`, `cpus`) set in Compose files +- [ ] No development ports (`8094`, `8095`, `8096`) exposed in the production Compose file +- [ ] PesaPal webhook URL configured to the production hostname (not `localhost`) + +**Rollback:** Hardening changes are config-only. Revert the relevant `.env` or `docker-compose.yml` line and redeploy the affected service. + +--- + +## 17. Go/No-Go Decision Framework + +Complete this section as the final gate before traffic is directed to the deployment. + +### Pre-conditions (all must be true) + +- [ ] Sections 1–16 above are fully checked with no open items +- [ ] At least one team member has reviewed this checklist independently +- [ ] A rollback plan is documented and the rollback person is identified and available +- [ ] A maintenance window has been communicated to users (if applicable) +- [ ] The on-call engineer is reachable for the 2-hour post-deployment watch period + +### Decision + +| Decision | Condition | +|---|---| +| **GO** | All pre-conditions met; all smoke tests pass | +| **NO-GO** | Any section has an open item; any smoke test fails; on-call unavailable | + +**Approver:** ___________________________ **Date/Time:** ___________________________ + +--- + +## 18. Post-Deployment Monitoring + +Run these checks in the 2 hours following deployment. + +- [ ] **T+5 min** — all container health checks green (`docker compose ps`) +- [ ] **T+5 min** — ML gateway `/health` returns `200` with all downstream services `healthy` + ```bash + curl -sf http://localhost:8080/health | python3 -m json.tool + ``` +- [ ] **T+10 min** — first real user JWT successfully validated (check auth service logs) +- [ ] **T+15 min** — Kafka `soko.transactions` consumer lag not growing (run `make kafka-ui` and compare partition offsets) +- [ ] **T+30 min** — price prediction responding within SLA (< 2 s p95) +- [ ] **T+30 min** — recommendation service responding within SLA (< 1 s p95) +- [ ] **T+60 min** — no spike in `ERROR` log lines relative to pre-deployment baseline +- [ ] **T+120 min** — declare deployment stable; remove rollback readiness posture + +**If any check fails:** Immediately execute the rollback procedure documented in the relevant section above, then open a post-incident review. + +--- + +*Checklist maintained alongside `README.md` — keep both in sync when adding new services or ports.* diff --git a/Makefile b/Makefile index 9646f5d..7a489e9 100644 --- a/Makefile +++ b/Makefile @@ -45,6 +45,8 @@ CORE_SERVICES := auth user produce order payment message notification blog ussd test test-price test-rec test-gateway test-location test-ingest \ health smoke-test smoke-route smoke-discover smoke-fallback smoke-tier3 smoke-ingest \ clean clean-models clean-docker \ + port-reference \ + fill-envs seed destroy-seed \ help # ============================================================================= @@ -163,6 +165,19 @@ help: @echo " make clean-models Remove trained model .pkl files" @echo " make clean-docker Remove all containers, volumes, and images (both stacks)" @echo "" + @echo "► REFERENCE" + @echo " ─────────────────────────────────────────────────────────" + @echo " make port-reference Show all container/host port mappings for both stacks" + @echo "" + @echo "► SEED & DESTROY" + @echo " ─────────────────────────────────────────────────────────" + @echo " make fill-envs Write consistent dev credentials to all service .env files" + @echo " make seed fill-envs + populate all services with Ugandan dummy data" + @echo " ↳ Phases: register users → profiles → listings → orders → messages" + @echo " ↳ blog posts → reviews → ML bootstrap → rec-service reload" + @echo " make destroy-seed Remove all seeded data from every service database" + @echo " ↳ Reads scripts/.seed_manifest.json written by 'make seed'" + @echo "" # ============================================================================= # FIRST-TIME SETUP @@ -507,10 +522,6 @@ health: @echo "=== Data Ingestion Service ===" && \ curl -sf http://localhost:8096/health | python3 -m json.tool || echo "UNREACHABLE" -seed: - @echo "Seeding Ugandan dummy data into all service databases..." - @python3 scripts/seed.py - smoke-test: @python3 scripts/smoke_test.py @@ -568,3 +579,72 @@ clean-docker: $(COMPOSE_ML) down -v --rmi all $(COMPOSE_CORE) down --rmi all @echo "All containers, volumes, and images removed." + +# ============================================================================= +# REFERENCE +# ============================================================================= + +port-reference: + @echo "" + @echo "╔══════════════════════════════════════════════════════════════════════╗" + @echo "║ Soko — Port Reference (container → host) ║" + @echo "╚══════════════════════════════════════════════════════════════════════╝" + @echo "" + @echo "► CORE STACK (network: soko_net — internal container ports only)" + @echo " ─────────────────────────────────────────────────────────────────" + @echo " nginx (API gateway) container :80 → host :80" + @echo " auth-service container :8001 → host: NOT EXPOSED" + @echo " user-service container :8002 → host: NOT EXPOSED" + @echo " produce-service container :8003 → host: NOT EXPOSED" + @echo " order-service container :8004 → host: NOT EXPOSED" + @echo " payment-service container :8005 → host: NOT EXPOSED" + @echo " message-service container :8006 → host: NOT EXPOSED" + @echo " notification-service container :8007 → host: NOT EXPOSED" + @echo " blog-service container :8008 → host: NOT EXPOSED" + @echo " ussd-service container :8009 → host: NOT EXPOSED" + @echo " core Redis container :6379 → host: NOT EXPOSED" + @echo " core PostgreSQL ×9 container :5432 → host: NOT EXPOSED" + @echo "" + @echo "► ML STACK (network: soko-ml-network; bridge: soko-ml-bridge)" + @echo " ─────────────────────────────────────────────────────────────" + @echo " ml-gateway-service container :8000 → host :8080 (production)" + @echo " price-prediction-service container :8001 → host :8094 (dev only)" + @echo " recommendation-service container :8002 → host :8095 (dev only)" + @echo " location-service container :8003 → host :8003" + @echo " data-ingestion-service container :8004 → host :8096 (dev only)" + @echo "" + @echo "► INFRASTRUCTURE (ML stack — internal only)" + @echo " ─────────────────────────────────────────────────────────────" + @echo " Kafka container :9092 → host: NOT EXPOSED" + @echo " Zookeeper container :2181 → host: NOT EXPOSED" + @echo " ML Redis container :6379 → host: NOT EXPOSED" + @echo " soko-ml-db (PostgreSQL) container :5432 → host: NOT EXPOSED" + @echo "" + @echo " NOTE: 'NOT EXPOSED' = reachable only within the Docker network." + @echo " Dev-only ports are mapped by docker-compose.dev.yml / make dev-*." + @echo "" + +# ============================================================================= +# SEED & DESTROY +# ============================================================================= + +fill-envs: + @python3 scripts/fill_envs.py + +seed: fill-envs + @echo "" + @echo "╔══════════════════════════════════════════════════════════════════════╗" + @echo "║ Soko — Seeding all services ║" + @echo "╚══════════════════════════════════════════════════════════════════════╝" + @echo "" + @echo "Both stacks must be running: make start" + @echo "" + @python3 scripts/seed.py + +destroy-seed: + @echo "" + @echo "╔══════════════════════════════════════════════════════════════════════╗" + @echo "║ Soko — Destroying seed data ║" + @echo "╚══════════════════════════════════════════════════════════════════════╝" + @echo "" + @python3 scripts/destroy_seed.py diff --git a/README.md b/README.md index b6d8eff..3eb6d9e 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ The platform runs as two independent but integrated Docker Compose stacks: 12. [Project Structure](#project-structure) 13. [Production Bug Report](#production-bug-report) 14. [Known Limitations](#known-limitations) +15. [Port Reference & Network Isolation](#port-reference--network-isolation) --- @@ -56,9 +57,10 @@ The platform runs as two independent but integrated Docker Compose stacks: └──┬────┬────┬────┬────┬────┬────┬────┬──────────────┬────────────────┘ │ │ │ │ │ │ │ │ │ ▼ ▼ ▼ ▼ ▼ ▼ ▼ ▼ ▼ - :8001:8002:8003:8004:8005:8006:8007:8008 ML stack - Auth User Prod Ord Pay Msg Not Blog USSD (see below) - :8009 + ── CORE STACK (internal container ports) ────────────────────────── + :8001 :8002 :8003 :8004 :8005 :8006 :8007 :8008 ML stack + Auth User Prod Ord Pay Msg Not Blog USSD (see below) + :8009 Each service owns its own PostgreSQL database. Core services share one Redis instance for caching. @@ -67,7 +69,7 @@ The platform runs as two independent but integrated Docker Compose stacks: ┌──────────────────────────────────────────────────────────────────────┐ │ ML STACK (services/soko-ml/) │ │ │ - │ nginx ──► ml-gateway-service (host :8080 / internal :8000) │ + │ nginx ──► ml-gateway-service (container port :8000 → host port :8080) │ │ │ circuit breakers · request logging · fallbacks │ │ ├──► price-prediction-service (:8001) │ │ │ Prophet .pkl models · Redis 24h cache │ @@ -99,6 +101,99 @@ The platform runs as two independent but integrated Docker Compose stacks: --- +## Port Reference & Network Isolation + +### Docker Network Topology + +Soko runs across three distinct Docker networks to enforce hard isolation boundaries: + +| Network | Belongs To | Purpose | +|---|---|---| +| `soko_net` | Core stack | Internal mesh for all core services + Nginx | +| `soko-ml-network` | ML stack | Internal mesh for all ML services | +| `soko-ml-bridge` | Both stacks | Shared bridge linking Nginx ↔ ml-gateway-service | + +``` + ┌─────────────────────────────────────────────────────────────────────┐ + │ soko_net (core stack) │ + │ nginx · auth · user · produce · order · payment │ + │ message · notification · blog · ussd · redis · postgres×9 │ + └────────────────────────┬────────────────────────────────────────────┘ + │ soko-ml-bridge + ┌────────────────────────▼────────────────────────────────────────────┐ + │ soko-ml-network (ML stack) │ + │ ml-gateway · price-prediction · recommendation · location │ + │ data-ingestion · kafka-agent · kafka · zookeeper · redis-ml │ + │ soko-ml-db (PostgreSQL feature store) │ + └─────────────────────────────────────────────────────────────────────┘ +``` + +Core services on `soko_net` **cannot** directly address ML services on `soko-ml-network`. The only cross-stack paths are: + +1. `Nginx → ml-gateway-service` over `soko-ml-bridge` (HTTP) +2. `order-service → Kafka → kafka-agent` over `soko-ml-bridge` (events) + +### Complete Port Mapping Table + +| Service | Container Port | Host Port | Network | Purpose | +|---|---|---|---|---| +| **CORE STACK** | | | | | +| nginx (API gateway) | 80 | 80 | `soko_net` + `soko-ml-bridge` | All public traffic entry point | +| auth-service | 8001 | — | `soko_net` | JWT issue & validation | +| user-service | 8002 | — | `soko_net` | User profiles | +| produce-service | 8003 | — | `soko_net` | Listings | +| order-service | 8004 | — | `soko_net` | Order lifecycle + Kafka pub | +| payment-service | 8005 | — | `soko_net` | PesaPal integration | +| message-service | 8006 | — | `soko_net` | WebSocket messaging | +| notification-service | 8007 | — | `soko_net` | WebSocket push | +| blog-service | 8008 | — | `soko_net` | Blog posts | +| ussd-service | 8009 | — | `soko_net` | Africa's Talking USSD | +| core PostgreSQL×9 | 5432 | — | `soko_net` | Per-service databases | +| core Redis | 6379 | — | `soko_net` | Shared caching | +| **ML STACK** | | | | | +| ml-gateway-service | 8000 | **8080** | `soko-ml-network` + `soko-ml-bridge` | ML traffic router, circuit breakers | +| price-prediction-service | 8001 | 8094 (dev only) | `soko-ml-network` | Prophet forecast models | +| recommendation-service | 8002 | 8095 (dev only) | `soko-ml-network` | Content scoring + Kafka boosts | +| location-service | 8003 | 8003 | `soko-ml-network` | Market routing, Haversine | +| data-ingestion-service | 8004 | 8096 (dev only) | `soko-ml-network` | Feature store bootstrap | +| **INFRASTRUCTURE (ML stack)** | | | | | +| Kafka | 9092 | — | `soko-ml-network` | Event broker (internal) | +| Zookeeper | 2181 | — | `soko-ml-network` | Kafka coordination | +| ML Redis | 6379 | — | `soko-ml-network` | ML service caching | +| soko-ml-db (PostgreSQL) | 5432 | — | `soko-ml-network` | ML feature store | + +> **Host port vs. container port:** A container port is the port the process listens on *inside* Docker. A host port is what is mapped to your machine. Only explicitly mapped ports are reachable from your host — all others are container-internal only. + +### Port Binding Rules + +1. **Production** — only Nginx (`:80`) and ml-gateway-service (container `:8000` → host `:8080`) are bound to the host. Every other container port is internal-only. +2. **Development** — `make dev-price`, `make dev-rec`, `make dev-ingest` bind additional host ports (`:8094`, `:8095`, `:8096`) for local hot-reload. These mappings do not exist in the production Compose file. +3. **No direct service access** — clients must never call `auth_service:8001` directly; all traffic routes through Nginx or ml-gateway. The port numbers in the architecture diagram are container-internal addresses, not public endpoints. + +### Service-to-Service Communication Examples + +```bash +# Nginx → auth-service (internal subrequest for JWT validation) +nginx → http://auth_service:8001/verify-token + +# Nginx → ml-gateway (cross-network via soko-ml-bridge) +nginx → http://ml-gateway-service:8000/price/predict + +# ml-gateway → price-prediction (ML-internal only) +ml-gateway-service → http://price-prediction-service:8001/predict + +# ml-gateway → recommendation (ML-internal only) +ml-gateway-service → http://recommendation-service:8002/recommend/{user_id} + +# data-ingestion → user-service (cross-network via soko-ml-bridge) +data-ingestion-service → http://user_service:8002/users/farmers + +# order-service → Kafka → kafka-agent (event-driven, cross-network) +order-service publishes to soko.transactions → kafka-agent consumes and boosts +``` + +--- + ## Core Services ### Auth Service — `:8001` diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 431fd2d..1c09c77 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -7,18 +7,9 @@ http { limit_req_zone $binary_remote_addr zone=api_limit:10m rate=30r/m; resolver 127.0.0.11 valid=30s; - - upstream auth_service { server auth_service:8001; } - upstream user_service { server user_service:8002; } - upstream produce_service { server produce_service:8003; } - upstream order_service { server order_service:8004; } - upstream payment_service { server payment_service:8005; } - upstream message_service { server message_service:8006; } - upstream notification_service { server notification_service:8007; } - upstream blog_service { server blog_service:8008; } - upstream ussd_service { server ussd_service:8009; } - # ml-gateway is resolved lazily via $ml_gw variable below — keeps nginx from - # refusing to start when the ML stack is not running. + # No static upstream blocks — all services use set $var + proxy_pass http://$var/ + # so Docker DNS is re-queried per connection via the resolver above. + # This prevents 502s when containers restart and get new IPs. server { listen 80; @@ -36,7 +27,8 @@ http { location = /_verify_token { internal; - proxy_pass http://auth_service/verify-token; + set $auth_svc "auth_service:8001"; + proxy_pass http://$auth_svc/verify-token; proxy_set_header Authorization $http_authorization; proxy_pass_request_body off; proxy_set_header Content-Length ""; @@ -45,7 +37,8 @@ http { location = /_verify_token_optional { internal; - proxy_pass http://auth_service/verify-token-optional; + set $auth_svc "auth_service:8001"; + proxy_pass http://$auth_svc/verify-token-optional; proxy_set_header Authorization $http_authorization; proxy_pass_request_body off; proxy_set_header Content-Length ""; @@ -61,13 +54,26 @@ http { return 403 '{"detail":"Forbidden"}'; } - location = /auth/docs { proxy_pass http://auth_service/docs; proxy_set_header Host $host; } - location = /auth/redoc { proxy_pass http://auth_service/redoc; proxy_set_header Host $host; } - location = /auth/openapi.json { proxy_pass http://auth_service/openapi.json; proxy_set_header Host $host; } + location = /auth/docs { + set $auth_svc "auth_service:8001"; + proxy_pass http://$auth_svc/docs; + proxy_set_header Host $host; + } + location = /auth/redoc { + set $auth_svc "auth_service:8001"; + proxy_pass http://$auth_svc/redoc; + proxy_set_header Host $host; + } + location = /auth/openapi.json { + set $auth_svc "auth_service:8001"; + proxy_pass http://$auth_svc/openapi.json; + proxy_set_header Host $host; + } location /auth/ { limit_req zone=api_limit burst=20 nodelay; - proxy_pass http://auth_service/; + set $auth_svc "auth_service:8001"; + proxy_pass http://$auth_svc/; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -75,15 +81,28 @@ http { location /oauth/ { limit_req zone=api_limit burst=20 nodelay; - proxy_pass http://auth_service/; + set $auth_svc "auth_service:8001"; + proxy_pass http://$auth_svc/; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; } - location = /users/docs { proxy_pass http://user_service/docs; proxy_set_header Host $host; } - location = /users/redoc { proxy_pass http://user_service/redoc; proxy_set_header Host $host; } - location = /users/openapi.json { proxy_pass http://user_service/openapi.json; proxy_set_header Host $host; } + location = /users/docs { + set $user_svc "user_service:8002"; + proxy_pass http://$user_svc/docs; + proxy_set_header Host $host; + } + location = /users/redoc { + set $user_svc "user_service:8002"; + proxy_pass http://$user_svc/redoc; + proxy_set_header Host $host; + } + location = /users/openapi.json { + set $user_svc "user_service:8002"; + proxy_pass http://$user_svc/openapi.json; + proxy_set_header Host $host; + } location /users/ { limit_req zone=api_limit burst=20 nodelay; @@ -93,7 +112,8 @@ http { auth_request_set $user_email $upstream_http_x_user_email; error_page 401 = @error401; error_page 403 = @error403; - proxy_pass http://user_service/; + set $user_svc "user_service:8002"; + proxy_pass http://$user_svc/; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -103,9 +123,21 @@ http { proxy_set_header Authorization $http_authorization; } - location = /listings/docs { proxy_pass http://produce_service/docs; proxy_set_header Host $host; } - location = /listings/redoc { proxy_pass http://produce_service/redoc; proxy_set_header Host $host; } - location = /listings/openapi.json { proxy_pass http://produce_service/openapi.json; proxy_set_header Host $host; } + location = /listings/docs { + set $produce_svc "produce_service:8003"; + proxy_pass http://$produce_svc/docs; + proxy_set_header Host $host; + } + location = /listings/redoc { + set $produce_svc "produce_service:8003"; + proxy_pass http://$produce_svc/redoc; + proxy_set_header Host $host; + } + location = /listings/openapi.json { + set $produce_svc "produce_service:8003"; + proxy_pass http://$produce_svc/openapi.json; + proxy_set_header Host $host; + } location /listings/ { limit_req zone=api_limit burst=20 nodelay; @@ -114,7 +146,8 @@ http { auth_request_set $user_id $upstream_http_x_user_id; auth_request_set $user_role $upstream_http_x_user_role; auth_request_set $user_email $upstream_http_x_user_email; - proxy_pass http://produce_service/; + set $produce_svc "produce_service:8003"; + proxy_pass http://$produce_svc/; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -124,9 +157,21 @@ http { proxy_set_header Authorization $http_authorization; } - location = /orders/docs { proxy_pass http://order_service/docs; proxy_set_header Host $host; } - location = /orders/redoc { proxy_pass http://order_service/redoc; proxy_set_header Host $host; } - location = /orders/openapi.json { proxy_pass http://order_service/openapi.json; proxy_set_header Host $host; } + location = /orders/docs { + set $order_svc "order_service:8004"; + proxy_pass http://$order_svc/docs; + proxy_set_header Host $host; + } + location = /orders/redoc { + set $order_svc "order_service:8004"; + proxy_pass http://$order_svc/redoc; + proxy_set_header Host $host; + } + location = /orders/openapi.json { + set $order_svc "order_service:8004"; + proxy_pass http://$order_svc/openapi.json; + proxy_set_header Host $host; + } location /orders/ { limit_req zone=api_limit burst=20 nodelay; @@ -136,7 +181,8 @@ http { auth_request_set $user_email $upstream_http_x_user_email; error_page 401 = @error401; error_page 403 = @error403; - proxy_pass http://order_service/; + set $order_svc "order_service:8004"; + proxy_pass http://$order_svc/; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -146,9 +192,21 @@ http { proxy_set_header Authorization $http_authorization; } - location = /payments/docs { proxy_pass http://payment_service/docs; proxy_set_header Host $host; } - location = /payments/redoc { proxy_pass http://payment_service/redoc; proxy_set_header Host $host; } - location = /payments/openapi.json { proxy_pass http://payment_service/openapi.json; proxy_set_header Host $host; } + location = /payments/docs { + set $payment_svc "payment_service:8005"; + proxy_pass http://$payment_svc/docs; + proxy_set_header Host $host; + } + location = /payments/redoc { + set $payment_svc "payment_service:8005"; + proxy_pass http://$payment_svc/redoc; + proxy_set_header Host $host; + } + location = /payments/openapi.json { + set $payment_svc "payment_service:8005"; + proxy_pass http://$payment_svc/openapi.json; + proxy_set_header Host $host; + } location /payments/ { limit_req zone=api_limit burst=20 nodelay; @@ -158,7 +216,8 @@ http { auth_request_set $user_email $upstream_http_x_user_email; error_page 401 = @error401; error_page 403 = @error403; - proxy_pass http://payment_service/; + set $payment_svc "payment_service:8005"; + proxy_pass http://$payment_svc/; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -169,18 +228,32 @@ http { } location /webhook/ { - proxy_pass http://payment_service/; + set $payment_svc "payment_service:8005"; + proxy_pass http://$payment_svc/; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; } - location = /message/docs { proxy_pass http://message_service/docs; proxy_set_header Host $host; } - location = /message/redoc { proxy_pass http://message_service/redoc; proxy_set_header Host $host; } - location = /message/openapi.json { proxy_pass http://message_service/openapi.json; proxy_set_header Host $host; } + location = /message/docs { + set $message_svc "message_service:8006"; + proxy_pass http://$message_svc/docs; + proxy_set_header Host $host; + } + location = /message/redoc { + set $message_svc "message_service:8006"; + proxy_pass http://$message_svc/redoc; + proxy_set_header Host $host; + } + location = /message/openapi.json { + set $message_svc "message_service:8006"; + proxy_pass http://$message_svc/openapi.json; + proxy_set_header Host $host; + } location /message/ws/ { - proxy_pass http://message_service/ws/; + set $message_svc "message_service:8006"; + proxy_pass http://$message_svc/ws/; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; @@ -197,7 +270,8 @@ http { auth_request_set $user_email $upstream_http_x_user_email; error_page 401 = @error401; error_page 403 = @error403; - proxy_pass http://message_service/; + set $message_svc "message_service:8006"; + proxy_pass http://$message_svc/; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -207,12 +281,25 @@ http { proxy_set_header Authorization $http_authorization; } - location = /notifications/docs { proxy_pass http://notification_service/docs; proxy_set_header Host $host; } - location = /notifications/redoc { proxy_pass http://notification_service/redoc; proxy_set_header Host $host; } - location = /notifications/openapi.json { proxy_pass http://notification_service/openapi.json; proxy_set_header Host $host; } + location = /notifications/docs { + set $notif_svc "notification_service:8007"; + proxy_pass http://$notif_svc/docs; + proxy_set_header Host $host; + } + location = /notifications/redoc { + set $notif_svc "notification_service:8007"; + proxy_pass http://$notif_svc/redoc; + proxy_set_header Host $host; + } + location = /notifications/openapi.json { + set $notif_svc "notification_service:8007"; + proxy_pass http://$notif_svc/openapi.json; + proxy_set_header Host $host; + } location /notifications/ws/ { - proxy_pass http://notification_service/ws/; + set $notif_svc "notification_service:8007"; + proxy_pass http://$notif_svc/ws/; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; @@ -229,7 +316,8 @@ http { auth_request_set $user_email $upstream_http_x_user_email; error_page 401 = @error401; error_page 403 = @error403; - proxy_pass http://notification_service/; + set $notif_svc "notification_service:8007"; + proxy_pass http://$notif_svc/; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -239,9 +327,21 @@ http { proxy_set_header Authorization $http_authorization; } - location = /posts/docs { proxy_pass http://blog_service/docs; proxy_set_header Host $host; } - location = /posts/redoc { proxy_pass http://blog_service/redoc; proxy_set_header Host $host; } - location = /posts/openapi.json { proxy_pass http://blog_service/openapi.json; proxy_set_header Host $host; } + location = /posts/docs { + set $blog_svc "blog_service:8008"; + proxy_pass http://$blog_svc/docs; + proxy_set_header Host $host; + } + location = /posts/redoc { + set $blog_svc "blog_service:8008"; + proxy_pass http://$blog_svc/redoc; + proxy_set_header Host $host; + } + location = /posts/openapi.json { + set $blog_svc "blog_service:8008"; + proxy_pass http://$blog_svc/openapi.json; + proxy_set_header Host $host; + } location /posts/ { limit_req zone=api_limit burst=20 nodelay; @@ -250,7 +350,8 @@ http { auth_request_set $user_id $upstream_http_x_user_id; auth_request_set $user_role $upstream_http_x_user_role; auth_request_set $user_email $upstream_http_x_user_email; - proxy_pass http://blog_service/; + set $blog_svc "blog_service:8008"; + proxy_pass http://$blog_svc/; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -261,7 +362,8 @@ http { } location /ussd/ { - proxy_pass http://ussd_service/; + set $ussd_svc "ussd_service:8009"; + proxy_pass http://$ussd_svc/; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -333,4 +435,4 @@ http { add_header Content-Type text/plain; } } -} \ No newline at end of file +} diff --git a/scripts/destroy_seed.py b/scripts/destroy_seed.py new file mode 100644 index 0000000..d47d52f --- /dev/null +++ b/scripts/destroy_seed.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +""" +Destroy all data written by seed.py. + +Reads scripts/.seed_manifest.json for the exact IDs that were seeded, +then runs targeted SQL deletes against each service database via docker exec. + +Safe to run even if services are not all reachable — each step is independent. +""" +import json +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +MANIFEST = Path(__file__).parent / ".seed_manifest.json" + + +# ── SQL helper ──────────────────────────────────────────────────────────────── + +def psql(container: str, user: str, db: str, sql: str, compose_file: str = "docker-compose.yml") -> bool: + """Run SQL in a postgres container. Returns True on success.""" + cmd = [ + "docker", "compose", "-f", compose_file, + "exec", "-T", container, + "psql", "-U", user, "-d", db, "-c", sql, + ] + result = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True) + if result.returncode != 0: + stderr = result.stderr.strip() + # Silence "table not found" and "no rows" — those are fine + if "does not exist" not in stderr and stderr: + print(f" WARN ({container}): {stderr[:120]}") + return False + return True + + +def ids_literal(ids: list[str]) -> str: + """Build a SQL literal list: 'id1'::uuid, 'id2'::uuid, ...""" + return ", ".join(f"'{i}'::uuid" for i in ids) + + +# ── Step functions ──────────────────────────────────────────────────────────── + +def destroy_blog_posts(farmer_ids: list[str]) -> None: + if not farmer_ids: + return + print(" blog posts + sections + likes ...") + lit = ids_literal(farmer_ids) + # post_sections and post_likes cascade via FK if set; otherwise delete explicitly + psql("blog_db", "blog_user", "blog_db", + f"DELETE FROM post_likes WHERE post_id IN (SELECT id FROM posts WHERE author_id IN ({lit}));") + psql("blog_db", "blog_user", "blog_db", + f"DELETE FROM comments WHERE post_id IN (SELECT id FROM posts WHERE author_id IN ({lit}));") + psql("blog_db", "blog_user", "blog_db", + f"DELETE FROM post_sections WHERE post_id IN (SELECT id FROM posts WHERE author_id IN ({lit}));") + psql("blog_db", "blog_user", "blog_db", + f"DELETE FROM posts WHERE author_id IN ({lit});") + + +def destroy_product_reviews(buyer_ids: list[str]) -> None: + if not buyer_ids: + return + print(" product reviews + helpful votes ...") + lit = ids_literal(buyer_ids) + psql("produce_db", "produce_user", "produce_db", + f"DELETE FROM product_review_helpful WHERE review_id IN " + f"(SELECT id FROM product_reviews WHERE reviewer_id IN ({lit}));") + psql("produce_db", "produce_user", "produce_db", + f"DELETE FROM product_reviews WHERE reviewer_id IN ({lit});") + + +def destroy_listings(farmer_ids: list[str]) -> None: + if not farmer_ids: + return + print(" produce listings + price tiers + images ...") + lit = ids_literal(farmer_ids) + listing_subq = f"SELECT id FROM listings WHERE farmer_id IN ({lit})" + psql("produce_db", "produce_user", "produce_db", + f"DELETE FROM product_review_helpful WHERE review_id IN " + f"(SELECT id FROM product_reviews WHERE listing_id IN ({listing_subq}));") + psql("produce_db", "produce_user", "produce_db", + f"DELETE FROM product_reviews WHERE listing_id IN ({listing_subq});") + psql("produce_db", "produce_user", "produce_db", + f"DELETE FROM price_tiers WHERE listing_id IN ({listing_subq});") + psql("produce_db", "produce_user", "produce_db", + f"DELETE FROM listing_images WHERE listing_id IN ({listing_subq});") + psql("produce_db", "produce_user", "produce_db", + f"DELETE FROM listings WHERE farmer_id IN ({lit});") + + +def destroy_messages(all_user_ids: list[str]) -> None: + if not all_user_ids: + return + print(" conversations + messages ...") + lit = ids_literal(all_user_ids) + conv_subq = f"SELECT id FROM conversations WHERE buyer_id IN ({lit}) OR farmer_id IN ({lit})" + psql("message_db", "message_user", "message_db", + f"DELETE FROM messages WHERE conversation_id IN ({conv_subq});") + psql("message_db", "message_user", "message_db", + f"DELETE FROM conversations WHERE buyer_id IN ({lit}) OR farmer_id IN ({lit});") + + +def destroy_payments(order_ids: list[str]) -> None: + if not order_ids: + return + print(" payment transactions ...") + lit = ids_literal(order_ids) + psql("payment_db", "payment_user", "payment_db", + f"DELETE FROM transactions WHERE order_id IN ({lit});") + + +def destroy_orders(buyer_ids: list[str]) -> None: + if not buyer_ids: + return + print(" orders + order items ...") + lit = ids_literal(buyer_ids) + order_subq = f"SELECT id FROM orders WHERE buyer_id IN ({lit})" + psql("order_db", "order_user", "order_db", + f"DELETE FROM order_items WHERE order_id IN ({order_subq});") + psql("order_db", "order_user", "order_db", + f"DELETE FROM orders WHERE buyer_id IN ({lit});") + + +def destroy_user_profiles(all_user_ids: list[str]) -> None: + if not all_user_ids: + return + print(" user profiles + stats + follows ...") + lit = ids_literal(all_user_ids) + psql("user_db", "user_user", "user_db", + f"DELETE FROM farmer_follows WHERE follower_id IN ({lit}) OR farmer_id IN ({lit});") + psql("user_db", "user_user", "user_db", + f"DELETE FROM review_helpful WHERE voter_id IN ({lit});") + psql("user_db", "user_user", "user_db", + f"DELETE FROM farmer_reviews WHERE farmer_id IN ({lit}) OR reviewer_id IN ({lit});") + psql("user_db", "user_user", "user_db", + f"DELETE FROM user_settings WHERE user_id IN ({lit});") + psql("user_db", "user_user", "user_db", + f"DELETE FROM buyer_stats WHERE user_id IN ({lit});") + psql("user_db", "user_user", "user_db", + f"DELETE FROM farmer_stats WHERE user_id IN ({lit});") + psql("user_db", "user_user", "user_db", + f"DELETE FROM user_profiles WHERE id IN ({lit});") + + +def destroy_auth_credentials() -> None: + print(" auth credentials (@sokodev.ug accounts) ...") + psql("auth_db", "auth_user", "auth_db", + "DELETE FROM auth_credentials WHERE email LIKE '%@sokodev.ug';") + + +def reset_ml_feature_store() -> None: + print(" ML feature store (user_profiles, price_observations, interactions, coverage_gaps) ...") + ml_compose = str(ROOT / "services" / "soko-ml" / "docker-compose.yml") + result = subprocess.run( + [ + "docker", "compose", "-f", ml_compose, + "--project-directory", str(ROOT / "services" / "soko-ml"), + "exec", "-T", "soko-ml-db", + "psql", "-U", "soko_ml", "-d", "soko_ml_db", + "-c", "TRUNCATE user_profiles, price_observations, interactions, coverage_gaps RESTART IDENTITY CASCADE;", + ], + cwd=ROOT, + capture_output=True, + text=True, + ) + if result.returncode != 0 and "does not exist" not in result.stderr: + print(f" WARN (soko-ml-db): {result.stderr.strip()[:120]}") + + +# ── Entry point ─────────────────────────────────────────────────────────────── + +def main() -> None: + if not MANIFEST.exists(): + print("No seed manifest found (scripts/.seed_manifest.json).") + print("If you seeded manually, delete data by hand or truncate the databases.") + sys.exit(0) + + manifest = json.loads(MANIFEST.read_text()) + farmer_ids = [f["id"] for f in manifest.get("farmers", [])] + buyer_ids = [b["id"] for b in manifest.get("buyers", [])] + order_ids = manifest.get("order_ids", []) + all_ids = farmer_ids + buyer_ids + + if not all_ids: + print("Manifest contains no user IDs. Nothing to destroy.") + MANIFEST.unlink(missing_ok=True) + sys.exit(0) + + print(f"\nDestroying seed data for {len(farmer_ids)} farmer(s) and {len(buyer_ids)} buyer(s)...\n") + + destroy_blog_posts(farmer_ids) + destroy_product_reviews(buyer_ids) + destroy_listings(farmer_ids) + destroy_messages(all_ids) + destroy_payments(order_ids) + destroy_orders(buyer_ids) + destroy_user_profiles(all_ids) + destroy_auth_credentials() + reset_ml_feature_store() + + MANIFEST.unlink(missing_ok=True) + print("\n Manifest removed.") + print("\nAll seed data destroyed. Run 'make seed' to re-seed.") + + +if __name__ == "__main__": + main() diff --git a/scripts/fill_envs.py b/scripts/fill_envs.py new file mode 100644 index 0000000..d8ed246 --- /dev/null +++ b/scripts/fill_envs.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +""" +Write consistent dev credentials to all core service .env files. + +Idempotent: only fills keys that are currently missing or empty, unless +--force is passed (which overwrites everything). + +Run before 'make seed' or after a fresh 'make setup'. +""" +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +SERVICES = ROOT / "services" + +# ── Shared dev values ───────────────────────────────────────────────────────── +INTERNAL_SECRET = "internal-secret" # matches ML INTERNAL_API_KEY +SECRET_KEY = "soko-dev-secret-key-2026-change-before-production" +FRONTEND_URL = "http://localhost:3000" + +# ── Per-service desired env values ──────────────────────────────────────────── +ENVS: dict[str, dict[str, str]] = { + "auth": { + "DATABASE_URL": "postgresql://auth_user:auth_pass@auth_db:5432/auth_db", + "SECRET_KEY": SECRET_KEY, + "INTERNAL_SECRET": INTERNAL_SECRET, + "FRONTEND_URL": FRONTEND_URL, + "USER_SERVICE_URL": "http://user_service:8002", + "GOOGLE_CLIENT_ID": "", + "GOOGLE_CLIENT_SECRET": "", + "GOOGLE_REDIRECT_URI": "http://localhost/auth/google/callback", + }, + "user": { + "DATABASE_URL": "postgresql://user_user:user_pass@user_db:5432/user_db", + "INTERNAL_SECRET": INTERNAL_SECRET, + "AUTH_SERVICE_URL": "http://auth_service:8001", + }, + "produce": { + "DATABASE_URL": "postgresql://produce_user:produce_pass@produce_db:5432/produce_db", + "INTERNAL_SECRET": INTERNAL_SECRET, + "USER_SERVICE_URL": "http://user_service:8002", + "REDIS_URL": "redis://redis:6379/0", + "CLOUDINARY_CLOUD_NAME": "", + "CLOUDINARY_API_KEY": "", + "CLOUDINARY_API_SECRET": "", + }, + "order": { + "DATABASE_URL": "postgresql://order_user:order_pass@order_db:5432/order_db", + "INTERNAL_SECRET": INTERNAL_SECRET, + "PRODUCE_SERVICE_URL": "http://produce_service:8003", + "USER_SERVICE_URL": "http://user_service:8002", + "PAYMENT_SERVICE_URL": "http://payment_service:8005", + "NOTIFICATION_SERVICE_URL": "http://notification_service:8007", + }, + "payment": { + "DATABASE_URL": "postgresql://payment_user:payment_pass@payment_db:5432/payment_db", + "INTERNAL_SECRET": INTERNAL_SECRET, + "PESAPAL_CONSUMER_KEY": "", + "PESAPAL_CONSUMER_SECRET": "", + "PESAPAL_ENV": "sandbox", + "PESAPAL_IPN_URL": "http://localhost/webhook/pesapal/ipn", + "PESAPAL_CALLBACK_URL": "http://localhost/webhook/pesapal/callback", + "ORDER_SERVICE_URL": "http://order_service:8004", + "USER_SERVICE_URL": "http://user_service:8002", + "NOTIFICATION_SERVICE_URL": "http://notification_service:8007", + "FRONTEND_URL": FRONTEND_URL, + }, + "message": { + "DATABASE_URL": "postgresql://message_user:message_pass@message_db:5432/message_db", + "INTERNAL_SECRET": INTERNAL_SECRET, + "SECRET_KEY": SECRET_KEY, + "USER_SERVICE_URL": "http://user_service:8002", + "PRODUCE_SERVICE_URL": "http://produce_service:8003", + "NOTIFICATION_SERVICE_URL": "http://notification_service:8007", + }, + "notification": { + "DATABASE_URL": "postgresql://notification_user:notification_pass@notification_db:5432/notification_db", + "INTERNAL_SECRET": INTERNAL_SECRET, + "SECRET_KEY": SECRET_KEY, + "ALGORITHM": "HS256", + "AT_USERNAME": "", + "AT_API_KEY": "", + "AT_SENDER_ID": "", + "USER_SERVICE_URL": "http://user_service:8002", + }, + "blog": { + "DATABASE_URL": "postgresql://blog_user:blog_pass@blog_db:5432/blog_db", + "INTERNAL_SECRET": INTERNAL_SECRET, + "REDIS_URL": "redis://redis:6379/1", + "USER_SERVICE_URL": "http://user_service:8002", + # Placeholder values so the service starts; image uploads will fail gracefully + "CLOUDINARY_CLOUD_NAME": "soko_dev", + "CLOUDINARY_API_KEY": "dev_key", + "CLOUDINARY_API_SECRET": "dev_secret", + }, + "ussd": { + "DATABASE_URL": "postgresql://ussd_user:ussd_pass@ussd_db:5432/ussd_db", + "INTERNAL_SECRET": INTERNAL_SECRET, + "AT_USERNAME": "", + "AT_API_KEY": "", + "PRODUCE_SERVICE_URL": "http://produce_service:8003", + "ORDER_SERVICE_URL": "http://order_service:8004", + "AUTH_SERVICE_URL": "http://auth_service:8001", + "USER_SERVICE_URL": "http://user_service:8002", + "NOTIFICATION_SERVICE_URL": "http://notification_service:8007", + }, +} + + +def _parse_env(path: Path) -> dict[str, str]: + result: dict[str, str] = {} + if not path.exists(): + return result + for raw in path.read_text().splitlines(): + line = raw.strip() + if not line or line.startswith("#"): + continue + if "=" in line: + k, _, v = line.partition("=") + result[k.strip()] = v.strip() + return result + + +def _write_env(path: Path, desired: dict[str, str], force: bool) -> None: + existing = _parse_env(path) + merged = dict(existing) + filled: list[str] = [] + + for key, val in desired.items(): + current = existing.get(key, None) + if force or current is None or current == "": + if current != val: + merged[key] = val + filled.append(key) + + path.write_text("\n".join(f"{k}={v}" for k, v in merged.items()) + "\n") + + rel = path.relative_to(ROOT) + if filled: + print(f" updated {rel} ({len(filled)} key(s): {', '.join(filled)})") + else: + print(f" ok {rel} (already complete)") + + +def main() -> None: + force = "--force" in sys.argv + if force: + print("--force: overwriting all .env values") + + print("\nFilling service .env files with dev credentials...\n") + + for service, env_vars in ENVS.items(): + env_path = SERVICES / service / ".env" + if not env_path.parent.exists(): + print(f" skip {service}/.env (directory not found)") + continue + # Create .env from .env.example if it doesn't exist yet + if not env_path.exists(): + example = env_path.with_suffix(".env.example") + if example.exists(): + import shutil + shutil.copy(example, env_path) + _write_env(env_path, env_vars, force=force) + + # ML stack .env — copy example if missing, leave values as-is + ml_env = SERVICES / "soko-ml" / ".env" + if not ml_env.exists(): + ml_ex = ml_env.with_suffix(".env.example") + if ml_ex.exists(): + import shutil + shutil.copy(ml_ex, ml_env) + print(f" created services/soko-ml/.env from .env.example") + else: + print(f" skip services/soko-ml/.env (no .env.example found)") + else: + # Ensure INTERNAL_API_KEY matches core INTERNAL_SECRET + _write_env(ml_env, {"INTERNAL_API_KEY": INTERNAL_SECRET}, force=force) + + print("\nDone. If services are already running, restart them to pick up new values:") + print(" make core-down && make core-up") + + +if __name__ == "__main__": + main() diff --git a/scripts/seed.py b/scripts/seed.py index 4a96dce..255b871 100644 --- a/scripts/seed.py +++ b/scripts/seed.py @@ -18,15 +18,167 @@ import subprocess import sys import time +from pathlib import Path import requests AUTH = "http://localhost:8001" USER = "http://localhost:8002" PRODUCE = "http://localhost:8003" +ORDER = "http://localhost:8004" +MESSAGE = "http://localhost:8006" +BLOG = "http://localhost:8008" INGEST = "http://localhost:8096" -PASS = "Soko2024!" +PASS = "Soko2024!" +DELIVERY_FEE = 5000 # UGX, matches order service constant +MANIFEST = Path(__file__).parent / ".seed_manifest.json" + +# ── Delivery addresses by district ─────────────────────────────────────────── +DELIVERY_ADDR = { + "Kampala": {"district": "Kampala", "subCounty": "Kawempe", "village": "Bwaise"}, + "Mbarara": {"district": "Mbarara", "subCounty": "Kakoba", "village": "Rutooma"}, + "Gulu": {"district": "Gulu", "subCounty": "Bardege", "village": "Layibi"}, + "Masaka": {"district": "Masaka", "subCounty": "Kimaanya", "village": "Bukakata"}, + "Mbale": {"district": "Mbale", "subCounty": "Wanale", "village": "Nakaloke"}, + "Lira": {"district": "Lira", "subCounty": "Adyel", "village": "Ojwina"}, +} + +# ── Order pairs: (buyer_index, listing_index) ─────────────────────────────── +# listing_index corresponds to the order listings are created: farmers in order +# of FARMERS list then EXTRA_FARMERS, each with their listings in definition order. +# +# Listing idx → (Farmer, Listing name) +# 0 → Nakato → Premium Maize Grain +# 2 → Ssebuliba → Irish Potatoes (Desiree) +# 4 → Okello → White Sorghum +# 6 → Nabukeera → Matoke (Mpologoma) +# 8 → Mugisha → Yellow Beans (K132) +# 10 → Atim → Finger Millet (Okileng) +# 12 → Kawesi → Roma Tomatoes +# 15 → Nambi → White Irish Potatoes +# 18 → Kyomugisha → Cassava Chips (NAADS Grade A) +# 24 → Waiswa → Yellow Beans (Bam 1) +ORDER_PAIRS = [ + (0, 0), # Ssali Martin → Nakato / Premium Maize Grain + (1, 2), # Nansubuga Rachel → Ssebuliba / Irish Potatoes + (2, 4), # Opio Samuel → Okello / White Sorghum + (3, 6), # Nakimuli Diana → Nabukeera / Matoke + (4, 8), # Kiggundu Alex → Mugisha / Yellow Beans K132 + (5, 12), # Katende Brian → Kawesi / Roma Tomatoes + (6, 15), # Birungi Agnes → Nambi / White Irish Potatoes + (7, 10), # Odong Charles → Atim / Finger Millet Okileng + (8, 18), # Nassaka Joyce → Kyomugisha / Cassava Chips NAADS + (9, 24), # Tumwesige Paul → Waiswa / Yellow Beans Bam 1 +] + +# ── Blog posts authored by seeded farmers ──────────────────────────────────── +BLOG_POSTS = [ + { + "farmer_idx": 0, # Nakato Aisha + "title": "How I halved my post-harvest losses with a simple solar dryer", + "excerpt": "Moisture was destroying 30% of my maize every season. This is the low-cost solar dryer setup that changed everything for my Natete farm.", + "category": "Soil & Crops", + "tags": ["maize", "post-harvest", "solar-drying", "storage"], + "body": [ + {"type": "heading", "content": "The problem: moisture ruining stored grain"}, + {"type": "paragraph", "content": "In Natete, the long rains push humidity above 80% for weeks at a time. Before I built my solar dryer, I was losing almost a third of my maize to mould before it ever reached the market. That loss was silent — it happened slowly inside the bag — so it took me two seasons to measure just how bad it was."}, + {"type": "heading", "content": "Building the dryer for under 400,000 UGX"}, + {"type": "paragraph", "content": "The frame is eucalyptus poles, the drying bed is galvanised wire mesh, and the cover is UV-stabilised polythene sheeting. Total cost was 380,000 UGX for materials. A local welder joined the corners. The dryer sits on raised legs to allow airflow from below, and the cover is angled south-facing to capture maximum sun."}, + {"type": "quote", "content": "I went from 30% post-harvest loss to under 5% in one season. The dryer paid for itself before the second harvest.", "attribution": "Nakato Aisha, Natete, Kampala"}, + {"type": "heading", "content": "Moisture targets that matter"}, + {"type": "paragraph", "content": "Maize destined for milling should reach 12–13% moisture. Maize for seed should go even lower — 10–11%. I use a simple grain moisture meter (bought at Owino Market for 45,000 UGX) to check before bagging. Once it is in the bag and sealed, moisture stays stable for up to six months in a cool store."}, + ], + }, + { + "farmer_idx": 1, # Ssebuliba John + "title": "Desiree vs. Victoria: choosing the right Irish potato variety for Ugandan highlands", + "excerpt": "After five seasons of comparative trials on my Rutooma farm, I can tell you which variety outperforms in highland Mbarara soils — and why the answer surprises most buyers.", + "category": "Soil & Crops", + "tags": ["potatoes", "highland", "mbarara", "varieties"], + "body": [ + {"type": "heading", "content": "Why variety matters more than fertiliser"}, + {"type": "paragraph", "content": "Most highland potato farmers in Mbarara default to whatever seed is cheapest at the agro-input shop. I spent five seasons trialling Desiree, Victoria, and Cruza 148 side by side on the same field with the same inputs. The yield gap between the best and worst variety was larger than anything I could achieve by changing fertiliser rates."}, + {"type": "heading", "content": "Trial results summary"}, + {"type": "paragraph", "content": "Desiree consistently delivered 18–22 tonnes per hectare on my clay-loam highland soils. Victoria came in at 14–17 t/ha but showed better late-blight resistance in wet seasons. Cruza 148 was the highest yielder at 24 t/ha but buyers reject it because the skin bruises easily during transport — a serious market problem on Mbarara's rough roads."}, + {"type": "quote", "content": "Marketability matters as much as yield. A 24 t/ha crop that arrives bruised earns you less than a 20 t/ha crop that looks perfect.", "attribution": "Ssebuliba John, Rutooma, Mbarara"}, + {"type": "heading", "content": "My recommendation"}, + {"type": "paragraph", "content": "For farmers selling to Kampala wholesalers via Soko, Desiree is the safest choice. It stores well, travels well, and buyers recognise the name. Use Victoria for local markets or if you expect a particularly wet season."}, + ], + }, + { + "farmer_idx": 4, # Mugisha Robert + "title": "K132 yellow beans: Uganda's most-exported legume and how to grow it right", + "excerpt": "K132 commands a 15–20% export premium over other yellow bean varieties. Here is exactly how we grow it on Mt Elgon volcanic soils to meet export grade.", + "category": "Business", + "tags": ["beans", "export", "k132", "mbale", "quality"], + "body": [ + {"type": "heading", "content": "What makes K132 export-grade"}, + {"type": "paragraph", "content": "K132 yellow bean is a climbing variety developed by NARO specifically for Uganda's eastern highlands. Its seed size is large and uniform (above 25g per 100 seeds), its skin is thin enough for quick cooking, and the colour holds after washing — qualities that meet EU and Middle East import standards."}, + {"type": "heading", "content": "Soil preparation on Mt Elgon slopes"}, + {"type": "paragraph", "content": "Our volcanic soils are naturally rich in phosphorus and potassium, which beans love. We add minimal DAP at planting (just 50 kg/ha) and top-dress with CAN at flowering. Over-fertilising with nitrogen suppresses nodule formation — beans fix their own nitrogen if you inoculate the seed with Rhizobium before planting."}, + {"type": "heading", "content": "Harvest and grading for export"}, + {"type": "paragraph", "content": "We harvest at 90% pod-yellowing and thresh within 48 hours to avoid discolouration. After threshing, we winnow, then hand-sort to remove split seeds, discoloured seeds, and any foreign matter. Export buyers use a tolerance of less than 2% defects — we aim for under 0.5% to ensure we pass."}, + {"type": "quote", "content": "The 15% export premium over local market price completely justifies the extra grading labour. One day of sorting earns more than two extra bags of lower-grade beans.", "attribution": "Mugisha Robert, Nakaloke, Mbale"}, + ], + }, + { + "farmer_idx": 6, # Kawesi Peter + "title": "Year-round Roma tomato supply near Kampala: my drip irrigation setup", + "excerpt": "Most Kampala-area tomato farmers harvest only twice a year and watch prices crash at peak supply. Drip irrigation and staggered planting changed my business model completely.", + "category": "Irrigation", + "tags": ["tomatoes", "drip-irrigation", "kampala", "year-round"], + "body": [ + {"type": "heading", "content": "The seasonal price trap"}, + {"type": "paragraph", "content": "In Wakiso, the two main rainy seasons flood the market with tomatoes and collapse prices to 600–800 UGX/kg. During dry spells, the same tomatoes fetch 2,500–3,000 UGX/kg. I used to join everyone else at the low-price peak. Now I do the opposite."}, + {"type": "heading", "content": "The drip system — cost and setup"}, + {"type": "paragraph", "content": "I installed a gravity-fed drip system from a 10,000-litre tank elevated 3 metres above the field. Total cost was 2.8 million UGX for 0.4 hectares. Drip lines run every 60 cm, emitters every 30 cm. The tank fills from a borehole with a solar pump — 120,000 UGX for the pump, 400,000 UGX for the borehole contribution. I share the borehole with three neighbours."}, + {"type": "heading", "content": "Staggered planting for stable income"}, + {"type": "paragraph", "content": "I plant four batches per year, eight weeks apart. At any given time, one batch is flowering, one is fruiting, one is being harvested, and one is just transplanted. Total annual harvest is about 18 tonnes from 0.4 ha. I sell to processors and supermarkets on monthly contracts — the consistent supply is what earns the contract."}, + {"type": "quote", "content": "Consistent supply is worth more than maximum yield. My buyers pay 1,800–2,200 UGX/kg year-round on a contract. That beats the lottery of seasonal prices.", "attribution": "Kawesi Peter, Wakiso, Kampala"}, + ], + }, +] + +# ── Conversation scripts (buyer messages farmer, farmer replies) ────────────── +CONV_SCRIPTS = [ + # (order_pair_index, buyer_opening, farmer_reply) + (0, "Hello Nakato, I just placed an order for your Premium Maize Grain. Is it current-season stock? I need it for milling so moisture is important.", + "Hello! Yes, this is from the April 2026 harvest, solar-dried to 12% moisture. I can confirm the grade before dispatch if you send me your lab's requirements."), + (1, "Hi, I ordered the Desiree potatoes. Can they be delivered to Kakoba? We need them by Friday for our restaurant.", + "Hello! Kakoba is no problem. I usually deliver Mbarara town every Tuesday and Friday. Friday works perfectly — I will add your address to the route."), + (2, "Okello, I need the white sorghum for malting. Is it the low-tannin variety suitable for beer? Can you do a 200 kg order next time?", + "Yes, this is specifically the low-tannin white sorghum — ideal for malting and brewing. 200 kg is fine, I have plenty in stock. Just place the order when you are ready."), + (3, "Hi Nabukeera, your Matoke Mpologoma looked great in the photos. How green are they right now? I need them to ripen by Sunday.", + "Hello! The bunches I am harvesting this week are at full green — they will ripen naturally in 3 days at room temperature, so ordering today gets you Sunday-ripe matoke. Perfect timing."), + (4, "Mugisha, I am interested in the K132 yellow beans for export. What is your grading standard and can you do a certificate of analysis?", + "Hello! We grade to below 0.5% defects — better than the 2% export tolerance. I can provide a lab certificate from the Mbale NARO office if you need it for your buyer's requirements."), + (5, "Hi Kawesi, I run a restaurant in Kampala. Do you supply Roma tomatoes on a weekly basis? We need about 30 kg every Monday.", + "Hello! Yes, weekly supply is exactly what we do. I supply on Monday and Thursday mornings. 30 kg weekly is manageable — let us agree on a monthly contract and I will give you a stable price of 1,800 UGX/kg."), + (6, "Nambi, I need the white Irish potatoes for my hotel kitchen. How consistent is the sizing? We need 60–80g tubers.", + "Hello! Our Mbarara highland potatoes are graded before dispatch. The white variety comes out 60–80g very consistently — this is what the hotels in Mbarara already specify from us. I will set aside a hotel-grade batch for your next order."), + (7, "Atim, is the Okileng millet traditional-variety or improved? My buyer specifically wants traditional Acholi millet for authenticity.", + "Hello! Okileng is 100% traditional Acholi millet — no hybrid, no improved variety. We have maintained the same seed stock for 20 years. Your buyer will not find more authentic millet than this."), + (8, "Kyomugisha, the NAADS Cassava Chips — are they free from aflatoxin? I need a safety certificate for my export shipment.", + "Hello! All our cassava chips are processed under the NAADS quality programme. I can provide the NAADS inspection certificate and the moisture test result. Aflatoxin testing can be arranged at the Masaka district lab if your buyer requires it."), + (9, "Waiswa, I am looking for Bam 1 yellow beans for a local buyer. What is the minimum I can order and how soon can you deliver?", + "Hello! Minimum order is 50 kg as listed. I deliver to Mbale town twice a week — Tuesday and Saturday. Place the order today and I will include you on Saturday's delivery run."), +] + +# ── Product review data ─────────────────────────────────────────────────────── +REVIEWS = [ + # (order_pair_index, rating, body) + (0, 5, "Excellent maize — 12% moisture as stated, uniform grain size, and the solar drying shows in how clean and dry it arrived. Milled perfectly with zero rejects."), + (1, 5, "Desiree potatoes were exactly as described: consistent 60–80g sizing, clean skin, no bruising despite the Mbarara road. Restaurant customers loved the taste."), + (2, 4, "Good quality white sorghum, low tannin as described. One bag had slightly uneven drying but overall very acceptable for malting. Will reorder."), + (3, 5, "Mpologoma matoke arrived at perfect green stage and ripened beautifully in three days. Large bunches, firm flesh, great flavour. Buying again."), + (4, 5, "K132 yellow beans are exceptional — uniform large seed, clean grade, minimal defects. Our export buyer passed them first inspection. Mugisha is now our primary bean supplier."), + (5, 4, "Roma tomatoes were firm and fresh. One crate had a few over-ripe tomatoes at the bottom but Kawesi resolved it immediately with a replacement. Good supplier."), + (6, 5, "White Irish potatoes are exactly hotel grade — 60–80g, consistent, no greening. Delivered on time as promised. Signed a monthly supply contract."), + (7, 5, "Traditional Okileng millet is exactly what we needed. Our brewery buyer confirmed authenticity and placed a standing order. Atim is reliable and honest."), + (8, 5, "NAADS Grade A cassava chips arrived with all certificates. Perfect 14% moisture, no mould, no aflatoxin. Export shipment cleared customs first attempt."), + (9, 4, "Good Bam 1 beans. Grading was mostly clean though a few split seeds in one bag. Delivery was punctual. Would order again with a note about the split seeds."), +] # ── Farmer data ─────────────────────────────────────────────────────────────── @@ -620,19 +772,218 @@ def create_listings(farmers: list) -> list: ok(f" Publish: {listing_data['name']}", resp) all_listings.append({ - "id": listing_id, - "name": listing_data["name"], - "farmer": f["fullName"], - "district": listing_data["district"], + "id": listing_id, + "name": listing_data["name"], + "farmer": f["fullName"], + "farmer_id": f["id"], + "district": listing_data["district"], + "price": listing_data["price"], + "minimumOrder": listing_data.get("minimumOrder", 50), + "unit": listing_data.get("unit", "kg"), }) return all_listings -# ── Phase 4: Trigger ML bootstrap ──────────────────────────────────────────── +# ── Phase 4: Place orders ──────────────────────────────────────────────────── + +def place_orders(buyers: list, listings: list) -> list: + print("\n── Phase 4: Placing orders (cash on delivery) ──────────────────") + order_ids = [] + + for buyer_idx, listing_idx in ORDER_PAIRS: + if buyer_idx >= len(buyers) or listing_idx >= len(listings): + print(f" SKIP index out of range: buyer={buyer_idx} listing={listing_idx}") + continue + + b = buyers[buyer_idx] + listing = listings[listing_idx] + qty = listing["minimumOrder"] + price = listing["price"] + subtotal = qty * price + total = subtotal + DELIVERY_FEE + + addr_template = DELIVERY_ADDR.get( + b["district"], + {"district": b["district"], "subCounty": "Central", "village": "Town Centre"}, + ) + + payload = { + "items": [{ + "productId": listing["id"], + "quantity": qty, + "unitPrice": price, + "subtotal": subtotal, + }], + "deliveryAddress": { + "fullName": b["fullName"], + "phone": b["phone"], + "district": addr_template["district"], + "subCounty": addr_template["subCounty"], + "village": addr_template["village"], + "landmark": f"Near {addr_template['subCounty']} market", + }, + "paymentMethod": {"type": "cash_on_delivery"}, + "totalAmount": total, + "currency": "UGX", + } + + resp = requests.post( + f"{ORDER}/orders/", + json=payload, + headers=buyer_headers(b["id"]), + ) + if not resp.ok: + print(f" ✗ Order {b['fullName']} → {listing['name']}: {resp.status_code} — {resp.text[:120]}") + continue + + order_id = resp.json().get("id") or resp.json().get("orderId", "") + order_ids.append(order_id) + print(f" ✓ Order: {b['fullName']:<22} → {listing['name']:<30} ({qty} {listing['unit']} × {price:,} UGX)") + + print(f" {len(order_ids)} order(s) created.") + return order_ids + + +# ── Phase 5: Start conversations + replies ──────────────────────────────────── + +def create_conversations(buyers: list, listings: list) -> None: + print("\n── Phase 5: Creating buyer–farmer conversations ─────────────────") + + for script_idx, buyer_msg, farmer_reply in CONV_SCRIPTS: + if script_idx >= len(ORDER_PAIRS): + continue + buyer_idx, listing_idx = ORDER_PAIRS[script_idx] + if buyer_idx >= len(buyers) or listing_idx >= len(listings): + continue + + b = buyers[buyer_idx] + listing = listings[listing_idx] + + # Buyer opens conversation + resp = requests.post( + f"{MESSAGE}/conversations", + json={ + "farmer_id": listing["farmer_id"], + "listing_id": listing["id"], + "first_message": buyer_msg, + }, + headers=buyer_headers(b["id"]), + ) + if not resp.ok: + print(f" ✗ Conv start {b['fullName']} → {listing['farmer']}: {resp.status_code}") + continue + + data = resp.json() + conv_id = data.get("conversation", {}).get("id") or data.get("id", "") + + # Farmer replies + if conv_id: + resp2 = requests.post( + f"{MESSAGE}/conversations/{conv_id}/messages", + json={"body": farmer_reply}, + headers=farmer_headers(listing["farmer_id"]), + ) + if resp2.ok: + print(f" ✓ Conv: {b['fullName']:<22} ↔ {listing['farmer']}") + else: + print(f" ~ Conv opened but reply failed: {resp2.status_code}") + else: + print(f" ~ Conv opened (no conv_id in response to reply)") + + +# ── Phase 6: Blog posts ─────────────────────────────────────────────────────── + +def create_blog_posts(farmers: list) -> None: + print("\n── Phase 6: Publishing farmer blog posts ────────────────────────") + + all_farmers = farmers # same order as FARMERS + EXTRA_FARMERS + for post_def in BLOG_POSTS: + idx = post_def["farmer_idx"] + if idx >= len(all_farmers): + print(f" SKIP blog post (farmer_idx {idx} out of range)") + continue + + f = all_farmers[idx] + payload = { + "title": post_def["title"], + "excerpt": post_def["excerpt"], + "image": "https://images.unsplash.com/photo-1628352081506-83c43123a6b9?w=800", + "category": post_def["category"], + "tags": post_def["tags"], + "body": post_def["body"], + } + + resp = requests.post( + f"{BLOG}/posts/", + json=payload, + headers=farmer_headers(f["id"]), + ) + if not resp.ok: + print(f" ✗ Blog post '{post_def['title'][:50]}': {resp.status_code} — {resp.text[:120]}") + continue + + post_id = resp.json().get("id", "") + + # Publish the draft + resp2 = requests.post( + f"{BLOG}/posts/{post_id}/publish", + headers=farmer_headers(f["id"]), + ) + if resp2.ok: + print(f" ✓ Published: '{post_def['title'][:55]}' ({f['fullName']})") + else: + print(f" ~ Draft created but publish failed: {resp2.status_code}") + + +# ── Phase 7: Product reviews ────────────────────────────────────────────────── + +def create_reviews(buyers: list, listings: list) -> None: + print("\n── Phase 7: Adding product reviews ─────────────────────────────") + + for pair_idx, rating, body in REVIEWS: + if pair_idx >= len(ORDER_PAIRS): + continue + buyer_idx, listing_idx = ORDER_PAIRS[pair_idx] + if buyer_idx >= len(buyers) or listing_idx >= len(listings): + continue + + b = buyers[buyer_idx] + listing = listings[listing_idx] + + resp = requests.post( + f"{PRODUCE}/listings/{listing['id']}/reviews", + json={"rating": rating, "body": body}, + headers={ + **buyer_headers(b["id"]), + "X-User-Name": b["fullName"], + }, + ) + if resp.ok: + print(f" ✓ Review ({rating}★): {b['fullName']:<22} → {listing['name']}") + elif resp.status_code == 409: + print(f" ~ Already reviewed: {b['fullName']} → {listing['name']}") + else: + print(f" ✗ Review failed {b['fullName']} → {listing['name']}: {resp.status_code} — {resp.text[:100]}") + + +# ── Write seed manifest ─────────────────────────────────────────────────────── + +def write_manifest(farmers: list, buyers: list, listings: list, order_ids: list) -> None: + manifest = { + "farmers": [{"id": f["id"], "name": f["fullName"], "email": f["email"]} for f in farmers], + "buyers": [{"id": b["id"], "name": b["fullName"], "email": b["email"]} for b in buyers], + "listing_ids": [l["id"] for l in listings], + "order_ids": order_ids, + } + MANIFEST.write_text(json.dumps(manifest, indent=2)) + print(f"\n Manifest written → {MANIFEST.name}") + + +# ── Phase 8: Trigger ML bootstrap ──────────────────────────────────────────── def trigger_bootstrap(): - print("\n── Phase 4: Triggering data-ingestion bootstrap ────────────────") + print("\n── Phase 8: Triggering data-ingestion bootstrap ────────────────") resp = requests.post(f"{INGEST}/bootstrap") ok("Bootstrap triggered", resp) print(" Waiting 15s for bootstrap to complete...") @@ -648,10 +999,10 @@ def trigger_bootstrap(): print(f" coverage_pairs : {s['coverage_pairs']}") -# ── Phase 5: Restart recommendation service ─────────────────────────────────── +# ── Phase 9: Restart recommendation service ─────────────────────────────────── def reload_recommendation_service(): - print("\n── Phase 5: Reloading recommendation service ───────────────────") + print("\n── Phase 9: Reloading recommendation service ───────────────────") subprocess.run( ["docker", "compose", "restart", "recommendation-service"], cwd="/home/the-icemann/Documents/soko/services/soko-ml", @@ -669,10 +1020,10 @@ def reload_recommendation_service(): # ── Summary ─────────────────────────────────────────────────────────────────── -def print_summary(farmers: list, buyers: list, listings: list): - print("\n" + "═" * 60) - print("SEED COMPLETE — copy these IDs for smoke tests") - print("═" * 60) +def print_summary(farmers: list, buyers: list, listings: list, order_ids: list): + print("\n" + "═" * 68) + print("SEED COMPLETE") + print("═" * 68) print(f"\nFarmers ({len(farmers)}):") for f in farmers: @@ -686,13 +1037,19 @@ def print_summary(farmers: list, buyers: list, listings: list): for l in listings: print(f" {l['id']} {l['name']:<35} {l['farmer']}") - if farmers: - print(f"\nSample smoke-test commands:") + print(f"\nOrders placed : {len(order_ids)}") + + if farmers and buyers: fid = farmers[0]["id"] bid = buyers[0]["id"] + print(f"\nSample smoke-test commands:") print(f" curl -s 'http://localhost:8080/recommend/farmers-for-buyer/{bid}?top_n=3' | python3 -m json.tool") print(f" curl -s 'http://localhost:8080/recommend/buyers-for-farmer/{fid}?top_n=3' | python3 -m json.tool") + print(f" curl -s 'http://localhost:8080/price/predict' \\") + print(f" -H 'Content-Type: application/json' \\") + print(f" -d '{{\"crop\":\"maize_grain\",\"market\":\"Kampala\",\"forecast_days\":7}}' | python3 -m json.tool") + print(f"\n Run 'make destroy-seed' to undo all of the above.") print() @@ -703,7 +1060,12 @@ def print_summary(farmers: list, buyers: list, listings: list): farmers, buyers = register_users() update_profiles(farmers) - listings = create_listings(farmers) + listings = create_listings(farmers) + order_ids = place_orders(buyers, listings) + create_conversations(buyers, listings) + create_blog_posts(farmers) + create_reviews(buyers, listings) trigger_bootstrap() reload_recommendation_service() - print_summary(farmers, buyers, listings) + write_manifest(farmers, buyers, listings, order_ids) + print_summary(farmers, buyers, listings, order_ids) From 33a4b183a4b32023faa578c6cc5ec02f452b3dc5 Mon Sep 17 00:00:00 2001 From: the-icemann Date: Sat, 16 May 2026 11:47:54 +0300 Subject: [PATCH 04/24] Fully integrated the recommendation service to interact with the front-end, added a seed to mimic already present farmers, each all accessing the notification service and providing necessary listings per interest --- Makefile | 12 +- nginx/nginx.conf | 108 +++++++++--- scripts/.seed_manifest.json | 160 ++++++++++++++++++ services/order/app/main.py | 4 +- services/order/app/routers/internal.py | 29 +++- .../src/clients/listing_client.py | 4 +- services/soko-ml/docker-compose.yml | 1 + .../price-prediction-service/models/.gitkeep | 0 8 files changed, 282 insertions(+), 36 deletions(-) create mode 100644 scripts/.seed_manifest.json create mode 100644 services/soko-ml/price-prediction-service/models/.gitkeep diff --git a/Makefile b/Makefile index 7a489e9..4dc86ed 100644 --- a/Makefile +++ b/Makefile @@ -339,6 +339,10 @@ train: $(abspath $(PRICE_VENV))/bin/python -c \ "from src.predictor import train_all_models; train_all_models()" @echo "Models trained → $(ML_DIR)/price-prediction-service/models/" + @if docker inspect soko-ml-price > /dev/null 2>&1; then \ + echo "Restarting price-prediction-service to reload models..."; \ + docker restart soko-ml-price; \ + fi # ============================================================================= # ML — DEVELOPMENT (local uvicorn with hot-reload) @@ -529,28 +533,28 @@ smoke-route: @echo "=== Smoke: Market Route (farmer sell signal) ===" @curl -sf -X POST http://localhost:8080/location/route \ -H 'Content-Type: application/json' \ - -d '{"farmer_id":"F0001","crop":"maize_grain","quantity_kg":500,"harvest_month":8}' \ + -d '{"farmer_id":"48191d0d-86a0-49e0-90ff-078546060a2e","farmer_lat":0.3476,"farmer_lng":32.5825,"crop":"maize_grain","quantity_kg":500}' \ | python3 -m json.tool smoke-discover: @echo "=== Smoke: Discover Farmers Near Buyer ===" @curl -sf -X POST http://localhost:8080/location/discover \ -H 'Content-Type: application/json' \ - -d '{"buyer_id":"B0001","crop":"maize_grain","max_distance_km":150,"max_price_ugx":2000,"top_n":5}' \ + -d '{"buyer_id":"aca85b8c-0be1-48db-831c-359b439783eb","buyer_lat":0.3476,"buyer_lng":32.5825,"crop":"maize_grain","max_distance_km":150,"max_price_ugx":2000,"top_n":5}' \ | python3 -m json.tool smoke-fallback: @echo "=== Smoke: Tier 2 fallback (sesame seed — limited coverage) ===" @curl -sf -X POST http://localhost:8080/location/route \ -H 'Content-Type: application/json' \ - -d '{"farmer_id":"F0001","crop":"sesame","quantity_kg":200,"harvest_month":10}' \ + -d '{"farmer_id":"48191d0d-86a0-49e0-90ff-078546060a2e","farmer_lat":0.3476,"farmer_lng":32.5825,"crop":"sesame","quantity_kg":200}' \ | python3 -m json.tool smoke-tier3: @echo "=== Smoke: Tier 3 unknown crop ===" @curl -sf -X POST http://localhost:8080/location/route \ -H 'Content-Type: application/json' \ - -d '{"farmer_id":"F0001","crop":"moringa","quantity_kg":50,"harvest_month":6}' \ + -d '{"farmer_id":"48191d0d-86a0-49e0-90ff-078546060a2e","farmer_lat":0.3476,"farmer_lng":32.5825,"crop":"moringa","quantity_kg":50}' \ | python3 -m json.tool smoke-ingest: diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 1c09c77..e5a4b8a 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -7,9 +7,73 @@ http { limit_req_zone $binary_remote_addr zone=api_limit:10m rate=30r/m; resolver 127.0.0.11 valid=30s; - # No static upstream blocks — all services use set $var + proxy_pass http://$var/ - # so Docker DNS is re-queried per connection via the resolver above. - # This prevents 502s when containers restart and get new IPs. + + # ── Path-stripping maps ──────────────────────────────────────────────────── + # When proxy_pass uses a variable, Nginx does NOT strip the location prefix + # automatically. These maps pre-compute the upstream path from $request_uri + # so each proxy_pass can forward the correct path while still using a + # variable host (which keeps Docker DNS re-resolution alive). + # ────────────────────────────────────────────────────────────────────────── + + map $request_uri $auth_upstream_path { + ~^/auth(/.*)?$ $1; + default /; + } + map $request_uri $oauth_upstream_path { + ~^/oauth(/.*)?$ $1; + default /; + } + map $request_uri $users_upstream_path { + ~^/users(/.*)?$ $1; + default /; + } + map $request_uri $listings_upstream_path { + ~^/listings(/.*)?$ $1; + default /; + } + map $request_uri $orders_upstream_path { + ~^/orders(/.*)?$ $1; + default /; + } + map $request_uri $payments_upstream_path { + ~^/payments(/.*)?$ $1; + default /; + } + map $request_uri $webhook_upstream_path { + ~^/webhook(/.*)?$ $1; + default /; + } + # WebSocket path must be checked before the generic /message/ prefix + map $request_uri $message_upstream_path { + ~^/message/ws(/.*)?$ /ws$1; + ~^/message(/.*)?$ $1; + default /; + } + map $request_uri $notif_upstream_path { + ~^/notifications/ws(/.*)?$ /ws$1; + ~^/notifications(/.*)?$ $1; + default /; + } + map $request_uri $posts_upstream_path { + ~^/posts(/.*)?$ $1; + default /; + } + map $request_uri $ussd_upstream_path { + ~^/ussd(/.*)?$ $1; + default /; + } + map $request_uri $rec_upstream_path { + ~^/recommendations(/.*)?$ /recommend$1; + default /; + } + map $request_uri $ml_price_upstream_path { + ~^/ml/price(/.*)?$ /price$1; + default /; + } + map $request_uri $ml_rec_upstream_path { + ~^/ml/recommend(/.*)?$ /recommend$1; + default /; + } server { listen 80; @@ -73,7 +137,7 @@ http { location /auth/ { limit_req zone=api_limit burst=20 nodelay; set $auth_svc "auth_service:8001"; - proxy_pass http://$auth_svc/; + proxy_pass http://$auth_svc$auth_upstream_path; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -82,7 +146,7 @@ http { location /oauth/ { limit_req zone=api_limit burst=20 nodelay; set $auth_svc "auth_service:8001"; - proxy_pass http://$auth_svc/; + proxy_pass http://$auth_svc$oauth_upstream_path; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -113,7 +177,7 @@ http { error_page 401 = @error401; error_page 403 = @error403; set $user_svc "user_service:8002"; - proxy_pass http://$user_svc/; + proxy_pass http://$user_svc$users_upstream_path; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -147,7 +211,7 @@ http { auth_request_set $user_role $upstream_http_x_user_role; auth_request_set $user_email $upstream_http_x_user_email; set $produce_svc "produce_service:8003"; - proxy_pass http://$produce_svc/; + proxy_pass http://$produce_svc$listings_upstream_path; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -182,7 +246,7 @@ http { error_page 401 = @error401; error_page 403 = @error403; set $order_svc "order_service:8004"; - proxy_pass http://$order_svc/; + proxy_pass http://$order_svc$orders_upstream_path; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -217,7 +281,7 @@ http { error_page 401 = @error401; error_page 403 = @error403; set $payment_svc "payment_service:8005"; - proxy_pass http://$payment_svc/; + proxy_pass http://$payment_svc$payments_upstream_path; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -229,7 +293,7 @@ http { location /webhook/ { set $payment_svc "payment_service:8005"; - proxy_pass http://$payment_svc/; + proxy_pass http://$payment_svc$webhook_upstream_path; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -253,7 +317,7 @@ http { location /message/ws/ { set $message_svc "message_service:8006"; - proxy_pass http://$message_svc/ws/; + proxy_pass http://$message_svc$message_upstream_path; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; @@ -271,7 +335,7 @@ http { error_page 401 = @error401; error_page 403 = @error403; set $message_svc "message_service:8006"; - proxy_pass http://$message_svc/; + proxy_pass http://$message_svc$message_upstream_path; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -299,7 +363,7 @@ http { location /notifications/ws/ { set $notif_svc "notification_service:8007"; - proxy_pass http://$notif_svc/ws/; + proxy_pass http://$notif_svc$notif_upstream_path; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; @@ -317,7 +381,7 @@ http { error_page 401 = @error401; error_page 403 = @error403; set $notif_svc "notification_service:8007"; - proxy_pass http://$notif_svc/; + proxy_pass http://$notif_svc$notif_upstream_path; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -351,7 +415,7 @@ http { auth_request_set $user_role $upstream_http_x_user_role; auth_request_set $user_email $upstream_http_x_user_email; set $blog_svc "blog_service:8008"; - proxy_pass http://$blog_svc/; + proxy_pass http://$blog_svc$posts_upstream_path; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -363,7 +427,7 @@ http { location /ussd/ { set $ussd_svc "ussd_service:8009"; - proxy_pass http://$ussd_svc/; + proxy_pass http://$ussd_svc$ussd_upstream_path; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -384,7 +448,7 @@ http { error_page 403 = @error403; set $ml_gw "ml-gateway:8000"; - proxy_pass http://$ml_gw/recommend/; + proxy_pass http://$ml_gw$rec_upstream_path; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -394,21 +458,19 @@ http { # ════════════════════════════════════════ # ML GATEWAY - # /ml/price/* — price predictions (public — useful for price discovery) - # /ml/recommend/* — recommendations (protected — personal to the user) + # /ml/price/* — price predictions (public) + # /ml/recommend/* — recommendations (protected) # ════════════════════════════════════════ - # Price endpoints — public so the frontend can show market prices without login location /ml/price/ { limit_req zone=api_limit burst=20 nodelay; set $ml_gw "ml-gateway:8000"; - proxy_pass http://$ml_gw/price/; + proxy_pass http://$ml_gw$ml_price_upstream_path; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; } - # Recommendation endpoints — require a valid token location /ml/recommend/ { limit_req zone=api_limit burst=20 nodelay; @@ -419,7 +481,7 @@ http { error_page 403 = @error403; set $ml_gw "ml-gateway:8000"; - proxy_pass http://$ml_gw/recommend/; + proxy_pass http://$ml_gw$ml_rec_upstream_path; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; diff --git a/scripts/.seed_manifest.json b/scripts/.seed_manifest.json new file mode 100644 index 0000000..453f607 --- /dev/null +++ b/scripts/.seed_manifest.json @@ -0,0 +1,160 @@ +{ + "farmers": [ + { + "id": "48191d0d-86a0-49e0-90ff-078546060a2e", + "name": "Nakato Aisha", + "email": "nakato.aisha@sokodev.ug" + }, + { + "id": "9aaf3d29-f10d-40a3-a50b-8abcd7e41d5c", + "name": "Ssebuliba John", + "email": "ssebuliba.john@sokodev.ug" + }, + { + "id": "a713a5c1-32ad-415c-ac55-94252ad9daad", + "name": "Okello David", + "email": "okello.david@sokodev.ug" + }, + { + "id": "29d0ba18-d17b-4bae-b300-58d16217f68e", + "name": "Nabukeera Grace", + "email": "nabukeera.grace@sokodev.ug" + }, + { + "id": "33ef24dd-29c7-434d-a0bc-75a48175904f", + "name": "Mugisha Robert", + "email": "mugisha.robert@sokodev.ug" + }, + { + "id": "b2be6502-09e2-43ff-92a6-10353dea8e2c", + "name": "Atim Sarah", + "email": "atim.sarah@sokodev.ug" + }, + { + "id": "01f6e455-cd78-48a8-8930-0d8f0f6bc3ac", + "name": "Kawesi Peter", + "email": "kawesi.peter@sokodev.ug" + }, + { + "id": "8dbf16aa-1f72-4aa8-8954-eddf44e7f7bf", + "name": "Nambi Faith", + "email": "nambi.faith@sokodev.ug" + }, + { + "id": "f489f8b3-f755-4aed-abc2-28b5c0612abf", + "name": "Oluru Emmanuel", + "email": "oluru.emmanuel@sokodev.ug" + }, + { + "id": "d5aee3a8-4a73-4314-bae8-4e63ee5dd1b5", + "name": "Kyomugisha Miriam", + "email": "kyomugisha.miriam@sokodev.ug" + }, + { + "id": "9c38383c-7e0b-425b-bf61-f432ea36792b", + "name": "Asiimwe Patrick", + "email": "asiimwe.patrick@sokodev.ug" + }, + { + "id": "e816f13e-28ff-4cda-b6b8-233da325815e", + "name": "Achen Mary", + "email": "achen.mary@sokodev.ug" + }, + { + "id": "459481de-9952-43a4-9ad8-868e616c1675", + "name": "Waiswa Daniel", + "email": "waiswa.daniel@sokodev.ug" + } + ], + "buyers": [ + { + "id": "aca85b8c-0be1-48db-831c-359b439783eb", + "name": "Ssali Martin", + "email": "ssali.martin@sokodev.ug" + }, + { + "id": "a185f32c-8186-4b44-9649-e87758071f24", + "name": "Nansubuga Rachel", + "email": "nansubuga.rachel@sokodev.ug" + }, + { + "id": "f8ad6153-0e11-4f90-8444-2719b2ce68a3", + "name": "Opio Samuel", + "email": "opio.samuel@sokodev.ug" + }, + { + "id": "0caa200f-fbf7-4221-9835-19566de74ca5", + "name": "Nakimuli Diana", + "email": "nakimuli.diana@sokodev.ug" + }, + { + "id": "c3131841-a114-457a-95ad-f2c2865dbac8", + "name": "Kiggundu Alex", + "email": "kiggundu.alex@sokodev.ug" + }, + { + "id": "7a0d5772-fda0-4342-b5f9-c9b49143845c", + "name": "Katende Brian", + "email": "katende.brian@sokodev.ug" + }, + { + "id": "29f23c12-ab53-48d6-92b4-793878356c70", + "name": "Birungi Agnes", + "email": "birungi.agnes@sokodev.ug" + }, + { + "id": "255b4241-644a-47a3-b2d4-5fff4dd78263", + "name": "Odong Charles", + "email": "odong.charles@sokodev.ug" + }, + { + "id": "1f8cde82-5e9c-4a92-8f25-9b56be0d1444", + "name": "Nassaka Joyce", + "email": "nassaka.joyce@sokodev.ug" + }, + { + "id": "15729957-1773-4976-988a-181c92ae3ba6", + "name": "Tumwesige Paul", + "email": "tumwesige.paul@sokodev.ug" + } + ], + "listing_ids": [ + "a1881336-12c1-4253-b7e7-6f601b9e0a08", + "e75af1bf-f6d4-4f4b-a387-1f54e8676ed1", + "0e776356-9934-4eba-b911-33370174032d", + "44a36d19-11f9-44c1-b6eb-1475b38fb667", + "b1443480-2920-4d64-a3d8-a9a61dc89596", + "75dbfa7a-6650-4938-9809-9c92ee4afd68", + "7ebc20bb-2f7d-4aca-a530-7a684c6e1bf6", + "b19c1a23-ce80-4666-92b0-2ef8859bfde8", + "b604e46b-2060-4b00-82d6-187f18d79173", + "65ab4997-c4a3-4e34-bccd-532820a9440f", + "f8e7ef91-222d-437e-a40d-ecd91ba8d97c", + "32945110-6c5c-4d8c-9e4a-e8bbc7ba7444", + "7ac3b02a-116c-43e8-a25d-f1968d169a8c", + "7fc508ca-4e1d-4132-9dcf-db149996cd0a", + "ebf680f4-c76b-4c1a-a977-bb1ca7664f8f", + "99e36ee6-9515-475d-8ca8-1731eedff989", + "cbbc017c-38c0-4f01-a77e-92d5aaf7fef5", + "463c30f7-f1e1-426e-bb84-2cecfa429fb8", + "cc260dec-2cfc-487a-a697-d148c4042ba2", + "8dbe2177-a4ed-4e15-9693-bc03cea8a596", + "f8b79aac-2386-474c-9b65-cddd71a2c4fa", + "b870c505-f284-43a6-9f53-46010ab1cb32", + "5fbbf2ac-4fda-4b2c-8eb5-4ef1942a3bc5", + "503950c9-94a4-47c4-91c2-e11dc85c8339", + "68db090c-b472-4de2-8fa5-f849d746bec6" + ], + "order_ids": [ + "73a5edda-740c-40b3-8a0a-183cc3b43b6e", + "6a6b74ea-2735-41fc-9abc-a7cd546b1009", + "bff8c355-fb1a-4a20-a3f6-7a97c01d6d34", + "c9967bd1-2185-4a45-b240-fb2dc2420089", + "37b40b06-9586-4d94-aa86-954fc8cea3f6", + "e218e66c-ae80-4c6b-8303-60fcd289e5fc", + "263e19c2-22c5-4cf7-9eb6-42ceb734caa4", + "5b0b77bb-d634-45ac-9c07-99df5ee06428", + "adffbd85-5e0a-4380-b4fb-fac5891d83ba", + "38ef29b6-a314-48a7-b078-77054b5fe7c6" + ] +} \ No newline at end of file diff --git a/services/order/app/main.py b/services/order/app/main.py index e5af0ce..4e72d20 100644 --- a/services/order/app/main.py +++ b/services/order/app/main.py @@ -1,7 +1,7 @@ from contextlib import asynccontextmanager from fastapi import FastAPI from app.db.database import Base, engine -from app.routers import orders +from app.routers import internal, orders @asynccontextmanager @@ -17,7 +17,7 @@ async def lifespan(app: FastAPI): root_path="/orders" ) -#app.include_router(internal.router, prefix="/internal") +app.include_router(internal.router, prefix="/internal") app.include_router(orders.router) diff --git a/services/order/app/routers/internal.py b/services/order/app/routers/internal.py index 07c41dc..91aa377 100644 --- a/services/order/app/routers/internal.py +++ b/services/order/app/routers/internal.py @@ -1,17 +1,17 @@ import uuid import logging from datetime import datetime +from typing import List, Optional -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends, HTTPException, Query from sqlalchemy.orm import Session from app.core.dependencies import internal_only from app.db.database import get_db from app.helpers.builders import build_order_out from app.models.order import Order, OrderStatus -from app.schemas.order import PaymentConfirmPayload, PaymentFailedPayload -from app.routers.orders import restore_stock, notify_order_event -from app.routers.orders import notify_order_event, update_buyer_stats +from app.schemas.order import OrderOut, PaymentConfirmPayload, PaymentFailedPayload +from app.routers.orders import restore_stock, notify_order_event, update_buyer_stats logger = logging.getLogger(__name__) router = APIRouter(tags=["Internal"], dependencies=[Depends(internal_only)]) @@ -71,4 +71,23 @@ async def payment_failed( await notify_order_event(order, "payment_failed") logger.info(f"Order {order.id} cancelled — payment failed: {payload.reason}") - return {"message": "Order cancelled", "order_id": str(order.id)} \ No newline at end of file + return {"message": "Order cancelled", "order_id": str(order.id)} + + +@router.get("/orders", response_model=List[OrderOut]) +def list_orders( + status: Optional[str] = Query(default=None), + page: int = Query(default=1, ge=1), + limit: int = Query(default=100, le=500), + db: Session = Depends(get_db), +): + q = db.query(Order) + if status: + try: + q = q.filter(Order.status == OrderStatus(status)) + except ValueError: + raise HTTPException(status_code=400, detail=f"Unknown status: {status}") + return [ + build_order_out(o) + for o in q.offset((page - 1) * limit).limit(limit).all() + ] \ No newline at end of file diff --git a/services/soko-ml/data-ingestion-service/src/clients/listing_client.py b/services/soko-ml/data-ingestion-service/src/clients/listing_client.py index 2f399e7..612c971 100644 --- a/services/soko-ml/data-ingestion-service/src/clients/listing_client.py +++ b/services/soko-ml/data-ingestion-service/src/clients/listing_client.py @@ -25,11 +25,11 @@ async def fetch_all_listings() -> AsyncIterator[dict]: Produce-service uses /listings not /internal/listings. """ page = 1 - async with httpx.AsyncClient(timeout=30.0) as client: + async with httpx.AsyncClient(timeout=30.0, follow_redirects=True) as client: while True: try: resp = await client.get( - f"{LISTING_SERVICE_URL}/listings", + f"{LISTING_SERVICE_URL}/listings/", params={"page": page, "limit": PAGE_LIMIT}, ) resp.raise_for_status() diff --git a/services/soko-ml/docker-compose.yml b/services/soko-ml/docker-compose.yml index e6d8d73..b568183 100644 --- a/services/soko-ml/docker-compose.yml +++ b/services/soko-ml/docker-compose.yml @@ -280,6 +280,7 @@ services: - USER_SERVICE_URL=${USER_SERVICE_URL:-http://user_service:8002} - ORDER_SERVICE_URL=${ORDER_SERVICE_URL:-http://order_service:8004} - PRODUCE_SERVICE_URL=${PRODUCE_SERVICE_URL:-http://produce_service:8003} + - LISTING_SERVICE_URL=${LISTING_SERVICE_URL:-http://produce_service:8003} - REC_SERVICE_URL=http://recommendation-service:8002 - INTERNAL_API_KEY=${INTERNAL_API_KEY:-internal-secret} - BOOTSTRAP_ON_STARTUP=${BOOTSTRAP_ON_STARTUP:-true} diff --git a/services/soko-ml/price-prediction-service/models/.gitkeep b/services/soko-ml/price-prediction-service/models/.gitkeep new file mode 100644 index 0000000..e69de29 From 30e6cacd1a79a94881ef70cdacd73855565f94a8 Mon Sep 17 00:00:00 2001 From: the-icemann Date: Sat, 16 May 2026 12:02:12 +0300 Subject: [PATCH 05/24] Full report for the service --- RECOMMENDATION_SERVICE_REPORT.txt | 669 ++++++++++++++++++++++++++++++ 1 file changed, 669 insertions(+) create mode 100644 RECOMMENDATION_SERVICE_REPORT.txt diff --git a/RECOMMENDATION_SERVICE_REPORT.txt b/RECOMMENDATION_SERVICE_REPORT.txt new file mode 100644 index 0000000..0846e1e --- /dev/null +++ b/RECOMMENDATION_SERVICE_REPORT.txt @@ -0,0 +1,669 @@ +================================================================================ + SOKO PLATFORM — RECOMMENDATION SERVICE & NOTIFICATION PIPELINE + Engineering Report · May 2026 +================================================================================ + +──────────────────────────────────────────────────────────────────────────────── + 1. SCOPE OF WORK +──────────────────────────────────────────────────────────────────────────────── + +This report covers the full design, integration, debugging, and hardening of +the ML-driven recommendation service and its downstream notification pipeline +on the Soko agricultural marketplace platform. The work spanned the entire +stack: ML feature stores, the FastAPI ML gateway, the notification service, +the user service, the produce service, and the React/TanStack frontend client. + +The system is designed for production scale — hundreds of thousands of farmers +and buyers — with no hardcoded users, roles, or identifiers at any layer. + + +──────────────────────────────────────────────────────────────────────────────── + 2. SYSTEM ARCHITECTURE OVERVIEW +──────────────────────────────────────────────────────────────────────────────── + + Main Stack (docker-compose.yml at project root) + ───────────────────────────────────────────────── + auth_service :8001 Credential management, JWT issuance + user_service :8002 Profiles, stats, settings + produce_service :8003 Listings, images, reviews, browse cache + order_service :8004 Order lifecycle + payment_service :8005 Mpesa/payment processing + message_service :8006 Direct messaging + notification_service :8007 In-app + SMS + WebSocket push notifications + blog_service :8008 Blog posts + ussd_service :8009 USSD gateway (feature phone support) + redis shared Produce browse cache + farmer listings cache + nginx :80 API gateway / reverse proxy + + ML Stack (services/soko-ml/docker-compose.yml) + ───────────────────────────────────────────────── + ml-gateway :8080 Orchestration hub — receives hooks, dispatches + price-prediction :8094 Prophet time-series price forecasting + recommendation :8095 Cosine similarity matching engine + data-ingestion :8096 Kafka consumer / price observation pipeline + location :8097 District price coverage analysis + soko-ml-db Postgres — farmer_features, buyer_features, + price_observations, coverage_gaps tables + soko-ml-redis Recommendation result cache (rec:* keys) + kafka + zookeeper Price event streaming + + Network Bridge + ───────────────────────────────────────────────── + soko-ml-bridge (external Docker network) connects the two stacks. + Services that talk to ML: user_service, produce_service, order_service, + notification_service, nginx. + + +──────────────────────────────────────────────────────────────────────────────── + 3. RECOMMENDATION SERVICE — HOW IT WORKS +──────────────────────────────────────────────────────────────────────────────── + + Feature Stores + ───────────────────────────────────────────────── + farmer_features Stores: farmer_id, name, district, crops[], total_listings + buyer_features Stores: buyer_id, name, district, preferred_crops[] + + Both stores are populated via two event hooks: + + a) /listing-published — fired by produce_service as a BackgroundTask + when a farmer activates a listing. Upserts the farmer into + farmer_features and triggers a recommendation reload. + + b) /user-registered — fired by user_service as a BackgroundTask + immediately after a new user profile is committed to the DB. + Upserts the user into the appropriate feature store (or both, + for role=both). Triggers recommendation reload and cross-role + notifications (see Section 5). + + Matching Algorithm + ───────────────────────────────────────────────── + The recommendation service computes cosine similarity between: + - A farmer's crop specialties vs. a buyer's crop interests + - Optionally weighted by district proximity + + Endpoint: GET /recommend/buyers-for-farmer/{farmer_id}?top_n=N + Endpoint: GET /recommend/farmers-for-buyer/{buyer_id}?top_n=N + + Results are cached in soko-ml-redis under keys: + rec:buyers:{farmer_id}:{top_n} + rec:farmers:{buyer_id}:{top_n} + + Cache is flushed on every /internal/reload call, which is triggered + after every listing-published and user-registered event. + + Current Feature Store State (as of report date) + ───────────────────────────────────────────────── + farmer_features : 16 records + buyer_features : 13 records + Active listings : 71 (across 18 farmers + 1 both-role account) + User accounts : 18 farmers, 12 buyers, 1 both + + +──────────────────────────────────────────────────────────────────────────────── + 4. NOTIFICATION PIPELINE — HOW IT WORKS +──────────────────────────────────────────────────────────────────────────────── + + Notification Types + ───────────────────────────────────────────────── + listing_match Buyer receives when a matching farmer publishes or joins + buyer_match Farmer receives when a matching buyer joins + order_placed Farmer receives when a buyer places an order + payment_confirmed / payment_failed / order_dispatched / order_delivered + order_cancelled New message / new_review / new_follower / system + + Channels + ───────────────────────────────────────────────── + in_app Stored in notification_db, served via GET /notifications/me + sms Africa's Talking SMS for high-value events (order_placed, + payment_confirmed, payment_failed, system) + push Real-time WebSocket push via GET /notifications/ws/{user_id} + + Delivery Flow + ───────────────────────────────────────────────── + 1. Producer service (produce, order, payment, message, user) fires an + internal HTTP POST to notification_service/internal/notify with an + event name, user IDs, and optional meta dict. + + 2. notification_service/internal/notify: + - Fetches user profiles concurrently via asyncio.gather (to get phone, + role etc.) using a shared httpx.AsyncClient (no per-call TCP overhead) + - Resolves the template for (event, role) + - Applies deduplication: skips if an identical in-app notification for + the same (user, event, entity_id) was sent within DEDUP_MINUTES (10) + - Writes the in-app Notification record to notification_db + - Pushes a real-time payload over WebSocket + - Optionally sends an SMS for high-priority events + + 3. The frontend Zustand store opens a WebSocket on connect, receives + push payloads, and merges them into local state without a page reload. + + entity_type / entity_id Deep-Link Routing + ───────────────────────────────────────────────── + Every notification carries entity_type and entity_id, which the frontend + uses to navigate when the notification is tapped: + + entity_type entity_id Frontend route + ────────────────────────────────────────────────── + listing listing slug /marketplace/{slug} + profile farmer user_id /farmers/{user_id} + order order_id /profile + message message_id /messages + + Callers can override a template's default entity_type via meta["entity_type"] + in the notify payload. This is used by the user-registered hook to route + listing_match notifications to /farmers/{id} rather than /marketplace/{id}. + + +──────────────────────────────────────────────────────────────────────────────── + 5. USER REGISTRATION HOOK — CROSS-ROLE NOTIFICATIONS +──────────────────────────────────────────────────────────────────────────────── + + When a new user registers, the user_service fires /user-registered to + the ML gateway as a fire-and-forget BackgroundTask (does not block the + registration response). + + The ML gateway handles role routing: + + role = farmer + → Upsert into farmer_features + → Reload recommendations + → Fetch top N matching buyers → send each buyer a listing_match + notification (entity_type=profile, entity_id=farmer_user_id) + so the buyer can tap through to the farmer's profile page. + + role = buyer + → Upsert into buyer_features + → Reload recommendations + → Fetch top N matching farmers → send each farmer a buyer_match + notification (no entity_id — no public buyer profile route yet). + + role = both + → Upsert into both feature stores + → Both notification flows run concurrently via asyncio.gather + + Configurable via environment variable: USER_REGISTERED_TOP_N (default: 3) + Listing-published top N: LISTING_PUBLISHED_TOP_N (default: configurable) + + +──────────────────────────────────────────────────────────────────────────────── + 6. BUGS FIXED +──────────────────────────────────────────────────────────────────────────────── + + BUG 1 — Crop normalization mismatch (zero recommendations returned) + ───────────────────────────────────────────────────────────────────── + File: services/soko-ml/recommendation-service/src/feature_store_client.py + + Root cause: Farmer crops were stored as "maize_grain" (snake_case, with + commodity qualifier suffixes), while buyer interests were stored as "Maize" + (Title Case, no qualifier). The cosine similarity set intersection was + always empty, so the recommender returned no matches for any pair. + + Fix: Added _normalize_crop() — strips qualifier suffixes (_grain, _chips, + _flour, _dried etc.), lowercases, and replaces underscores with spaces. + Applied to both farmer specialties and buyer interests at load time when + building the in-memory DataFrames. DB values are unchanged; normalization + happens at read time only. + + _CROP_QUALIFIERS = ("_grain", "_chips", "_flour", "_dried", "_fresh", + "_organic", "_grade", "_premium") + + def _normalize_crop(raw: str) -> str: + c = raw.lower().strip() + for q in _CROP_QUALIFIERS: + if c.endswith(q): + c = c[:-len(q)] + return c.replace("_", " ") + + Impact: Without this fix, no buyer-farmer match was ever computed. + Recommendations returned empty lists for every query. + + + BUG 2 — destroy_seed.py truncating non-existent tables + ───────────────────────────────────────────────────────────────────── + File: scripts/destroy_seed.py + + Root cause: reset_ml_feature_store() was attempting TRUNCATE on tables + named "user_profiles" and "interactions" — neither of which exist in + soko_ml_db. The actual tables are farmer_features and buyer_features. + The function silently failed, leaving stale ML data after every seed + destroy, which caused incorrect recommendations on re-seed. + + Fix: + - DELETE from farmer_features WHERE farmer_id IN (seeded farmer IDs) + - DELETE from buyer_features WHERE buyer_id IN (seeded buyer IDs) + - TRUNCATE coverage_gaps, price_observations (safely regenerable) + - Added flush_ml_redis_cache() — scans and deletes all rec:* keys + - Added destroy_notifications() — cleans notification records for + seeded users so Billy/Fred/etc. start with a fresh slate on re-seed + + + BUG 3 — Farmer entity_id silently dropped in notify endpoint + ───────────────────────────────────────────────────────────────────── + File: services/notification/app/routers/internal.py + + Root cause: The farmer notification path in notify() used: + entity_id=payload.order_id + This meant buyer_match notifications from listing-published events (which + pass entity_id not order_id) always stored entity_id=None, making the + farmer's notification non-navigable. + + Fix: + entity_id=payload.order_id or payload.entity_id + + + BUG 4 — WebSocket push sending stale entity_type + ───────────────────────────────────────────────────────────────────── + File: services/notification/app/routers/internal.py + + Root cause: The DB write used effective_entity_type (which respects the + meta override), but the WebSocket push payload used template.entity_type + (the hardcoded template default). A user with the app open at the moment + of delivery would receive a real-time push with the wrong entity_type, + routing them to the wrong page on tap. + + Fix: Changed the push payload to use effective_entity_type, making the + DB record and the real-time push consistent. + + + BUG 5 — listing_match notification storing user_id as entity_id with + entity_type=listing (wrong entity_type for user-registered flow) + ───────────────────────────────────────────────────────────────────── + File: services/soko-ml/ml-gateway-service/src/main.py + + Root cause: When a new farmer registers, the ML gateway sends matching + buyers a listing_match notification with entity_id=farmer_user_id. The + listing_match template defaults to entity_type="listing", which would + route the buyer to /marketplace/{farmer_user_id} — a listing page lookup + that returns 404 because the value is a user UUID, not a listing slug. + + Fix: The ML gateway passes {"entity_type": "profile"} in the meta dict + of the notify payload. The notification service's deliver() function + applies meta["entity_type"] as an override over the template default. + Result: entity_type=profile stored in DB, routes to /farmers/{user_id}. + + + BUG 6 — Notification deep-link routing on listing tap: "Listing not found" + ───────────────────────────────────────────────────────────────────── + Files: services/produce/app/routers/listings.py, + services/soko-ml/ml-gateway-service/src/main.py, + soko_client_final/src/hooks/useProductDetail.ts + + Root cause: Three compounding issues: + + a) The produce service passed listing_id (UUID) to the ML gateway. + The ML gateway used that UUID as entity_id in listing_match + notifications. The frontend marketplace route calls fetchListingBySlug(), + which hits GET /listings/slug/{slug}. A UUID is not a slug — the + backend returns 404. + + b) The existing stale test notification used a placeholder entity_id + of "test-listing-003" which was never a real listing slug. + + c) Pre-fix notifications for user-registered flows had entity_type=listing + with a user_id as entity_id — doubly wrong (wrong type and wrong value). + + Fix (permanent, forward-looking): + - produce_service now passes listing_slug alongside listing_id when + calling the ML gateway hook + - ML gateway's ListingPublishedPayload added listing_slug: str = "" + - listing_nav_id = payload.listing_slug or payload.listing_id (slug + preferred; UUID fallback for safety) + - All new listing_match notifications store the slug as entity_id + + Fix (stale data in DB): + - Corrected all existing Billy notifications: entity_id=test-listing-003 + updated to "maize-87d1b5e6-67a1" (Fred's real active listing slug) + - entity_type corrected from "listing" to "profile" for the two + user-registered notifications that held user_ids as entity_id + + + BUG 7 — "My Listings" / Farmer profile showing all platform listings + ───────────────────────────────────────────────────────────────────── + File: soko_client_final/src/api/profile.api.ts + + Root cause: fetchFarmerListings() called: + GET /listings?farmer_id={farmerId}&limit=50 + This hits the general marketplace browse endpoint, which accepts no + farmer_id query parameter. The parameter was silently ignored; the + endpoint returned all active listings on the platform (71 listings + from 18+ farmers), not the specific farmer's listings. + + This affected two distinct views: + - Public farmer profile (/farmers/$id) — showed all 71 listings on + every farmer's profile, regardless of who was being viewed + - Own profile dashboard ("My Listings") — showed all 71 listings + to every logged-in farmer, including listings they didn't own + + Fix: Changed the API call to the correct dedicated endpoint: + GET /listings/farmer/{farmerId}?limit=50 + which filters by Listing.farmer_id == uuid.UUID(farmer_id) and + Listing.status == active. + + + BUG 8 — Per-call httpx.AsyncClient (TCP connection overhead) + ───────────────────────────────────────────────────────────────────── + Files: services/notification/app/main.py, + services/user/app/main.py + + Root cause: fetch_user() in the notification service was opening a new + httpx.AsyncClient (and a new TCP connection) for every single user fetch. + On every /notify call involving both a buyer and a farmer, this meant two + fresh connections to the user service. + + Fix: Shared httpx.AsyncClient created in FastAPI lifespan context and + stored on app.state.http_client. All internal HTTP calls reuse this + pooled client. Same pattern applied to user_service for the ML gateway + registration hook. Typical latency reduction: 30–80 ms per notify call. + + + BUG 9 — Sequential health checks in ML gateway startup + ───────────────────────────────────────────────────────────────────── + File: services/soko-ml/ml-gateway-service/src/main.py + + Root cause: The gateway's startup health checks for 4 downstream services + ran one after the other (sequential awaits), adding ~4× the latency of + a single check to every cold-start probe. + + Fix: Wrapped in asyncio.gather() so all 4 checks run concurrently. + + + BUG 10 — Hardcoded user references in diagnostic queries + ───────────────────────────────────────────────────────────────────── + Context: Smoke tests and diagnostic queries were written against specific + named users ("Fred", specific UUIDs) rather than system roles. + + Fix: All queries rewritten to use role/category filters, time windows, + and GROUP BY aggregations. The system at scale serves hundreds of + thousands of users — no query or code path should depend on knowing + a specific user's name or ID. + + +──────────────────────────────────────────────────────────────────────────────── + 7. GOTCHAS & KNOWN CONSTRAINTS +──────────────────────────────────────────────────────────────────────────────── + + GOTCHA 1 — destroy_seed.py must flush Redis before re-seeding + ───────────────────────────────────────────────────────────────────── + The produce service caches farmer listings under: + farmer_listings:{farmer_id}:p{page}:l{limit} + The ML stack caches recommendations under: + rec:buyers:{farmer_id}:{top_n} + rec:farmers:{buyer_id}:{top_n} + If destroy_seed.py runs without flushing Redis, re-seeded users will get + stale cached results from the previous seed run until TTL expiry. + destroy_seed.py now handles both caches. + + + GOTCHA 2 — listing_slug vs listing_id distinction + ───────────────────────────────────────────────────────────────────── + The frontend marketplace route (/marketplace/$id) fetches listings by + SLUG, not UUID. The backend has: + GET /listings/slug/{slug} ← used by the frontend + GET /listings/farmer/{farmer_id} ← used for farmer profile listings + GET /listings/{id} ← does NOT exist (would return 404) + Any code that stores a listing UUID as a navigable entity_id will produce + a "Listing not found" error on the frontend. Always use slug for navigation. + + + GOTCHA 3 — Crop qualifier normalization is read-time only + ───────────────────────────────────────────────────────────────────── + Raw crop values in farmer_features and buyer_features are stored as-is + (e.g., "maize_grain", "Maize", "yellow_beans"). Normalization happens + only when the feature store loads data into DataFrames. If you query the + DB directly, values will not look comparable — that is expected. The + recommender handles the mapping internally. + + + GOTCHA 4 — role=both requires ≥ 3 specialties at registration + ───────────────────────────────────────────────────────────────────── + The auth service enforces a domain rule: a user registering as role=both + (warehouse / aggregator) must declare at least 3 crop specialties. This + is a domain validation constraint, not a bug. Test payloads for "both" + role must include at least 3 specialty values. + + + GOTCHA 5 — No public buyer profile route + ───────────────────────────────────────────────────────────────────── + When a new buyer registers and matching farmers receive a buyer_match + notification, there is intentionally no entity_id set on those notifications. + Tapping the notification marks it as read but does not navigate anywhere. + This is because the platform currently has no /buyers/$id public profile + page. When that route is added, entity_type=profile and entity_id=buyer_id + can be wired up in _notify_farmers_of_new_buyer() in the ML gateway. + + + GOTCHA 6 — Deduplication window applies only to in_app channel + ───────────────────────────────────────────────────────────────────── + The DEDUP_MINUTES=10 deduplication check in deliver() only guards in_app + notifications (checked by channel==in_app). SMS notifications are sent + independently and are not deduplicated — they rely on the caller only + triggering SMS for meaningful events (order_placed, payment_confirmed etc.) + + + GOTCHA 7 — ML gateway INTERNAL_SECRET default in dev + ───────────────────────────────────────────────────────────────────── + The ML gateway logs a warning on startup if INTERNAL_SECRET is still + set to the default value: + "INTERNAL_SECRET is still the default — set it before production" + This is acceptable in development. In production, all services sharing + the soko-ml-bridge network must use a consistent, non-default secret + set via environment variables / secrets manager. + + + GOTCHA 8 — WebSocket URL must be dynamic + ───────────────────────────────────────────────────────────────────── + The frontend notification store previously hardcoded ws://localhost/... + as the WebSocket URL. This broke in any environment other than local + development (staging, mobile testing via LAN IP, production HTTPS). + Fixed to derive protocol and host from window.location: + const wsProtocol = window.location.protocol === "https:" ? "wss:" : "ws:"; + const socket = new WebSocket( + `${wsProtocol}//${window.location.host}/notifications/ws/${user.id}?token=${token}` + ); + + + GOTCHA 9 — produce_service has its own Redis cache independent of ML Redis + ───────────────────────────────────────────────────────────────────── + There are two Redis instances: + - redis (main stack) — produce browse cache, farmer listing cache + - soko-ml-redis (ML stack) — recommendation result cache + destroy_seed.py must target BOTH. The ML redis flush uses redis-cli + --scan pattern rec:* on soko-ml-redis. The produce cache invalidation + happens automatically when listings are deleted/archived via the + invalidate_farmer_listings() call in the produce service. + + + GOTCHA 10 — fetchFarmerListings in profile.api.ts vs listings.api.ts + ───────────────────────────────────────────────────────────────────── + There are two separate API modules for listings: + - src/api/listings.api.ts — fetchFarmerListings() → /listings/farmer/{id} + (used by the public farmer profile page via useFarmerListings hook) + - src/api/profile.api.ts — fetchFarmerListings() — SAME FUNCTION NAME + (used by the dashboard "My Listings" and the public farmer profile page) + These are different functions with the same name in different modules. + The profile.api.ts version previously called the wrong endpoint. Fixed. + Keep these two functions in sync if the endpoint contract changes. + + +──────────────────────────────────────────────────────────────────────────────── + 8. FILES MODIFIED (FULL INVENTORY) +──────────────────────────────────────────────────────────────────────────────── + + Backend — Notification Service + ───────────────────────────────────────────────── + services/notification/app/main.py + Added shared httpx.AsyncClient in FastAPI lifespan (app.state.http_client) + + services/notification/app/routers/internal.py + - fetch_user() accepts shared client parameter (no per-call client) + - deliver() applies entity_type override from meta dict + - deliver() WebSocket push now uses effective_entity_type (not template value) + - DEDUP_MINUTES env var for configurable deduplication window + - _FARMER_EVENTS constant defining which events trigger farmer notifications + - farmer entity_id fixed: payload.order_id or payload.entity_id + - asyncio.gather() for concurrent buyer/farmer profile fetches + + services/notification/app/schemas/notification.py + Added model_validator requiring at least one of buyer_id / farmer_id / + actor_id — prevents silent no-op notify calls + + Backend — User Service + ───────────────────────────────────────────────── + services/user/app/main.py + Added shared httpx.AsyncClient in FastAPI lifespan + + services/user/app/routers/internal.py + - create_user() converted to async def with Request + BackgroundTasks + - Added _notify_ml_user_registered() fire-and-forget helper + - Fires ML registration hook after successful DB commit + + services/user/app/core/config.py + Added ML_GATEWAY_URL: str = "" + + Backend — Produce Service + ───────────────────────────────────────────────── + services/produce/app/routers/listings.py + - _notify_ml_listing_published() now receives and forwards listing_slug + - listing_slug passed in the ML gateway hook payload alongside listing_id + + Backend — ML Gateway + ───────────────────────────────────────────────── + services/soko-ml/ml-gateway-service/src/main.py + - USER_REGISTERED_TOP_N env var (default 3) + - _upsert_farmer_features() refactored to accept crops: list[str] + - _upsert_buyer_features() added (was missing) + - ListingPublishedPayload: added listing_slug: str = "" + - listing_nav_id = slug or UUID used as entity_id in notifications + - UserRegisteredPayload with role validator (farmer/buyer/both only) + - _notify_farmers_of_new_buyer() — sends buyer_match, no entity_id + - _notify_buyers_of_new_farmer() — sends listing_match, entity_type=profile + - /user-registered endpoint handling all three role variants + - asyncio.gather() for startup health checks + - asyncio.gather() for parallel buyer notification dispatch + + ML Stack — Recommendation Service + ───────────────────────────────────────────────── + services/soko-ml/recommendation-service/src/feature_store_client.py + - _CROP_QUALIFIERS tuple + - _normalize_crop() function + - Applied to both farmer specialties and buyer interests on DataFrame load + + Scripts + ───────────────────────────────────────────────── + scripts/destroy_seed.py + - reset_ml_feature_store(): fixed to target farmer_features and + buyer_features (not non-existent user_profiles/interactions tables) + - flush_ml_redis_cache(): scans and deletes all rec:* keys via redis-cli + - destroy_notifications(): removes notification records for seeded users + - main() updated to call all three cleanup functions + + docker-compose.yml (root) + Added ML_GATEWAY_URL: http://ml-gateway:8000 to user_service environment + + Frontend — Client + ───────────────────────────────────────────────── + soko_client_final/src/store/notification-store.ts + Dynamic WebSocket URL derived from window.location (was hardcoded localhost) + + soko_client_final/src/components/notification-page/Item.tsx + - resolveRoute() maps entity_type to frontend route path + - handleClick() calls navigate({ to: route }) on tap + - Destructured entityType and entityId from notification object + + soko_client_final/src/api/profile.api.ts + fetchFarmerListings() corrected from: + GET /listings?farmer_id={id}&limit=50 (browse endpoint, ignores param) + to: + GET /listings/farmer/{id}?limit=50 (dedicated farmer listings endpoint) + + +──────────────────────────────────────────────────────────────────────────────── + 9. EXECUTION NOTES — LATENCY & SCALE +──────────────────────────────────────────────────────────────────────────────── + + All notification dispatches are fire-and-forget (BackgroundTask from produce + and user services). Listing creation and user registration return immediately; + ML processing and notification dispatch happen asynchronously behind the + response. This means zero added latency to the user-facing registration or + listing publish flow. + + Within the notification service itself, user profile fetches and SMS dispatch + run concurrently via asyncio.gather(). The shared httpx client reuses + connection pools, eliminating TCP handshake overhead on every call. + + Buyer notification dispatch (when a farmer publishes to N matching buyers) + runs as asyncio.gather(*[_notify_buyer(b) for b in buyers]) — all N POSTs + to the notification service are in flight simultaneously. At scale, this + bounds the dispatch time to the latency of one POST, not N × one POST. + + The recommendation cache means repeated queries for the same farmer/buyer + pair are served from Redis (sub-millisecond) rather than re-computing the + cosine similarity matrix. Cache is invalidated only when the underlying + feature data changes (new listing, new user). + + +──────────────────────────────────────────────────────────────────────────────── + 10. WHAT HAPPENS END-TO-END (PRODUCTION SCENARIO) +──────────────────────────────────────────────────────────────────────────────── + + New Farmer Registers + ───────────────────────────────────────────────── + 1. Farmer submits registration form → auth_service creates credentials + 2. auth_service calls user_service/internal/users → profile created in DB + 3. user_service fires BackgroundTask → POST /user-registered to ML gateway + 4. ML gateway upserts farmer into farmer_features + 5. ML gateway reloads recommendation cache + 6. ML gateway fetches top N matching buyers + 7. ML gateway POSTs listing_match to notification_service for each buyer + 8. notification_service writes in_app record, pushes over WebSocket, + optionally sends SMS + 9. Each matching buyer sees a real-time notification: "Kato Emmanuel from + Kampala offers maize and has just joined Soko. Tap to view their profile." + 10. Buyer taps → navigates to /farmers/{kato_user_id} ✓ + + Farmer Publishes a Listing + ───────────────────────────────────────────────── + 1. Farmer publishes listing → produce_service activates it in DB + 2. produce_service fires BackgroundTask → POST /listing-published to ML gateway + 3. ML gateway upserts updated farmer features (crop, total_listings) + 4. ML gateway reloads recommendation cache + 5. ML gateway fetches top N matching buyers + 6. ML gateway POSTs buyer_match to notification_service for the farmer + (entity_id = listing slug → /marketplace/{slug}) + 7. ML gateway POSTs listing_match to notification_service for each buyer + (entity_id = listing slug → /marketplace/{slug}) + 8. Buyer taps → navigates to /marketplace/maize-87d1b5e6-67a1 → listing + loads correctly via GET /listings/slug/maize-87d1b5e6-67a1 ✓ + 9. Farmer sees: "Billy Ogema in your area wants your Maize. Tap to connect." + + +──────────────────────────────────────────────────────────────────────────────── + 11. PENDING / FUTURE WORK +──────────────────────────────────────────────────────────────────────────────── + + - Public buyer profile page (/buyers/$id): Once this route exists, wire + entity_type=profile + entity_id=buyer_id into buyer_match notifications + from _notify_farmers_of_new_buyer() so farmers can tap through to + the buyer's profile. + + - /listings/id/{uuid} endpoint: The comment in useProductDetail.ts notes + this as a TODO. If listings are ever referenced by UUID in any other + context, add GET /listings/id/{uuid} to the produce service to avoid + requiring callers to know the slug. + + - INTERNAL_SECRET rotation: Before any production deployment, replace the + default INTERNAL_SECRET across all services with a generated secret stored + in a secrets manager. + + - Recommendation quality tuning: The current cosine similarity model uses + crop overlap as the primary signal. District proximity and purchase history + (once sufficient order data exists) should be weighted in as secondary + signals to improve match quality at scale. + + - Notification preferences: Users currently receive all notification types. + UserSettings (already in the schema) should gate which channels and event + types each user receives. + +================================================================================ + END OF REPORT +================================================================================ From 4c046bf2ea8782669ff6fcaa396c755c5d7fc87d Mon Sep 17 00:00:00 2001 From: the-icemann Date: Mon, 18 May 2026 15:51:56 +0300 Subject: [PATCH 06/24] Renderred all ML price compositions basing on user roles into the front-end --- .github/workflows/ci.yml | 36 +- .github/workflows/deploy.yml | 62 +++ .gitignore | 7 + DEPLOY.md | 414 ++++++++++++++++++ docker-compose.yml | 37 +- infrastructure/cloudfront.tf | 87 ++++ infrastructure/ec2.tf | 147 +++++++ infrastructure/main.tf | 32 ++ infrastructure/outputs.tf | 44 ++ infrastructure/s3.tf | 74 ++++ infrastructure/secrets.tf | 79 ++++ infrastructure/terraform.tfvars.example | 11 + infrastructure/user_data.sh | 65 +++ infrastructure/variables.tf | 45 ++ infrastructure/vpc.tf | 37 ++ nginx/nginx.conf | 22 +- scripts/bootstrap-tf-state.sh | 46 ++ scripts/fetch-ml-secrets.sh | 31 ++ scripts/fetch-secrets.sh | 115 +++++ services/auth/app/schemas/auth.py | 12 +- services/message/app/routers/messages.py | 4 +- .../data-ingestion-service/src/main.py | 143 +++++- .../data-ingestion-service/src/schemas.py | 12 +- services/soko-ml/docker-compose.yml | 1 + services/user/app/core/config.py | 5 +- services/user/app/helpers/builders.py | 4 + services/user/app/routers/internal.py | 113 ++++- services/user/app/routers/profile.py | 33 +- services/user/app/schemas/schemas.py | 14 + 29 files changed, 1679 insertions(+), 53 deletions(-) create mode 100644 .github/workflows/deploy.yml create mode 100644 DEPLOY.md create mode 100644 infrastructure/cloudfront.tf create mode 100644 infrastructure/ec2.tf create mode 100644 infrastructure/main.tf create mode 100644 infrastructure/outputs.tf create mode 100644 infrastructure/s3.tf create mode 100644 infrastructure/secrets.tf create mode 100644 infrastructure/terraform.tfvars.example create mode 100644 infrastructure/user_data.sh create mode 100644 infrastructure/variables.tf create mode 100644 infrastructure/vpc.tf create mode 100755 scripts/bootstrap-tf-state.sh create mode 100755 scripts/fetch-ml-secrets.sh create mode 100755 scripts/fetch-secrets.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad9b40b..d57b6fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,18 +15,23 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: "3.12" - name: Install ruff run: pip install ruff - name: Lint all services run: | - ruff check services/auth/app \ - services/farmer/app \ - services/buyer/app \ - services/produce/app \ - services/recommendation/app + ruff check \ + services/auth/app \ + services/user/app \ + services/produce/app \ + services/order/app \ + services/payment/app \ + services/notification/app \ + services/message/app \ + services/blog/app \ + services/ussd/app integration-tests: name: Integration Tests @@ -36,25 +41,28 @@ jobs: steps: - uses: actions/checkout@v4 + # CI uses dev defaults — no real secrets needed + - name: Create bridge network for ML stack + run: docker network create soko-ml-bridge || true + - name: Build and start services run: docker compose up --build -d - timeout-minutes: 10 + timeout-minutes: 15 - - name: Wait for all services to be healthy + - name: Wait for all core services to be healthy run: | - echo "Waiting for all 5 services to respond..." - for port in 8001 8002 8003 8004 8005; do + echo "Waiting for core services..." + for port in 8001 8002 8003 8004 8005 8007 8008 8009; do for i in $(seq 1 40); do if curl -sf http://localhost:${port}/health > /dev/null 2>&1; then - echo "Port ${port} is up." + echo " :${port} up" break fi if [ "$i" = "40" ]; then - echo "Timed out waiting for port ${port}" + echo " :${port} timed out" docker compose logs exit 1 fi - echo "Port ${port} not ready, attempt $i — waiting 5s..." sleep 5 done done @@ -62,7 +70,7 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: "3.12" - name: Install test dependencies run: pip install -r tests/integration/requirements.txt diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..fcf0deb --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,62 @@ +name: Deploy to Production + +on: + push: + branches: [main] + workflow_dispatch: # allow manual trigger from GitHub UI + +concurrency: + group: deploy-prod + cancel-in-progress: false # never cancel a running deploy + +jobs: + deploy: + name: Deploy Backend + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Deploy via SSH + uses: appleboy/ssh-action@v1.2.0 + with: + host: ${{ secrets.EC2_HOST }} + username: ubuntu + key: ${{ secrets.EC2_SSH_KEY }} + timeout: 600s + script: | + set -e + + echo "=== Pulling latest code ===" + cd /opt/soko + git fetch origin main + git reset --hard origin/main + + echo "=== Refreshing secrets ===" + bash scripts/fetch-secrets.sh + bash scripts/fetch-ml-secrets.sh + + echo "=== Restarting core platform ===" + docker compose pull --ignore-pull-failures 2>/dev/null || true + docker compose up -d --build --remove-orphans + + echo "=== Restarting ML stack ===" + cd services/soko-ml + docker compose up -d --build --remove-orphans + + echo "=== Pruning unused images ===" + docker image prune -f + + echo "=== Verifying health ===" + sleep 15 + for port in 8001 8002 8003 8004 8005 8007 8008 8009; do + status=$(curl -sf --max-time 5 http://localhost:${port}/health | jq -r '.status' 2>/dev/null || echo "unreachable") + echo "Service :${port} -> ${status}" + done + + - name: Notify on failure + if: failure() + run: | + echo "::error::Deployment failed — check SSH logs above" + # Add Slack/email notification here if desired diff --git a/.gitignore b/.gitignore index ea82f31..abe8e6a 100644 --- a/.gitignore +++ b/.gitignore @@ -94,3 +94,10 @@ erl_crash.dump # Claude Code .claude/ + +# Terraform +infrastructure/.terraform/ +infrastructure/terraform.tfvars +infrastructure/*.tfstate +infrastructure/*.tfstate.backup +infrastructure/.terraform.lock.hcl diff --git a/DEPLOY.md b/DEPLOY.md new file mode 100644 index 0000000..f249751 --- /dev/null +++ b/DEPLOY.md @@ -0,0 +1,414 @@ +# Soko — Production Deployment Guide (Option A) + +Single EC2 in af-south-1 (Cape Town) + S3 + CloudFront. +Estimated monthly cost: ~$90–130 USD. + +--- + +## Overview — what gets created + +``` + Users (Uganda) + | + CloudFront (South Africa edge) + / \ + React PWA (S3) NGINX :80/:443 (EC2) + | + ┌──────────────┴──────────────┐ + │ EC2 t3.xlarge (af-south-1) │ + │ │ + │ Docker Compose (core): │ + │ auth, user, produce, │ + │ order, payment, message, │ + │ notification, blog, ussd │ + │ + 9x Postgres + Redis │ + │ │ + │ Docker Compose (ML): │ + │ ml-gateway, price, rec, │ + │ location, ingest, kafka │ + │ + Postgres + Redis │ + └──────────────────────────────┘ + | + AWS Secrets Manager + AWS S3 (produce images) +``` + +--- + +## Prerequisites + +Before starting, you need: +- [ ] AWS account with billing set up +- [ ] AWS CLI installed locally: `brew install awscli` or https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html +- [ ] Terraform installed: `brew install terraform` or https://developer.hashicorp.com/terraform/install +- [ ] A domain name (recommended; can skip for initial testing) +- [ ] SSH key pair — you'll create one in Step 2 + +Configure AWS CLI with your credentials: +```bash +aws configure +# AWS Access Key ID: +# AWS Secret Access Key: +# Default region: af-south-1 +# Default output format: json +``` + +--- + +## Step 1 — Create an EC2 Key Pair + +This is the SSH key that lets you (and GitHub Actions) connect to the server. + +```bash +# Create key pair and save the .pem file +aws ec2 create-key-pair \ + --key-name soko-prod-key \ + --region af-south-1 \ + --query "KeyMaterial" \ + --output text > ~/.ssh/soko-prod-key.pem + +chmod 400 ~/.ssh/soko-prod-key.pem +``` + +--- + +## Step 2 — Bootstrap Terraform State Storage + +Terraform needs an S3 bucket to store its state file. Run this ONCE: + +```bash +chmod +x scripts/bootstrap-tf-state.sh +./scripts/bootstrap-tf-state.sh +``` + +This creates: +- S3 bucket `soko-terraform-state` (versioned, encrypted) +- DynamoDB table `soko-terraform-locks` (prevents concurrent applies) + +--- + +## Step 3 — Configure Terraform Variables + +```bash +cd infrastructure +cp terraform.tfvars.example terraform.tfvars +``` + +Edit `terraform.tfvars`: +```hcl +aws_region = "af-south-1" +environment = "prod" +instance_type = "t3.xlarge" +ec2_key_name = "soko-prod-key" +allowed_ssh_cidr = "YOUR_HOME_IP/32" # get it: curl ifconfig.me +domain_name = "yourdomain.com" # or "" if you don't have one yet +alert_email = "andrewssuubi@gmail.com" +github_repo = "the-icemann/soko" +``` + +> `terraform.tfvars` is gitignored — it never gets committed. + +--- + +## Step 4 — Provision Infrastructure + +```bash +cd infrastructure +terraform init +terraform plan # review what will be created +terraform apply # type 'yes' when prompted +``` + +Takes ~3 minutes. At the end, copy the outputs: +``` +ec2_public_ip = "x.x.x.x" ← point your domain A record here +cloudfront_domain = "xxxx.cloudfront.net" +cloudfront_distribution_id = "EXXXXXXXXX" +frontend_bucket_name = "soko-frontend-prod" +images_bucket_name = "soko-produce-images-prod" +ssh_command = "ssh -i ~/.ssh/soko-prod-key.pem ubuntu@x.x.x.x" +``` + +--- + +## Step 5 — Fill In Real Secrets + +Terraform created skeleton secrets in AWS Secrets Manager with `CHANGE_ME` placeholders. +Now fill in the real values. + +### Option A — AWS Console (easiest) +1. Open https://console.aws.amazon.com/secretsmanager +2. Click **soko/platform** → **Retrieve secret value** → **Edit** +3. Replace each `CHANGE_ME` with the real value +4. Click **Save** +5. Repeat for **soko/ml** + +### Option B — AWS CLI +```bash +aws secretsmanager put-secret-value \ + --secret-id soko/platform \ + --region af-south-1 \ + --secret-string '{ + "AUTH_DB_PASS": "your-strong-password-here", + "USER_DB_PASS": "your-strong-password-here", + "PRODUCE_DB_PASS": "your-strong-password-here", + "ORDER_DB_PASS": "your-strong-password-here", + "PAYMENT_DB_PASS": "your-strong-password-here", + "MESSAGE_DB_PASS": "your-strong-password-here", + "NOTIFICATION_DB_PASS": "your-strong-password-here", + "BLOG_DB_PASS": "your-strong-password-here", + "USSD_DB_PASS": "your-strong-password-here", + "SECRET_KEY": "a-64-char-random-string", + "INTERNAL_SECRET": "another-random-string", + "ALGORITHM": "HS256", + "GOOGLE_CLIENT_ID": "your-google-client-id", + "GOOGLE_CLIENT_SECRET": "your-google-client-secret", + "PESAPAL_CONSUMER_KEY": "your-pesapal-key", + "PESAPAL_CONSUMER_SECRET": "your-pesapal-secret", + "PESAPAL_ENV": "production", + "AT_USERNAME": "your-africas-talking-username", + "AT_API_KEY": "your-at-api-key", + "AT_SENDER_ID": "SOKO", + "SENDGRID_API_KEY": "SG.xxxxx", + "SENDGRID_FROM_EMAIL": "noreply@yourdomain.com", + "DOMAIN": "yourdomain.com" + }' + +aws secretsmanager put-secret-value \ + --secret-id soko/ml \ + --region af-south-1 \ + --secret-string '{ + "ML_DB_PASS": "your-strong-password-here", + "ML_REDIS_PASSWORD": "", + "INTERNAL_API_KEY": "another-random-string", + "GOOGLE_MAPS_API_KEY": "your-maps-api-key-or-empty" + }' +``` + +> Generate strong passwords: `openssl rand -base64 32` +> Generate a SECRET_KEY: `openssl rand -hex 32` + +--- + +## Step 6 — Add Secrets to GitHub + +You need to add secrets to **both** GitHub repos. + +### Backend repo (github.com/the-icemann/soko) + +Go to: **Settings → Secrets and variables → Actions → New repository secret** + +| Secret name | Value | +|---|---| +| `EC2_HOST` | The EC2 public IP from Terraform output | +| `EC2_SSH_KEY` | Full contents of `~/.ssh/soko-prod-key.pem` | +| `AWS_ACCESS_KEY_ID` | Your AWS access key (create a deploy-only IAM user) | +| `AWS_SECRET_ACCESS_KEY` | Your AWS secret key | + +### Frontend repo (github.com/the-icemann/soko_client_final) + +Same **Secrets** tab: + +| Secret name | Value | +|---|---| +| `AWS_ACCESS_KEY_ID` | Same AWS key | +| `AWS_SECRET_ACCESS_KEY` | Same AWS secret | + +Then go to **Settings → Secrets and variables → Variables → New repository variable**: + +| Variable name | Value | +|---|---| +| `VITE_API_BASE_URL` | `https://yourdomain.com` or `http://x.x.x.x` | +| `FRONTEND_BUCKET` | `soko-frontend-prod` (from Terraform output) | +| `CF_DISTRIBUTION_ID` | The CloudFront distribution ID from Terraform output | + +> **Security tip**: Create a separate IAM user for GitHub Actions with only `s3:*` on the frontend bucket and `cloudfront:CreateInvalidation`. Keep your personal admin key separate. + +--- + +## Step 7 — First Server Setup + +SSH into the EC2 instance. The user_data script ran automatically on boot and: +- Installed Docker, git, AWS CLI +- Cloned the repo to `/opt/soko` +- Fetched secrets and started all services + +Check status: +```bash +ssh -i ~/.ssh/soko-prod-key.pem ubuntu@YOUR_EC2_IP + +# Check core services +cd /opt/soko +docker compose ps + +# Check ML stack +cd services/soko-ml +docker compose ps + +# Check logs +docker compose logs --tail=50 auth_service +docker compose logs --tail=50 soko-ml-gateway +``` + +If the user_data script didn't run completely (can take 5-10 min on first boot): +```bash +# Run manually +cd /opt/soko +chmod +x scripts/fetch-secrets.sh scripts/fetch-ml-secrets.sh +bash scripts/fetch-secrets.sh +docker network create soko-ml-bridge 2>/dev/null || true +docker compose up -d --build + +cd services/soko-ml +bash /opt/soko/scripts/fetch-ml-secrets.sh +docker compose up -d --build +``` + +--- + +## Step 8 — SSL with Let's Encrypt + +Once your domain's A record points to the EC2 Elastic IP: + +```bash +ssh -i ~/.ssh/soko-prod-key.pem ubuntu@YOUR_EC2_IP + +# Install certbot +sudo apt-get install -y certbot python3-certbot-nginx + +# Get certificate (NGINX must be running on port 80) +sudo certbot --nginx -d yourdomain.com -d www.yourdomain.com \ + --non-interactive --agree-tos -m andrewssuubi@gmail.com + +# Certbot auto-renews via systemd timer — verify: +sudo systemctl status certbot.timer +``` + +Then update the PesaPal callback URLs in Secrets Manager to use `https://`. + +--- + +## Step 9 — Deploy Frontend (First Time) + +Push to main in the frontend repo to trigger the GitHub Actions workflow, or run manually: + +```bash +cd /home/the-icemann/Desktop/soko_client_final + +# Set the API URL for local build test +VITE_API_BASE_URL=https://yourdomain.com npm run build + +# Deploy manually (AWS CLI must be configured) +aws s3 sync dist/assets/ s3://soko-frontend-prod/assets/ \ + --cache-control "public, max-age=31536000, immutable" --delete + +aws s3 sync dist/ s3://soko-frontend-prod/ \ + --exclude "assets/*" \ + --cache-control "public, max-age=0, must-revalidate" --delete + +aws cloudfront create-invalidation \ + --distribution-id YOUR_CF_DISTRIBUTION_ID \ + --paths "/*" +``` + +Frontend is now live at the CloudFront domain (or your custom subdomain if configured). + +--- + +## Day-to-Day Operations + +### Deploying changes +```bash +# Backend: just push to main — GitHub Actions SSHes in and redeploys +git push origin main + +# Frontend: just push to main in the client repo +cd /path/to/soko_client_final && git push origin main +``` + +### Viewing logs +```bash +ssh -i ~/.ssh/soko-prod-key.pem ubuntu@YOUR_EC2_IP + +# All core services +docker compose -f /opt/soko/docker-compose.yml logs -f + +# Specific service +docker compose -f /opt/soko/docker-compose.yml logs -f auth_service + +# ML stack +docker compose -f /opt/soko/services/soko-ml/docker-compose.yml logs -f +``` + +### Updating secrets +```bash +# Update in Secrets Manager (console or CLI), then: +ssh -i ~/.ssh/soko-prod-key.pem ubuntu@YOUR_EC2_IP +cd /opt/soko +bash scripts/fetch-secrets.sh +docker compose up -d # restarts with new env +``` + +### Full restart +```bash +cd /opt/soko +docker compose down && docker compose up -d --build +``` + +### Database backups +```bash +# Backup all databases (run on EC2 or via ssh) +for db in auth user produce order payment message notification blog ussd; do + docker exec ${db}_db pg_dump -U ${db}_user ${db}_db \ + | gzip > /opt/backups/${db}_$(date +%Y%m%d).sql.gz +done +# Upload to S3 +aws s3 cp /opt/backups/ s3://soko-produce-images-prod/backups/ --recursive +``` + +--- + +## Secrets Flow Diagram + +``` +AWS Secrets Manager + soko/platform ←── You fill in real values (Step 5) + soko/ml ←── You fill in real values (Step 5) + | + | fetch-secrets.sh (runs on EC2 at deploy time) + ↓ + /opt/soko/.env ← Docker Compose reads for ${VAR} substitution + services/auth/.env ← auth_service reads via env_file + services/payment/.env ← payment_service reads via env_file + services/notification/.env + services/ussd/.env + services/message/.env + services/soko-ml/.env ← ML docker-compose reads + | + ↓ + Docker containers get secrets as environment variables + Postgres containers get strong passwords (not the dev defaults) + Services get real API keys at runtime + +GitHub Actions Secrets + EC2_SSH_KEY + EC2_HOST → deploy.yml SSHes in to run git pull + docker compose + AWS_ACCESS_KEY_ID/SECRET → frontend deploy.yml syncs to S3 + invalidates CloudFront + +Frontend (browser) + VITE_API_BASE_URL → baked into the JS bundle at build time (not secret) + Set as GitHub Actions variable (not secret) +``` + +--- + +## Troubleshooting + +| Symptom | Fix | +|---|---| +| `docker compose up` fails with env var errors | Run `bash scripts/fetch-secrets.sh` first | +| Services can't reach ML gateway | Run `docker network create soko-ml-bridge` | +| NGINX 503 on prices page | Check ML stack: `cd services/soko-ml && docker compose ps` | +| SSL certificate not renewing | `sudo certbot renew --dry-run` to test; check `/var/log/letsencrypt` | +| EC2 out of disk | `docker system prune -a` removes unused images | +| GitHub Actions deploy fails | Check `EC2_SSH_KEY` — must be the full PEM contents including headers | +| CloudFront shows old version | Trigger invalidation: `aws cloudfront create-invalidation --distribution-id ID --paths "/*"` | diff --git a/docker-compose.yml b/docker-compose.yml index e7e5f9b..c54857f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -69,7 +69,7 @@ services: container_name: auth_db environment: POSTGRES_USER: auth_user - POSTGRES_PASSWORD: auth_pass + POSTGRES_PASSWORD: ${AUTH_DB_PASS:-auth_pass} POSTGRES_DB: auth_db volumes: - auth_db_data:/var/lib/postgresql/data @@ -88,7 +88,7 @@ services: - "127.0.0.1:8001:8001" env_file: ./services/auth/.env environment: - DATABASE_URL: postgresql://auth_user:auth_pass@auth_db:5432/auth_db + DATABASE_URL: postgresql://auth_user:${AUTH_DB_PASS:-auth_pass}@auth_db:5432/auth_db USER_SERVICE_URL: http://user_service:8002 depends_on: auth_db: @@ -105,7 +105,7 @@ services: container_name: user_db environment: POSTGRES_USER: user_user - POSTGRES_PASSWORD: user_pass + POSTGRES_PASSWORD: ${USER_DB_PASS:-user_pass} POSTGRES_DB: user_db volumes: - user_db_data:/var/lib/postgresql/data @@ -124,7 +124,7 @@ services: - "127.0.0.1:8002:8002" env_file: ./services/user/.env environment: - DATABASE_URL: postgresql://user_user:user_pass@user_db:5432/user_db + DATABASE_URL: postgresql://user_user:${USER_DB_PASS:-user_pass}@user_db:5432/user_db depends_on: user_db: condition: service_healthy @@ -141,7 +141,7 @@ services: container_name: produce_db environment: POSTGRES_USER: produce_user - POSTGRES_PASSWORD: produce_pass + POSTGRES_PASSWORD: ${PRODUCE_DB_PASS:-produce_pass} POSTGRES_DB: produce_db volumes: - produce_db_data:/var/lib/postgresql/data @@ -160,7 +160,7 @@ services: - "127.0.0.1:8003:8003" env_file: ./services/produce/.env environment: - DATABASE_URL: postgresql://produce_user:produce_pass@produce_db:5432/produce_db + DATABASE_URL: postgresql://produce_user:${PRODUCE_DB_PASS:-produce_pass}@produce_db:5432/produce_db REDIS_URL: redis://redis:6379/0 USER_SERVICE_URL: http://user_service:8002 ML_GATEWAY_URL: http://ml-gateway:8000 @@ -182,7 +182,7 @@ services: container_name: order_db environment: POSTGRES_USER: order_user - POSTGRES_PASSWORD: order_pass + POSTGRES_PASSWORD: ${ORDER_DB_PASS:-order_pass} POSTGRES_DB: order_db volumes: - order_db_data:/var/lib/postgresql/data @@ -201,7 +201,7 @@ services: - "127.0.0.1:8004:8004" env_file: ./services/order/.env environment: - DATABASE_URL: postgresql://order_user:order_pass@order_db:5432/order_db + DATABASE_URL: postgresql://order_user:${ORDER_DB_PASS:-order_pass}@order_db:5432/order_db PRODUCE_SERVICE_URL: http://produce_service:8003 USER_SERVICE_URL: http://user_service:8002 PAYMENT_SERVICE_URL: http://payment_service:8005 @@ -228,7 +228,7 @@ services: container_name: payment_db environment: POSTGRES_USER: payment_user - POSTGRES_PASSWORD: payment_pass + POSTGRES_PASSWORD: ${PAYMENT_DB_PASS:-payment_pass} POSTGRES_DB: payment_db volumes: - payment_db_data:/var/lib/postgresql/data @@ -247,7 +247,7 @@ services: - "127.0.0.1:8005:8005" env_file: ./services/payment/.env environment: - DATABASE_URL: postgresql://payment_user:payment_pass@payment_db:5432/payment_db + DATABASE_URL: postgresql://payment_user:${PAYMENT_DB_PASS:-payment_pass}@payment_db:5432/payment_db ORDER_SERVICE_URL: http://order_service:8004 USER_SERVICE_URL: http://user_service:8002 NOTIFICATION_SERVICE_URL: http://notification_service:8007 @@ -266,7 +266,7 @@ services: container_name: message_db environment: POSTGRES_USER: message_user - POSTGRES_PASSWORD: message_pass + POSTGRES_PASSWORD: ${MESSAGE_DB_PASS:-message_pass} POSTGRES_DB: message_db volumes: - message_db_data:/var/lib/postgresql/data @@ -285,7 +285,7 @@ services: - "127.0.0.1:8006:8006" env_file: ./services/message/.env environment: - DATABASE_URL: postgresql://message_user:message_pass@message_db:5432/message_db + DATABASE_URL: postgresql://message_user:${MESSAGE_DB_PASS:-message_pass}@message_db:5432/message_db USER_SERVICE_URL: http://user_service:8002 PRODUCE_SERVICE_URL: http://produce_service:8003 NOTIFICATION_SERVICE_URL: http://notification_service:8007 @@ -304,7 +304,7 @@ services: container_name: notification_db environment: POSTGRES_USER: notification_user - POSTGRES_PASSWORD: notification_pass + POSTGRES_PASSWORD: ${NOTIFICATION_DB_PASS:-notification_pass} POSTGRES_DB: notification_db volumes: - notification_db_data:/var/lib/postgresql/data @@ -323,13 +323,14 @@ services: - "127.0.0.1:8007:8007" env_file: ./services/notification/.env environment: - DATABASE_URL: postgresql://notification_user:notification_pass@notification_db:5432/notification_db + DATABASE_URL: postgresql://notification_user:${NOTIFICATION_DB_PASS:-notification_pass}@notification_db:5432/notification_db USER_SERVICE_URL: http://user_service:8002 depends_on: notification_db: condition: service_healthy networks: - soko_net + - soko-ml-bridge restart: on-failure # ───────────────────────────────────────── @@ -340,7 +341,7 @@ services: container_name: blog_db environment: POSTGRES_USER: blog_user - POSTGRES_PASSWORD: blog_pass + POSTGRES_PASSWORD: ${BLOG_DB_PASS:-blog_pass} POSTGRES_DB: blog_db volumes: - blog_db_data:/var/lib/postgresql/data @@ -359,7 +360,7 @@ services: - "127.0.0.1:8008:8008" env_file: ./services/blog/.env environment: - DATABASE_URL: postgresql://blog_user:blog_pass@blog_db:5432/blog_db + DATABASE_URL: postgresql://blog_user:${BLOG_DB_PASS:-blog_pass}@blog_db:5432/blog_db REDIS_URL: redis://redis:6379/1 USER_SERVICE_URL: http://user_service:8002 depends_on: @@ -379,7 +380,7 @@ services: container_name: ussd_db environment: POSTGRES_USER: ussd_user - POSTGRES_PASSWORD: ussd_pass + POSTGRES_PASSWORD: ${USSD_DB_PASS:-ussd_pass} POSTGRES_DB: ussd_db volumes: - ussd_db_data:/var/lib/postgresql/data @@ -398,7 +399,7 @@ services: - "127.0.0.1:8009:8009" env_file: ./services/ussd/.env environment: - DATABASE_URL: postgresql://ussd_user:ussd_pass@ussd_db:5432/ussd_db + DATABASE_URL: postgresql://ussd_user:${USSD_DB_PASS:-ussd_pass}@ussd_db:5432/ussd_db PRODUCE_SERVICE_URL: http://produce_service:8003 ORDER_SERVICE_URL: http://order_service:8004 AUTH_SERVICE_URL: http://auth_service:8001 diff --git a/infrastructure/cloudfront.tf b/infrastructure/cloudfront.tf new file mode 100644 index 0000000..96cfcdb --- /dev/null +++ b/infrastructure/cloudfront.tf @@ -0,0 +1,87 @@ +# Origin Access Control — lets CloudFront fetch from private S3 +resource "aws_cloudfront_origin_access_control" "frontend" { + name = "soko-frontend-oac" + origin_access_control_origin_type = "s3" + signing_behavior = "always" + signing_protocol = "sigv4" +} + +resource "aws_cloudfront_distribution" "frontend" { + enabled = true + is_ipv6_enabled = true + default_root_object = "index.html" + price_class = "PriceClass_200" # includes South Africa edge nodes + + aliases = var.domain_name != "" ? ["app.${var.domain_name}"] : [] + + origin { + domain_name = aws_s3_bucket.frontend.bucket_regional_domain_name + origin_id = "s3-frontend" + origin_access_control_id = aws_cloudfront_origin_access_control.frontend.id + } + + default_cache_behavior { + target_origin_id = "s3-frontend" + viewer_protocol_policy = "redirect-to-https" + allowed_methods = ["GET", "HEAD"] + cached_methods = ["GET", "HEAD"] + compress = true + + forwarded_values { + query_string = false + cookies { forward = "none" } + } + + min_ttl = 0 + default_ttl = 86400 # 1 day for HTML + max_ttl = 31536000 + } + + # Cache JS/CSS/images aggressively (content-hashed filenames by Vite) + ordered_cache_behavior { + path_pattern = "/assets/*" + target_origin_id = "s3-frontend" + viewer_protocol_policy = "redirect-to-https" + allowed_methods = ["GET", "HEAD"] + cached_methods = ["GET", "HEAD"] + compress = true + + forwarded_values { + query_string = false + cookies { forward = "none" } + } + + min_ttl = 0 + default_ttl = 31536000 # 1 year — Vite hashes filenames on every build + max_ttl = 31536000 + } + + # SPA routing: 404 from S3 → serve index.html so React Router handles it + custom_error_response { + error_code = 404 + response_code = 200 + response_page_path = "/index.html" + error_caching_min_ttl = 10 + } + + custom_error_response { + error_code = 403 + response_code = 200 + response_page_path = "/index.html" + error_caching_min_ttl = 10 + } + + restrictions { + geo_restriction { restriction_type = "none" } + } + + viewer_certificate { + # If you have an ACM certificate for your domain, replace this with: + # acm_certificate_arn = aws_acm_certificate.frontend.arn + # ssl_support_method = "sni-only" + # minimum_protocol_version = "TLSv1.2_2021" + cloudfront_default_certificate = true + } + + tags = { Name = "soko-frontend-cdn" } +} diff --git a/infrastructure/ec2.tf b/infrastructure/ec2.tf new file mode 100644 index 0000000..c45b2b9 --- /dev/null +++ b/infrastructure/ec2.tf @@ -0,0 +1,147 @@ +# ── Latest Ubuntu 22.04 LTS AMI (auto-resolves per region) ─────────────────── +data "aws_ami" "ubuntu" { + most_recent = true + owners = ["099720109477"] # Canonical + + filter { + name = "name" + values = ["ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-*"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } +} + +# ── Security Group ──────────────────────────────────────────────────────────── +resource "aws_security_group" "soko" { + name = "soko-sg" + description = "Soko platform security group" + vpc_id = aws_vpc.soko.id + + ingress { + description = "HTTP" + from_port = 80 + to_port = 80 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + ingress { + description = "HTTPS" + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + ingress { + description = "SSH" + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = [var.allowed_ssh_cidr] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { Name = "soko-sg" } +} + +# ── IAM Role for EC2 (allows Secrets Manager + S3 access) ──────────────────── +resource "aws_iam_role" "ec2" { + name = "soko-ec2-role" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Action = "sts:AssumeRole" + Effect = "Allow" + Principal = { Service = "ec2.amazonaws.com" } + }] + }) +} + +resource "aws_iam_role_policy" "ec2_secrets" { + name = "soko-ec2-secrets-policy" + role = aws_iam_role.ec2.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = ["secretsmanager:GetSecretValue", "secretsmanager:DescribeSecret"] + Resource = [ + aws_secretsmanager_secret.platform.arn, + aws_secretsmanager_secret.ml.arn, + ] + }, + { + Effect = "Allow" + Action = [ + "s3:PutObject", + "s3:GetObject", + "s3:DeleteObject", + "s3:ListBucket", + ] + Resource = [ + aws_s3_bucket.images.arn, + "${aws_s3_bucket.images.arn}/*", + ] + }, + { + # Allow CloudWatch Logs for container monitoring + Effect = "Allow" + Action = ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"] + Resource = "arn:aws:logs:*:*:*" + } + ] + }) +} + +resource "aws_iam_instance_profile" "ec2" { + name = "soko-ec2-profile" + role = aws_iam_role.ec2.name +} + +# ── EC2 Instance ────────────────────────────────────────────────────────────── +resource "aws_instance" "soko" { + ami = data.aws_ami.ubuntu.id + instance_type = var.instance_type + subnet_id = aws_subnet.public.id + vpc_security_group_ids = [aws_security_group.soko.id] + key_name = var.ec2_key_name + iam_instance_profile = aws_iam_instance_profile.ec2.name + + root_block_device { + volume_size = 40 # GB — Docker images + Postgres volumes + ML models + volume_type = "gp3" + encrypted = true + delete_on_termination = false # retain data on accidental instance stop + } + + user_data = base64encode(templatefile("${path.module}/user_data.sh", { + aws_region = var.aws_region + })) + + tags = { Name = "soko-server" } + + lifecycle { + # Prevent accidental replacement if AMI is updated + ignore_changes = [ami, user_data] + } +} + +# ── Elastic IP (stable DNS target even after stop/start) ───────────────────── +resource "aws_eip" "soko" { + instance = aws_instance.soko.id + domain = "vpc" + tags = { Name = "soko-eip" } +} diff --git a/infrastructure/main.tf b/infrastructure/main.tf new file mode 100644 index 0000000..d0ac0c4 --- /dev/null +++ b/infrastructure/main.tf @@ -0,0 +1,32 @@ +terraform { + required_version = ">= 1.6" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } + + # Remote state — S3 bucket must exist before running `terraform init` + # Run `scripts/bootstrap-tf-state.sh` once to create it. + backend "s3" { + bucket = "soko-terraform-state" + key = "prod/terraform.tfstate" + region = "af-south-1" + dynamodb_table = "soko-terraform-locks" + encrypt = true + } +} + +provider "aws" { + region = var.aws_region + + default_tags { + tags = { + Project = "soko" + Environment = var.environment + ManagedBy = "terraform" + } + } +} diff --git a/infrastructure/outputs.tf b/infrastructure/outputs.tf new file mode 100644 index 0000000..f662b35 --- /dev/null +++ b/infrastructure/outputs.tf @@ -0,0 +1,44 @@ +output "ec2_public_ip" { + description = "Elastic IP — point your domain A record here" + value = aws_eip.soko.public_ip +} + +output "ec2_instance_id" { + description = "EC2 instance ID" + value = aws_instance.soko.id +} + +output "cloudfront_domain" { + description = "CloudFront domain — use as CNAME for app.yourdomain.com" + value = aws_cloudfront_distribution.frontend.domain_name +} + +output "cloudfront_distribution_id" { + description = "CloudFront distribution ID — needed for cache invalidations in CI" + value = aws_cloudfront_distribution.frontend.id +} + +output "frontend_bucket_name" { + description = "S3 bucket name for frontend assets" + value = aws_s3_bucket.frontend.bucket +} + +output "images_bucket_name" { + description = "S3 bucket name for produce images" + value = aws_s3_bucket.images.bucket +} + +output "platform_secret_arn" { + description = "ARN of the platform secrets in Secrets Manager" + value = aws_secretsmanager_secret.platform.arn +} + +output "ml_secret_arn" { + description = "ARN of the ML secrets in Secrets Manager" + value = aws_secretsmanager_secret.ml.arn +} + +output "ssh_command" { + description = "SSH command to connect to the server" + value = "ssh -i ~/.ssh/${var.ec2_key_name}.pem ubuntu@${aws_eip.soko.public_ip}" +} diff --git a/infrastructure/s3.tf b/infrastructure/s3.tf new file mode 100644 index 0000000..32026ef --- /dev/null +++ b/infrastructure/s3.tf @@ -0,0 +1,74 @@ +# ── Produce images bucket ───────────────────────────────────────────────────── +resource "aws_s3_bucket" "images" { + bucket = "soko-produce-images-${var.environment}" + tags = { Name = "soko-produce-images" } +} + +resource "aws_s3_bucket_versioning" "images" { + bucket = aws_s3_bucket.images.id + versioning_configuration { status = "Enabled" } +} + +resource "aws_s3_bucket_server_side_encryption_configuration" "images" { + bucket = aws_s3_bucket.images.id + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" + } + } +} + +resource "aws_s3_bucket_cors_configuration" "images" { + bucket = aws_s3_bucket.images.id + cors_rule { + allowed_headers = ["*"] + allowed_methods = ["GET", "PUT", "POST"] + allowed_origins = var.domain_name != "" ? ["https://${var.domain_name}"] : ["*"] + max_age_seconds = 3000 + } +} + +# ── Frontend static assets bucket ──────────────────────────────────────────── +resource "aws_s3_bucket" "frontend" { + bucket = "soko-frontend-${var.environment}" + tags = { Name = "soko-frontend" } +} + +resource "aws_s3_bucket_versioning" "frontend" { + bucket = aws_s3_bucket.frontend.id + versioning_configuration { status = "Enabled" } +} + +resource "aws_s3_bucket_public_access_block" "frontend" { + bucket = aws_s3_bucket.frontend.id + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + +# Only CloudFront can read frontend assets — no direct public S3 access +resource "aws_s3_bucket_policy" "frontend" { + bucket = aws_s3_bucket.frontend.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Sid = "AllowCloudFrontOAC" + Effect = "Allow" + Principal = { + Service = "cloudfront.amazonaws.com" + } + Action = "s3:GetObject" + Resource = "${aws_s3_bucket.frontend.arn}/*" + Condition = { + StringEquals = { + "AWS:SourceArn" = aws_cloudfront_distribution.frontend.arn + } + } + }] + }) + + depends_on = [aws_s3_bucket_public_access_block.frontend] +} diff --git a/infrastructure/secrets.tf b/infrastructure/secrets.tf new file mode 100644 index 0000000..e55a6b0 --- /dev/null +++ b/infrastructure/secrets.tf @@ -0,0 +1,79 @@ +# ── Platform secrets (all core services) ───────────────────────────────────── +resource "aws_secretsmanager_secret" "platform" { + name = "soko/platform" + description = "All Soko core platform secrets" + recovery_window_in_days = 7 + + tags = { Name = "soko-platform-secrets" } +} + +# Skeleton — set real values via AWS Console or `aws secretsmanager put-secret-value` +# after `terraform apply`. Do NOT put real secrets in this file. +resource "aws_secretsmanager_secret_version" "platform" { + secret_id = aws_secretsmanager_secret.platform.id + + secret_string = jsonencode({ + # ── Database passwords ─────────────────────────────────────────────────── + AUTH_DB_PASS = "CHANGE_ME" + USER_DB_PASS = "CHANGE_ME" + PRODUCE_DB_PASS = "CHANGE_ME" + ORDER_DB_PASS = "CHANGE_ME" + PAYMENT_DB_PASS = "CHANGE_ME" + MESSAGE_DB_PASS = "CHANGE_ME" + NOTIFICATION_DB_PASS = "CHANGE_ME" + BLOG_DB_PASS = "CHANGE_ME" + USSD_DB_PASS = "CHANGE_ME" + + # ── Auth / JWT ──────────────────────────────────────────────────────────── + SECRET_KEY = "CHANGE_ME" + INTERNAL_SECRET = "CHANGE_ME" + ALGORITHM = "HS256" + + # ── Google OAuth ────────────────────────────────────────────────────────── + GOOGLE_CLIENT_ID = "CHANGE_ME" + GOOGLE_CLIENT_SECRET = "CHANGE_ME" + + # ── PesaPal (Payments) ──────────────────────────────────────────────────── + PESAPAL_CONSUMER_KEY = "CHANGE_ME" + PESAPAL_CONSUMER_SECRET = "CHANGE_ME" + PESAPAL_ENV = "production" + + # ── Africa's Talking (USSD + SMS) ───────────────────────────────────────── + AT_USERNAME = "CHANGE_ME" + AT_API_KEY = "CHANGE_ME" + AT_SENDER_ID = "SOKO" + + # ── SendGrid / Email ────────────────────────────────────────────────────── + SENDGRID_API_KEY = "CHANGE_ME" + SENDGRID_FROM_EMAIL = "noreply@CHANGE_ME" + }) + + lifecycle { + # Terraform manages the secret skeleton only — real values updated externally + ignore_changes = [secret_string] + } +} + +# ── ML stack secrets ────────────────────────────────────────────────────────── +resource "aws_secretsmanager_secret" "ml" { + name = "soko/ml" + description = "Soko ML layer secrets" + recovery_window_in_days = 7 + + tags = { Name = "soko-ml-secrets" } +} + +resource "aws_secretsmanager_secret_version" "ml" { + secret_id = aws_secretsmanager_secret.ml.id + + secret_string = jsonencode({ + ML_DB_PASS = "CHANGE_ME" + ML_REDIS_PASSWORD = "" + INTERNAL_API_KEY = "CHANGE_ME" + GOOGLE_MAPS_API_KEY = "CHANGE_ME" + }) + + lifecycle { + ignore_changes = [secret_string] + } +} diff --git a/infrastructure/terraform.tfvars.example b/infrastructure/terraform.tfvars.example new file mode 100644 index 0000000..266c687 --- /dev/null +++ b/infrastructure/terraform.tfvars.example @@ -0,0 +1,11 @@ +# Copy this to terraform.tfvars and fill in your values. +# terraform.tfvars is gitignored — never commit it. + +aws_region = "af-south-1" +environment = "prod" +instance_type = "t3.xlarge" +ec2_key_name = "soko-prod-key" # Name of key pair in AWS EC2 console +allowed_ssh_cidr = "YOUR_IP/32" # Replace with your home/office IP +domain_name = "yourdomain.com" # Leave empty string "" if no domain yet +alert_email = "andrewssuubi@gmail.com" +github_repo = "the-icemann/soko" diff --git a/infrastructure/user_data.sh b/infrastructure/user_data.sh new file mode 100644 index 0000000..762cec6 --- /dev/null +++ b/infrastructure/user_data.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# EC2 first-boot bootstrap — runs once as root at instance launch +set -e + +export DEBIAN_FRONTEND=noninteractive +AWS_REGION="${aws_region}" + +# ── System update ───────────────────────────────────────────────────────────── +apt-get update -y +apt-get upgrade -y + +# ── Docker ──────────────────────────────────────────────────────────────────── +apt-get install -y ca-certificates curl gnupg lsb-release jq git make +install -m 0755 -d /etc/apt/keyrings +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg +chmod a+r /etc/apt/keyrings/docker.gpg + +echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] \ + https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" \ + > /etc/apt/sources.list.d/docker.list + +apt-get update -y +apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + +systemctl enable docker +systemctl start docker +usermod -aG docker ubuntu + +# ── AWS CLI v2 ──────────────────────────────────────────────────────────────── +curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o /tmp/awscliv2.zip +cd /tmp && unzip -q awscliv2.zip && ./aws/install && cd / +rm -rf /tmp/aws /tmp/awscliv2.zip + +# ── Clone Soko repo ─────────────────────────────────────────────────────────── +mkdir -p /opt/soko +git clone https://github.com/the-icemann/soko.git /opt/soko +chown -R ubuntu:ubuntu /opt/soko + +# ── Fetch secrets and write .env files ─────────────────────────────────────── +chmod +x /opt/soko/scripts/fetch-secrets.sh +sudo -u ubuntu bash /opt/soko/scripts/fetch-secrets.sh + +# ── Create the external Docker bridge network for ML stack ─────────────────── +docker network create soko-ml-bridge 2>/dev/null || true + +# ── Start core platform ─────────────────────────────────────────────────────── +cd /opt/soko +sudo -u ubuntu docker compose up -d --build + +# ── Start ML stack ──────────────────────────────────────────────────────────── +cd /opt/soko/services/soko-ml +chmod +x /opt/soko/scripts/fetch-ml-secrets.sh +sudo -u ubuntu bash /opt/soko/scripts/fetch-ml-secrets.sh + +# Bootstrap ML data (generate synthetic data + train Prophet models) +# This runs in background — takes 5-10 min; service starts with fallback until done +sudo -u ubuntu bash -c " + cd /opt/soko/services/soko-ml + docker compose up -d --build + sleep 30 + docker compose exec -T data-ingestion-service python -m src.main & +" & + +echo "Soko bootstrap complete. Check 'docker compose logs' for status." diff --git a/infrastructure/variables.tf b/infrastructure/variables.tf new file mode 100644 index 0000000..99c452f --- /dev/null +++ b/infrastructure/variables.tf @@ -0,0 +1,45 @@ +variable "aws_region" { + description = "AWS region — af-south-1 (Cape Town) is closest to Uganda" + type = string + default = "af-south-1" +} + +variable "environment" { + description = "Deployment environment" + type = string + default = "prod" +} + +variable "instance_type" { + description = "EC2 instance type" + type = string + default = "t3.xlarge" +} + +variable "ec2_key_name" { + description = "Name of the EC2 key pair (must already exist in AWS)" + type = string +} + +variable "allowed_ssh_cidr" { + description = "CIDR block allowed to SSH into the EC2 instance. Use your own IP: x.x.x.x/32" + type = string + default = "0.0.0.0/0" +} + +variable "domain_name" { + description = "Your domain name (e.g. soko.ug). Used for CORS and PesaPal callback URLs." + type = string + default = "" +} + +variable "alert_email" { + description = "Email address for CloudWatch billing/infra alerts" + type = string +} + +variable "github_repo" { + description = "GitHub repo in owner/name format for OIDC trust (e.g. the-icemann/soko)" + type = string + default = "the-icemann/soko" +} diff --git a/infrastructure/vpc.tf b/infrastructure/vpc.tf new file mode 100644 index 0000000..687c074 --- /dev/null +++ b/infrastructure/vpc.tf @@ -0,0 +1,37 @@ +resource "aws_vpc" "soko" { + cidr_block = "10.0.0.0/16" + enable_dns_hostnames = true + enable_dns_support = true + + tags = { Name = "soko-vpc" } +} + +resource "aws_internet_gateway" "soko" { + vpc_id = aws_vpc.soko.id + tags = { Name = "soko-igw" } +} + +resource "aws_subnet" "public" { + vpc_id = aws_vpc.soko.id + cidr_block = "10.0.1.0/24" + availability_zone = "${var.aws_region}a" + map_public_ip_on_launch = true + + tags = { Name = "soko-public-subnet" } +} + +resource "aws_route_table" "public" { + vpc_id = aws_vpc.soko.id + + route { + cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.soko.id + } + + tags = { Name = "soko-public-rt" } +} + +resource "aws_route_table_association" "public" { + subnet_id = aws_subnet.public.id + route_table_id = aws_route_table.public.id +} diff --git a/nginx/nginx.conf b/nginx/nginx.conf index e5a4b8a..e42a40d 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -4,7 +4,7 @@ events { http { - limit_req_zone $binary_remote_addr zone=api_limit:10m rate=30r/m; + limit_req_zone $binary_remote_addr zone=api_limit:10m rate=120r/m; resolver 127.0.0.11 valid=30s; @@ -74,6 +74,10 @@ http { ~^/ml/recommend(/.*)?$ /recommend$1; default /; } + map $request_uri $ml_location_upstream_path { + ~^/ml/location(/.*)?$ /location$1; + default /; + } server { listen 80; @@ -471,6 +475,22 @@ http { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; } + location /ml/location/ { + limit_req zone=api_limit burst=30 nodelay; + auth_request /_verify_token; + auth_request_set $user_id $upstream_http_x_user_id; + auth_request_set $user_role $upstream_http_x_user_role; + error_page 401 = @error401; + error_page 403 = @error403; + set $ml_gw "ml-gateway:8000"; + proxy_pass http://$ml_gw$ml_location_upstream_path; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-User-Id $user_id; + proxy_set_header X-User-Role $user_role; + } + location /ml/recommend/ { limit_req zone=api_limit burst=20 nodelay; diff --git a/scripts/bootstrap-tf-state.sh b/scripts/bootstrap-tf-state.sh new file mode 100755 index 0000000..d5e42d5 --- /dev/null +++ b/scripts/bootstrap-tf-state.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Run ONCE before `terraform init` to create the S3 + DynamoDB Terraform backend. +# After this runs, commit infrastructure/ and push — Terraform state lives in S3. + +set -euo pipefail + +REGION="af-south-1" +BUCKET="soko-terraform-state" +TABLE="soko-terraform-locks" + +echo "Creating Terraform state bucket: $BUCKET" +aws s3api create-bucket \ + --bucket "$BUCKET" \ + --region "$REGION" \ + --create-bucket-configuration LocationConstraint="$REGION" + +aws s3api put-bucket-versioning \ + --bucket "$BUCKET" \ + --versioning-configuration Status=Enabled + +aws s3api put-bucket-encryption \ + --bucket "$BUCKET" \ + --server-side-encryption-configuration '{ + "Rules":[{"ApplyServerSideEncryptionByDefault":{"SSEAlgorithm":"AES256"}}] + }' + +aws s3api put-public-access-block \ + --bucket "$BUCKET" \ + --public-access-block-configuration \ + "BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true" + +echo "Creating DynamoDB lock table: $TABLE" +aws dynamodb create-table \ + --table-name "$TABLE" \ + --attribute-definitions AttributeName=LockID,AttributeType=S \ + --key-schema AttributeName=LockID,KeyType=HASH \ + --billing-mode PAY_PER_REQUEST \ + --region "$REGION" + +echo "" +echo "Bootstrap complete. Now run:" +echo " cd infrastructure" +echo " terraform init" +echo " cp terraform.tfvars.example terraform.tfvars # fill in your values" +echo " terraform plan" +echo " terraform apply" diff --git a/scripts/fetch-ml-secrets.sh b/scripts/fetch-ml-secrets.sh new file mode 100755 index 0000000..cc0b159 --- /dev/null +++ b/scripts/fetch-ml-secrets.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Fetches soko/ml from AWS Secrets Manager and writes: +# - services/soko-ml/.env (ML docker-compose substitution) + +set -euo pipefail + +REPO_DIR="$(cd "$(dirname "$0")/.." && pwd)" +REGION="${AWS_DEFAULT_REGION:-af-south-1}" +SECRET_NAME="soko/ml" + +echo "[fetch-ml-secrets] Pulling $SECRET_NAME from Secrets Manager ($REGION)..." + +RAW=$(aws secretsmanager get-secret-value \ + --secret-id "$SECRET_NAME" \ + --region "$REGION" \ + --query "SecretString" \ + --output text) + +s() { echo "$RAW" | jq -r ".${1}"; } + +cat > "$REPO_DIR/services/soko-ml/.env" </.env (per-service env files) +# +# Requires: aws-cli v2, jq +# IAM: EC2 instance profile must have secretsmanager:GetSecretValue on soko/platform + +set -euo pipefail + +REPO_DIR="$(cd "$(dirname "$0")/.." && pwd)" +REGION="${AWS_DEFAULT_REGION:-af-south-1}" +SECRET_NAME="soko/platform" + +echo "[fetch-secrets] Pulling $SECRET_NAME from Secrets Manager ($REGION)..." + +RAW=$(aws secretsmanager get-secret-value \ + --secret-id "$SECRET_NAME" \ + --region "$REGION" \ + --query "SecretString" \ + --output text) + +# Helper: extract a key from the JSON +s() { echo "$RAW" | jq -r ".${1}"; } + +# ── Root .env (docker-compose substitution) ─────────────────────────────────── +# Docker Compose auto-loads this file for ${VAR} substitution +cat > "$REPO_DIR/.env" < "$REPO_DIR/services/auth/.env" </dev/null || echo "yourdomain.com")/auth/google/callback +FRONTEND_URL=https://$(s DOMAIN 2>/dev/null || echo "yourdomain.com") +INTERNAL_SECRET=$(s INTERNAL_SECRET) +USER_SERVICE_URL=http://user_service:8002 +EOF +chmod 600 "$REPO_DIR/services/auth/.env" + +# ── Payment Service ─────────────────────────────────────────────────────────── +cat > "$REPO_DIR/services/payment/.env" </dev/null || echo "yourdomain.com")/payments/webhook/pesapal/ipn +PESAPAL_CALLBACK_URL=https://$(s DOMAIN 2>/dev/null || echo "yourdomain.com")/payments/callback +ORDER_SERVICE_URL=http://order_service:8004 +USER_SERVICE_URL=http://user_service:8002 +NOTIFICATION_SERVICE_URL=http://notification_service:8007 +FRONTEND_URL=https://$(s DOMAIN 2>/dev/null || echo "yourdomain.com") +EOF +chmod 600 "$REPO_DIR/services/payment/.env" + +# ── Notification Service ────────────────────────────────────────────────────── +cat > "$REPO_DIR/services/notification/.env" < "$REPO_DIR/services/ussd/.env" < "$REPO_DIR/services/message/.env" < "$REPO_DIR/services/$svc/.env" < None: @@ -52,6 +53,95 @@ async def _notify_recommendation_reload() -> None: except Exception as exc: log.warning(f"recommendation_reload_failed: {exc}") + +async def _broadcast_match_notifications() -> None: + """ + After profiles are loaded into the recommendation service, fan out + match notifications in both directions: + • For every buyer → notify each high-scoring matched farmer + • For every farmer → notify each high-scoring matched buyer + Threshold ≥ 0.20 keeps noise low while still catching real overlap. + """ + headers = {"x-internal-secret": INTERNAL_API_KEY} + + try: + pool = await get_pool() + async with pool.acquire() as conn: + buyers = await conn.fetch( + "SELECT buyer_id, name, district, preferred_crops FROM buyer_features" + ) + farmers = await conn.fetch( + "SELECT farmer_id, name, district, crops_offered FROM farmer_features" + ) + + async with httpx.AsyncClient(timeout=60.0) as client: + # ── buyers → notify matched farmers ─────────────────────────────── + for b in buyers: + try: + r = await client.get( + f"{REC_SERVICE_URL}/recommend/farmers-for-buyer/{b['buyer_id']}", + params={"top_n": 5}, + ) + if r.status_code != 200: + continue + crops = ", ".join((b["preferred_crops"] or [])[:2]) or "your interests" + for farmer in r.json().get("recommended_farmers", []): + if farmer.get("matchScore", 0) < 0.20: + continue + await client.post( + f"{NOTIFICATION_SERVICE_URL}/internal/notify", + json={ + "event": "system", + "farmer_id": farmer["id"], + "meta": { + "message": ( + f"{b['name']} from {b['district'] or 'Uganda'} " + f"is looking for {crops} — they match your produce. " + f"See them on your home feed." + ) + }, + }, + headers=headers, + ) + except Exception as exc: + log.warning("broadcast_buyer_match_failed", buyer=b["buyer_id"], error=str(exc)) + + # ── farmers → notify matched buyers ─────────────────────────────── + for f in farmers: + try: + r = await client.get( + f"{REC_SERVICE_URL}/recommend/buyers-for-farmer/{f['farmer_id']}", + params={"top_n": 5}, + ) + if r.status_code != 200: + continue + crops = ", ".join((f["crops_offered"] or [])[:2]) or "fresh produce" + for buyer in r.json().get("recommended_buyers", []): + if buyer.get("matchScore", 0) < 0.20: + continue + await client.post( + f"{NOTIFICATION_SERVICE_URL}/internal/notify", + json={ + "event": "system", + "buyer_id": buyer["id"], + "meta": { + "message": ( + f"{f['name']} from {f['district'] or 'Uganda'} " + f"grows {crops} — matching what you're looking for. " + f"See them on your home feed." + ) + }, + }, + headers=headers, + ) + except Exception as exc: + log.warning("broadcast_farmer_match_failed", farmer=f["farmer_id"], error=str(exc)) + + log.info("match_broadcast_complete", buyers=len(buyers), farmers=len(farmers)) + + except Exception as exc: + log.warning(f"broadcast_match_notifications_failed: {exc}") + _stream: TransactionStream | None = None _bootstrap_lock = asyncio.Lock() @@ -87,6 +177,7 @@ async def lifespan(app: FastAPI): result = await _run_bootstrap() log.info("bootstrap_complete", **result) await _notify_recommendation_reload() + await _broadcast_match_notifications() except Exception as exc: log.error(f"bootstrap_failed: {exc}") else: @@ -125,6 +216,7 @@ async def _do_bootstrap(): result = await _run_bootstrap() log.info("manual_bootstrap_complete", **result) await _notify_recommendation_reload() + await _broadcast_match_notifications() background_tasks.add_task(_do_bootstrap) return {"message": "Bootstrap triggered — running in background"} @@ -168,6 +260,47 @@ async def ingest_order_event(payload: IngestOrderEventPayload): return {"status": "inserted" if inserted else "rejected_outlier"} +@app.post("/ingest/user-created") +async def ingest_user_created(payload: UserCreatedPayload): + """ + Syncs a newly-registered user to the ML feature store immediately, + then triggers a recommendation-service profile reload. + Called by user-service as a fire-and-forget background task after account creation. + """ + from .transformers.farmer_transformer import transform_farmer + from .transformers.buyer_transformer import transform_buyer + from .feature_store import upsert_farmer, upsert_buyer + + role = payload.role.lower() + + if role in ("farmer", "both"): + record = transform_farmer({ + "id": payload.id, + "name": payload.full_name, + "district": payload.district or "", + "specialties": payload.specialties or [], + "averageRating": 0.0, + "totalSales": 0, + "totalListings": 0, + }) + await upsert_farmer(record) + + if role in ("buyer", "both"): + record = transform_buyer({ + "id": payload.id, + "name": payload.full_name, + "district": payload.district or "", + "interests": payload.interests or [], + "totalOrders": 0, + "totalSpent": 0, + }) + await upsert_buyer(record) + + await _notify_recommendation_reload() + log.info("user_created_synced", user_id=payload.id, role=role) + return {"status": "synced", "role": role} + + @app.get("/gaps/summary") async def gap_summary(): return await get_gap_summary() diff --git a/services/soko-ml/data-ingestion-service/src/schemas.py b/services/soko-ml/data-ingestion-service/src/schemas.py index 6a08658..4cfcdc9 100644 --- a/services/soko-ml/data-ingestion-service/src/schemas.py +++ b/services/soko-ml/data-ingestion-service/src/schemas.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import Optional +from typing import List, Optional from pydantic import BaseModel @@ -95,3 +95,13 @@ class BootstrapStatusResponse(BaseModel): orders_ingested: int coverage_pairs: int already_bootstrapped: bool + + +class UserCreatedPayload(BaseModel): + """Sent by user-service after a new account is created.""" + id: str + role: str + full_name: str + district: Optional[str] = None + specialties: Optional[List[str]] = None + interests: Optional[List[str]] = None diff --git a/services/soko-ml/docker-compose.yml b/services/soko-ml/docker-compose.yml index b568183..0329381 100644 --- a/services/soko-ml/docker-compose.yml +++ b/services/soko-ml/docker-compose.yml @@ -282,6 +282,7 @@ services: - PRODUCE_SERVICE_URL=${PRODUCE_SERVICE_URL:-http://produce_service:8003} - LISTING_SERVICE_URL=${LISTING_SERVICE_URL:-http://produce_service:8003} - REC_SERVICE_URL=http://recommendation-service:8002 + - NOTIFICATION_SERVICE_URL=http://notification_service:8007 - INTERNAL_API_KEY=${INTERNAL_API_KEY:-internal-secret} - BOOTSTRAP_ON_STARTUP=${BOOTSTRAP_ON_STARTUP:-true} - MIN_OBSERVATIONS_FOR_MODEL=${MIN_OBSERVATIONS_FOR_MODEL:-30} diff --git a/services/user/app/core/config.py b/services/user/app/core/config.py index 2101f9f..acc2219 100644 --- a/services/user/app/core/config.py +++ b/services/user/app/core/config.py @@ -4,7 +4,10 @@ class Settings(BaseSettings): DATABASE_URL: str INTERNAL_SECRET: str - AUTH_SERVICE_URL: str = "http://localhost:8001/docs" + AUTH_SERVICE_URL: str = "http://localhost:8001/docs" + INGEST_SERVICE_URL: str = "http://data-ingestion-service:8004" + REC_SERVICE_URL: str = "http://recommendation-service:8002" + NOTIFICATION_SERVICE_URL: str = "http://notification_service:8007" class Config: env_file = ".env" diff --git a/services/user/app/helpers/builders.py b/services/user/app/helpers/builders.py index 735981e..9b9479d 100644 --- a/services/user/app/helpers/builders.py +++ b/services/user/app/helpers/builders.py @@ -12,6 +12,8 @@ def make_initials(name: str) -> str: def build_authenticated_user(user: UserProfile) -> AuthenticatedUser: fs = user.farmer_stats bs = user.buyer_stats + specialties = [s.strip() for s in user.specialties.split(",") if s.strip()] if user.specialties else [] + interests = [i.strip() for i in user.interests.split(",") if i.strip()] if user.interests else [] return AuthenticatedUser( id=str(user.id), name=user.full_name or "", @@ -27,6 +29,8 @@ def build_authenticated_user(user: UserProfile) -> AuthenticatedUser: memberSince=user.created_at.isoformat(), farmerBio=user.farmer_bio, farmName=user.farm_name, + specialties=specialties, + interests=interests, totalOrders=bs.total_orders if bs else None, totalSpent=bs.total_spent if bs else None, wishlistCount=bs.wishlist_count if bs else None, diff --git a/services/user/app/routers/internal.py b/services/user/app/routers/internal.py index 5507a2d..cd2bf82 100644 --- a/services/user/app/routers/internal.py +++ b/services/user/app/routers/internal.py @@ -1,19 +1,113 @@ +import asyncio +import logging from sqlite3 import IntegrityError -from fastapi import APIRouter, Depends, HTTPException +import httpx +from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException from sqlalchemy.orm import Session -from app.db.database import get_db + +from app.core.config import settings from app.core.dependencies import internal_only +from app.db.database import get_db from app.models.user import UserProfile, FarmerStats, BuyerStats, UserSettings from app.schemas.schemas import CreateUserPayload, UpdateFarmerStats, UpdateBuyerStats import uuid +logger = logging.getLogger(__name__) router = APIRouter(tags=["Internal"], dependencies=[Depends(internal_only)]) +async def _notify(client: httpx.AsyncClient, target_id: str, is_farmer: bool, message: str) -> None: + key = "farmer_id" if is_farmer else "buyer_id" + await client.post( + f"{settings.NOTIFICATION_SERVICE_URL}/internal/notify", + json={"event": "system", key: target_id, "meta": {"message": message}}, + headers={"x-internal-secret": settings.INTERNAL_SECRET}, + ) + + +async def _post_signup_sync( + user_id: str, + role: str, + full_name: str, + district: str | None, + specialties: list[str] | None, + interests: list[str] | None, +) -> None: + """ + Fire-and-forget: syncs the new user to the ML feature store, reloads + the recommendation service, then sends real-time notifications to + matched counterparts in both directions. + """ + try: + async with httpx.AsyncClient(timeout=15.0) as client: + # ── 1. Sync to ML feature store ──────────────────────────────────── + await client.post( + f"{settings.INGEST_SERVICE_URL}/ingest/user-created", + json={ + "id": user_id, + "role": role, + "full_name": full_name, + "district": district, + "specialties": specialties or [], + "interests": interests or [], + }, + headers={"x-internal-secret": settings.INTERNAL_SECRET}, + ) + + await asyncio.sleep(4) # allow recommendation service to reload profiles + + location = district or "Uganda" + + # ── 2. New BUYER → notify matched farmers ────────────────────────── + if role in ("buyer", "both") and interests: + crops = ", ".join(interests[:2]) + rec = await client.get( + f"{settings.REC_SERVICE_URL}/recommend/farmers-for-buyer/{user_id}", + params={"top_n": 5}, + ) + if rec.status_code == 200: + for farmer in rec.json().get("recommended_farmers", []): + if farmer.get("matchScore", 0) < 0.15: + continue + await _notify( + client, farmer["id"], is_farmer=True, + message=( + f"{full_name} from {location} is looking for {crops}. " + f"Their interests match your produce — check them out on your home feed." + ), + ) + + # ── 3. New FARMER → notify matched buyers ────────────────────────── + if role in ("farmer", "both") and specialties: + crops = ", ".join(specialties[:2]) + rec = await client.get( + f"{settings.REC_SERVICE_URL}/recommend/buyers-for-farmer/{user_id}", + params={"top_n": 5}, + ) + if rec.status_code == 200: + for buyer in rec.json().get("recommended_buyers", []): + if buyer.get("matchScore", 0) < 0.15: + continue + await _notify( + client, buyer["id"], is_farmer=False, + message=( + f"{full_name} from {location} just listed {crops}. " + f"They match your interests — see their farm on your home feed." + ), + ) + + except Exception as exc: + logger.warning(f"Post-signup sync failed for {user_id}: {exc}") + + @router.post("/", status_code=201) -def create_user(payload: CreateUserPayload, db: Session = Depends(get_db)): +def create_user( + payload: CreateUserPayload, + background_tasks: BackgroundTasks, + db: Session = Depends(get_db), +): """Auth Service calls this after creating credentials.""" profile = UserProfile( id=uuid.UUID(payload.id), @@ -34,7 +128,6 @@ def create_user(payload: CreateUserPayload, db: Session = Depends(get_db)): try: db.commit() db.refresh(profile) - return {"id": str(profile.id)} except IntegrityError as e: db.rollback() detail = str(e.orig) @@ -44,6 +137,18 @@ def create_user(payload: CreateUserPayload, db: Session = Depends(get_db)): raise HTTPException(status_code=409, detail="Email already registered") raise HTTPException(status_code=409, detail="User already exists") + background_tasks.add_task( + _post_signup_sync, + user_id=str(profile.id), + role=payload.role, + full_name=payload.full_name, + district=payload.district, + specialties=payload.specialties, + interests=payload.interests, + ) + + return {"id": str(profile.id)} + @router.put("/{user_id}/stats/farmer") def update_farmer_stats( diff --git a/services/user/app/routers/profile.py b/services/user/app/routers/profile.py index f43885c..a132afd 100644 --- a/services/user/app/routers/profile.py +++ b/services/user/app/routers/profile.py @@ -5,8 +5,8 @@ from app.db.database import get_db from app.core.dependencies import get_current_user_id from app.models.user import UserProfile -from app.schemas.schemas import AuthenticatedUser, FarmerProfile, UpdateProfile, UserRole -from app.helpers.builders import build_authenticated_user, build_farmer_profile +from app.schemas.schemas import AuthenticatedUser, BuyerPublicProfile, FarmerProfile, UpdateProfile, UserRole +from app.helpers.builders import build_authenticated_user, build_farmer_profile, make_initials import uuid router = APIRouter(tags=["Profile"]) @@ -113,6 +113,35 @@ def get_buyers( return [build_authenticated_user(b) for b in buyers] +@router.get("/buyers/{user_id}", response_model=BuyerPublicProfile) +def get_buyer_profile(user_id: str, db: Session = Depends(get_db)): + try: + uid = uuid.UUID(user_id) + except ValueError: + raise HTTPException(status_code=400, detail="Invalid user ID") + + user = db.query(UserProfile).filter(UserProfile.id == uid).first() + if not user: + raise HTTPException(status_code=404, detail="User not found") + if user.role not in (UserRole.buyer, UserRole.both): + raise HTTPException(status_code=404, detail="User is not a buyer") + + interests = [i.strip() for i in user.interests.split(",") if i.strip()] if user.interests else [] + bs = user.buyer_stats + + return BuyerPublicProfile( + id=str(user.id), + name=user.full_name or "", + initials=make_initials(user.full_name or user.email), + avatarUrl=user.avatar_url, + district=user.district or "", + verified=user.verified, + interests=interests, + memberSince=user.created_at.isoformat(), + totalOrders=bs.total_orders if bs else None, + ) + + @router.get("/{user_id}", response_model=FarmerProfile) def get_farmer_profile(user_id: str, db: Session = Depends(get_db)): try: diff --git a/services/user/app/schemas/schemas.py b/services/user/app/schemas/schemas.py index 410b00f..b66dfcb 100644 --- a/services/user/app/schemas/schemas.py +++ b/services/user/app/schemas/schemas.py @@ -47,6 +47,8 @@ class AuthenticatedUser(BaseModel): # Farmer-specific farmerBio: Optional[str] farmName: Optional[str] + specialties: List[str] + interests: List[str] # Buyer stats totalOrders: Optional[int] totalSpent: Optional[int] @@ -81,6 +83,18 @@ class FarmerProfile(BaseModel): isFollowedByMe: Optional[bool] = None isRatedByMe: Optional[int] = None +# ── BuyerPublicProfile — public view (GET /users/buyers/{id}) +class BuyerPublicProfile(BaseModel): + id: str + name: str + initials: str + avatarUrl: Optional[str] + district: str + verified: bool + interests: List[str] + memberSince: str + totalOrders: Optional[int] + # ── FarmerReview class FarmerReviewOut(BaseModel): id: str From a6da0e00f90dec212b8b3cf91ffea5b9f6416927 Mon Sep 17 00:00:00 2001 From: the-icemann Date: Tue, 19 May 2026 18:08:23 +0300 Subject: [PATCH 07/24] Allow CloudFront origins in CORS policy --- .github/workflows/deploy.yml | 2 +- DEPLOY.md | 95 ++++++++++++++++++++++------------- infrastructure/acm.tf | 89 ++++++++++++++++++++++++++++++++ infrastructure/cloudfront.tf | 11 ++-- infrastructure/main.tf | 18 ++++++- nginx/nginx.conf | 7 +++ scripts/bootstrap-tf-state.sh | 11 +++- 7 files changed, 189 insertions(+), 44 deletions(-) create mode 100644 infrastructure/acm.tf diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index fcf0deb..930614d 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -2,7 +2,7 @@ name: Deploy to Production on: push: - branches: [main] + branches: [main, price_prediction_service] workflow_dispatch: # allow manual trigger from GitHub UI concurrency: diff --git a/DEPLOY.md b/DEPLOY.md index f249751..76f3bbb 100644 --- a/DEPLOY.md +++ b/DEPLOY.md @@ -228,59 +228,70 @@ Then go to **Settings → Secrets and variables → Variables → New repository ## Step 7 — First Server Setup -SSH into the EC2 instance. The user_data script ran automatically on boot and: -- Installed Docker, git, AWS CLI -- Cloned the repo to `/opt/soko` -- Fetched secrets and started all services - -Check status: +SSH into the EC2 instance: ```bash ssh -i ~/.ssh/soko-prod-key.pem ubuntu@YOUR_EC2_IP - -# Check core services -cd /opt/soko -docker compose ps - -# Check ML stack -cd services/soko-ml -docker compose ps - -# Check logs -docker compose logs --tail=50 auth_service -docker compose logs --tail=50 soko-ml-gateway ``` -If the user_data script didn't run completely (can take 5-10 min on first boot): +> **Note:** The user_data script clones from `main`. If your working branch is not yet merged +> into `main`, the clone will be incomplete. Run the manual setup below instead. + +**Manual setup (recommended until `price_prediction_service` is merged into `main`):** ```bash -# Run manually +# Clone the correct branch into /opt/soko +sudo git clone -b price_prediction_service https://github.com/the-icemann/soko.git /opt/soko +sudo chown -R ubuntu:ubuntu /opt/soko + +# Fetch secrets and start core platform cd /opt/soko chmod +x scripts/fetch-secrets.sh scripts/fetch-ml-secrets.sh bash scripts/fetch-secrets.sh docker network create soko-ml-bridge 2>/dev/null || true docker compose up -d --build -cd services/soko-ml +# Start ML stack +cd /opt/soko/services/soko-ml bash /opt/soko/scripts/fetch-ml-secrets.sh docker compose up -d --build ``` +Check status: +```bash +# Core services +cd /opt/soko && docker compose ps + +# ML stack +cd /opt/soko/services/soko-ml && docker compose ps + +# Logs +docker compose logs --tail=50 auth_service +docker compose logs --tail=50 soko-ml-gateway +``` + --- ## Step 8 — SSL with Let's Encrypt +> **Skip this step if you have no domain yet.** The app runs fine over HTTP on the EC2 IP +> in the meantime. Come back here once `soko-ug.com` NS propagation is confirmed working. + Once your domain's A record points to the EC2 Elastic IP: +1. In `terraform.tfvars` set `domain_name = "soko-ug.com"` +2. Uncomment the `us_east_1` provider alias in `main.tf` +3. Uncomment the contents of `acm.tf` +4. Uncomment the `depends_on` and `viewer_certificate` domain lines in `cloudfront.tf` +5. Run `terraform apply` — Terraform will create the ACM cert and Route 53 records automatically +6. Then run Certbot on EC2 for the NGINX certificate: + ```bash ssh -i ~/.ssh/soko-prod-key.pem ubuntu@YOUR_EC2_IP -# Install certbot sudo apt-get install -y certbot python3-certbot-nginx -# Get certificate (NGINX must be running on port 80) -sudo certbot --nginx -d yourdomain.com -d www.yourdomain.com \ +sudo certbot --nginx -d soko-ug.com -d www.soko-ug.com \ --non-interactive --agree-tos -m andrewssuubi@gmail.com -# Certbot auto-renews via systemd timer — verify: sudo systemctl status certbot.timer ``` @@ -290,15 +301,24 @@ Then update the PesaPal callback URLs in Secrets Manager to use `https://`. ## Step 9 — Deploy Frontend (First Time) -Push to main in the frontend repo to trigger the GitHub Actions workflow, or run manually: +Get your CloudFront URL and EC2 IP from Terraform outputs: +```bash +cd infrastructure +terraform output cloudfront_domain # e.g. xxxx.cloudfront.net ← frontend lives here +terraform output ec2_public_ip # e.g. 13.244.217.134 ← backend API +``` + +Update the `VITE_API_BASE_URL` GitHub Actions variable in the frontend repo +(**Settings → Secrets and variables → Variables**) to `http://` (or +`https://soko-ug.com` once the domain is live). +Deploy manually (or just push to main to trigger GitHub Actions): ```bash -cd /home/the-icemann/Desktop/soko_client_final +cd /path/to/soko_client_final -# Set the API URL for local build test -VITE_API_BASE_URL=https://yourdomain.com npm run build +# Build — use EC2 IP until domain is live +VITE_API_BASE_URL=http://13.244.217.134 npm run build -# Deploy manually (AWS CLI must be configured) aws s3 sync dist/assets/ s3://soko-frontend-prod/assets/ \ --cache-control "public, max-age=31536000, immutable" --delete @@ -311,7 +331,8 @@ aws cloudfront create-invalidation \ --paths "/*" ``` -Frontend is now live at the CloudFront domain (or your custom subdomain if configured). +Frontend is live at the CloudFront URL (`https://xxxx.cloudfront.net`). +Once `soko-ug.com` is wired up, it will also be at `https://app.soko-ug.com`. --- @@ -319,13 +340,17 @@ Frontend is now live at the CloudFront domain (or your custom subdomain if confi ### Deploying changes ```bash -# Backend: just push to main — GitHub Actions SSHes in and redeploys -git push origin main +# Backend: push to price_prediction_service (or main once merged) +git push origin price_prediction_service -# Frontend: just push to main in the client repo -cd /path/to/soko_client_final && git push origin main +# Frontend: push to price_preds (or main once merged) +cd /path/to/soko_client_final && git push origin price_preds ``` +> Both GitHub Actions workflows currently trigger on their respective working branches +> (`price_prediction_service` for backend, `price_preds` for frontend) as well as `main`. +> Once both branches are merged into `main`, only `main` pushes are needed. + ### Viewing logs ```bash ssh -i ~/.ssh/soko-prod-key.pem ubuntu@YOUR_EC2_IP diff --git a/infrastructure/acm.tf b/infrastructure/acm.tf new file mode 100644 index 0000000..da3c6f2 --- /dev/null +++ b/infrastructure/acm.tf @@ -0,0 +1,89 @@ +# Domain + ACM certificate setup — uncomment when soko-ug.com NS propagation is confirmed. +# To re-enable: set domain_name = "soko-ug.com" in terraform.tfvars and uncomment below. + +# resource "aws_acm_certificate" "frontend" { +# count = var.domain_name != "" ? 1 : 0 +# provider = aws.us_east_1 +# +# domain_name = "app.${var.domain_name}" +# subject_alternative_names = ["${var.domain_name}", "www.${var.domain_name}"] +# validation_method = "DNS" +# +# lifecycle { +# create_before_destroy = true +# } +# +# tags = { Name = "soko-frontend-cert" } +# } +# +# resource "aws_route53_zone" "main" { +# count = var.domain_name != "" ? 1 : 0 +# name = var.domain_name +# tags = { Name = "soko-zone" } +# } +# +# resource "aws_route53_record" "cert_validation" { +# for_each = var.domain_name != "" ? { +# for dvo in aws_acm_certificate.frontend[0].domain_validation_options : dvo.domain_name => { +# name = dvo.resource_record_name +# type = dvo.resource_record_type +# record = dvo.resource_record_value +# } +# } : {} +# +# zone_id = aws_route53_zone.main[0].zone_id +# name = each.value.name +# type = each.value.type +# ttl = 60 +# records = [each.value.record] +# } +# +# resource "aws_acm_certificate_validation" "frontend" { +# count = var.domain_name != "" ? 1 : 0 +# provider = aws.us_east_1 +# certificate_arn = aws_acm_certificate.frontend[0].arn +# validation_record_fqdns = [for r in aws_route53_record.cert_validation : r.fqdn] +# } +# +# resource "aws_route53_record" "root" { +# count = var.domain_name != "" ? 1 : 0 +# zone_id = aws_route53_zone.main[0].zone_id +# name = var.domain_name +# type = "A" +# ttl = 300 +# records = [aws_eip.soko.public_ip] +# } +# +# resource "aws_route53_record" "www" { +# count = var.domain_name != "" ? 1 : 0 +# zone_id = aws_route53_zone.main[0].zone_id +# name = "www.${var.domain_name}" +# type = "A" +# ttl = 300 +# records = [aws_eip.soko.public_ip] +# } +# +# resource "aws_route53_record" "app" { +# count = var.domain_name != "" ? 1 : 0 +# zone_id = aws_route53_zone.main[0].zone_id +# name = "app.${var.domain_name}" +# type = "CNAME" +# ttl = 300 +# records = [aws_cloudfront_distribution.frontend.domain_name] +# } +# +# output "route53_nameservers" { +# description = "Point your domain registrar to these nameservers" +# value = var.domain_name != "" ? aws_route53_zone.main[0].name_servers : [] +# } +# +# output "acm_validation_records" { +# description = "Add these DNS records to validate your ACM certificate" +# value = var.domain_name != "" ? { +# for dvo in aws_acm_certificate.frontend[0].domain_validation_options : dvo.domain_name => { +# name = dvo.resource_record_name +# type = dvo.resource_record_type +# value = dvo.resource_record_value +# } +# } : {} +# } diff --git a/infrastructure/cloudfront.tf b/infrastructure/cloudfront.tf index 96cfcdb..d4dea8a 100644 --- a/infrastructure/cloudfront.tf +++ b/infrastructure/cloudfront.tf @@ -14,6 +14,8 @@ resource "aws_cloudfront_distribution" "frontend" { aliases = var.domain_name != "" ? ["app.${var.domain_name}"] : [] + # depends_on = [aws_acm_certificate_validation.frontend] # uncomment with domain + origin { domain_name = aws_s3_bucket.frontend.bucket_regional_domain_name origin_id = "s3-frontend" @@ -76,11 +78,12 @@ resource "aws_cloudfront_distribution" "frontend" { } viewer_certificate { - # If you have an ACM certificate for your domain, replace this with: - # acm_certificate_arn = aws_acm_certificate.frontend.arn - # ssl_support_method = "sni-only" - # minimum_protocol_version = "TLSv1.2_2021" cloudfront_default_certificate = true + # Uncomment below and remove line above when domain is ready: + # acm_certificate_arn = aws_acm_certificate_validation.frontend[0].certificate_arn + # ssl_support_method = "sni-only" + # minimum_protocol_version = "TLSv1.2_2021" + # cloudfront_default_certificate = false } tags = { Name = "soko-frontend-cdn" } diff --git a/infrastructure/main.tf b/infrastructure/main.tf index d0ac0c4..2dd74fe 100644 --- a/infrastructure/main.tf +++ b/infrastructure/main.tf @@ -11,10 +11,10 @@ terraform { # Remote state — S3 bucket must exist before running `terraform init` # Run `scripts/bootstrap-tf-state.sh` once to create it. backend "s3" { - bucket = "soko-terraform-state" + bucket = "soko-terraform-state-491085424720" key = "prod/terraform.tfstate" region = "af-south-1" - dynamodb_table = "soko-terraform-locks" + use_lockfile = true encrypt = true } } @@ -30,3 +30,17 @@ provider "aws" { } } } + +# CloudFront ACM certificates must live in us-east-1 — uncomment when adding domain +# provider "aws" { +# alias = "us_east_1" +# region = "us-east-1" +# +# default_tags { +# tags = { +# Project = "soko" +# Environment = var.environment +# ManagedBy = "terraform" +# } +# } +# } diff --git a/nginx/nginx.conf b/nginx/nginx.conf index e42a40d..3ba0094 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -86,6 +86,13 @@ http { if ($http_origin ~* "^https?://localhost:(3000|5173|4173)$") { set $cors_origin $http_origin; } + if ($http_origin ~* "^https://[a-z0-9]+\.cloudfront\.net$") { + set $cors_origin $http_origin; + } + # Uncomment when domain is live: + # if ($http_origin = "https://app.soko-ug.com") { + # set $cors_origin $http_origin; + # } add_header Access-Control-Allow-Origin $cors_origin always; add_header Access-Control-Allow-Methods "GET, POST, PUT, PATCH, DELETE, OPTIONS" always; add_header Access-Control-Allow-Headers "Authorization, Content-Type, Accept, X-User-Id, X-User-Role" always; diff --git a/scripts/bootstrap-tf-state.sh b/scripts/bootstrap-tf-state.sh index d5e42d5..29e4d68 100755 --- a/scripts/bootstrap-tf-state.sh +++ b/scripts/bootstrap-tf-state.sh @@ -5,7 +5,8 @@ set -euo pipefail REGION="af-south-1" -BUCKET="soko-terraform-state" +ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) +BUCKET="soko-terraform-state-${ACCOUNT_ID}" TABLE="soko-terraform-locks" echo "Creating Terraform state bucket: $BUCKET" @@ -38,7 +39,13 @@ aws dynamodb create-table \ --region "$REGION" echo "" -echo "Bootstrap complete. Now run:" +echo "Bootstrap complete." +echo "State bucket: $BUCKET" +echo "" +echo "Now update infrastructure/main.tf — change the backend bucket to:" +echo " bucket = \"$BUCKET\"" +echo "" +echo "Then run:" echo " cd infrastructure" echo " terraform init" echo " cp terraform.tfvars.example terraform.tfvars # fill in your values" From 8678386e0ce1d8553c59ed952e2e24ccadc8ef8a Mon Sep 17 00:00:00 2001 From: the-icemann Date: Tue, 19 May 2026 18:20:40 +0300 Subject: [PATCH 08/24] Removed local IP address as root logger and allowed all addressing schemes to access the instance --- .github/workflows/deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 930614d..9fd89f3 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -30,8 +30,8 @@ jobs: echo "=== Pulling latest code ===" cd /opt/soko - git fetch origin main - git reset --hard origin/main + git fetch origin + git reset --hard origin/${{ github.ref_name }} echo "=== Refreshing secrets ===" bash scripts/fetch-secrets.sh From 8206bf8edcd53a09b5a800499a4cf72e71e6d602 Mon Sep 17 00:00:00 2001 From: the-icemann Date: Tue, 19 May 2026 19:57:53 +0300 Subject: [PATCH 09/24] Workaround for cloudfront serving --- nginx/nginx.conf | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 3ba0094..6210eae 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -79,20 +79,17 @@ http { default /; } + map $http_origin $cors_origin { + ~^https?://localhost:(3000|5173|4173)$ $http_origin; + ~^https://[a-z0-9]+\.cloudfront\.net$ $http_origin; + # Uncomment when domain is live: + # https://app.soko-ug.com $http_origin; + default ""; + } + server { listen 80; - set $cors_origin ""; - if ($http_origin ~* "^https?://localhost:(3000|5173|4173)$") { - set $cors_origin $http_origin; - } - if ($http_origin ~* "^https://[a-z0-9]+\.cloudfront\.net$") { - set $cors_origin $http_origin; - } - # Uncomment when domain is live: - # if ($http_origin = "https://app.soko-ug.com") { - # set $cors_origin $http_origin; - # } add_header Access-Control-Allow-Origin $cors_origin always; add_header Access-Control-Allow-Methods "GET, POST, PUT, PATCH, DELETE, OPTIONS" always; add_header Access-Control-Allow-Headers "Authorization, Content-Type, Accept, X-User-Id, X-User-Role" always; From d7f2341e6100545837461b9fa2483202a45112f9 Mon Sep 17 00:00:00 2001 From: the-icemann Date: Tue, 19 May 2026 20:06:55 +0300 Subject: [PATCH 10/24] Fix CORS preflight: add headers inside OPTIONS handler --- nginx/nginx.conf | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 6210eae..9b3f0cc 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -95,7 +95,13 @@ http { add_header Access-Control-Allow-Headers "Authorization, Content-Type, Accept, X-User-Id, X-User-Role" always; add_header Access-Control-Allow-Credentials "true" always; - if ($request_method = OPTIONS) { return 204; } + if ($request_method = OPTIONS) { + add_header Access-Control-Allow-Origin $cors_origin always; + add_header Access-Control-Allow-Methods "GET, POST, PUT, PATCH, DELETE, OPTIONS" always; + add_header Access-Control-Allow-Headers "Authorization, Content-Type, Accept, X-User-Id, X-User-Role" always; + add_header Access-Control-Allow-Credentials "true" always; + return 204; + } location = /_verify_token { internal; From d4e95714a8717fffbe558868e29d334e8d98b2f6 Mon Sep 17 00:00:00 2001 From: the-icemann Date: Tue, 19 May 2026 20:28:24 +0300 Subject: [PATCH 11/24] Fix nginx: remove add_header from server-level if block --- nginx/nginx.conf | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 9b3f0cc..6210eae 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -95,13 +95,7 @@ http { add_header Access-Control-Allow-Headers "Authorization, Content-Type, Accept, X-User-Id, X-User-Role" always; add_header Access-Control-Allow-Credentials "true" always; - if ($request_method = OPTIONS) { - add_header Access-Control-Allow-Origin $cors_origin always; - add_header Access-Control-Allow-Methods "GET, POST, PUT, PATCH, DELETE, OPTIONS" always; - add_header Access-Control-Allow-Headers "Authorization, Content-Type, Accept, X-User-Id, X-User-Role" always; - add_header Access-Control-Allow-Credentials "true" always; - return 204; - } + if ($request_method = OPTIONS) { return 204; } location = /_verify_token { internal; From 97f07aca9a82760903c21208715cc2a1d4970e6a Mon Sep 17 00:00:00 2001 From: the-icemann Date: Tue, 19 May 2026 23:33:14 +0300 Subject: [PATCH 12/24] feat: make quantity_kg optional in location service for specialty-only farmers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Farmers with specialties but no listings no longer send a fake 500 kg to the routing API. quantity_kg is now Optional[float] = None throughout the location service (schemas, market_router, fallback). When absent, total_net_value_ugx is set to 0.0 — all per-kg fields are unaffected. Co-Authored-By: Claude Sonnet 4.6 --- services/soko-ml/location-service/src/fallback.py | 4 ++-- services/soko-ml/location-service/src/market_router.py | 4 ++-- services/soko-ml/location-service/src/schemas.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/services/soko-ml/location-service/src/fallback.py b/services/soko-ml/location-service/src/fallback.py index 1a7a426..0c21ace 100644 --- a/services/soko-ml/location-service/src/fallback.py +++ b/services/soko-ml/location-service/src/fallback.py @@ -81,7 +81,7 @@ async def determine_tier(crop: str, market: str) -> int: async def build_tier2_response( farmer_id: str, crop: str, - quantity_kg: float, + quantity_kg: Optional[float], max_distance_km: float, ) -> dict: """Tier 2: returns category-level price band estimate.""" @@ -125,7 +125,7 @@ async def build_tier2_response( } -def build_tier3_response(farmer_id: str, crop: str, quantity_kg: float, reason: str = "") -> dict: +def build_tier3_response(farmer_id: str, crop: str, quantity_kg: Optional[float], reason: str = "") -> dict: """Tier 3: crop not recognised. Returns graceful no-data response. Never raises.""" return { "farmer_id": farmer_id, diff --git a/services/soko-ml/location-service/src/market_router.py b/services/soko-ml/location-service/src/market_router.py index 0f330b8..62c8a42 100644 --- a/services/soko-ml/location-service/src/market_router.py +++ b/services/soko-ml/location-service/src/market_router.py @@ -122,7 +122,7 @@ async def route( farmer_lat: float, farmer_lng: float, crop: str, - quantity_kg: float, + quantity_kg: float | None, max_distance_km: float = DEFAULT_MAX_KM, ) -> dict: """ @@ -168,7 +168,7 @@ async def route( week1_price = float(predictions[0]["predicted_price_ugx"]) net_value = week1_price - transport_cost - total_net = round(net_value * quantity_kg, 0) + total_net = round(net_value * quantity_kg, 0) if quantity_kg is not None else 0.0 signal_data = derive_signal(crop, predictions) diff --git a/services/soko-ml/location-service/src/schemas.py b/services/soko-ml/location-service/src/schemas.py index 7ee0836..38c2dec 100644 --- a/services/soko-ml/location-service/src/schemas.py +++ b/services/soko-ml/location-service/src/schemas.py @@ -7,7 +7,7 @@ class RouteRequest(BaseModel): farmer_lat: float farmer_lng: float crop: str - quantity_kg: float + quantity_kg: Optional[float] = None # None = specialty-only, no listing yet max_distance_km: float = 150.0 @@ -27,7 +27,7 @@ class MarketResult(BaseModel): class RouteResponse(BaseModel): farmer_id: str crop: str - quantity_kg: float + quantity_kg: Optional[float] currency: str = "UGX" tier: int ranked_markets: list[MarketResult] From 1ef5ec879aaab486e1e29995044d768fe9e1ecdc Mon Sep 17 00:00:00 2001 From: the-icemann Date: Wed, 20 May 2026 00:15:08 +0300 Subject: [PATCH 13/24] =?UTF-8?q?feat:=20make=20messaging=20role-agnostic?= =?UTF-8?q?=20=E2=80=94=20any=20user=20can=20initiate=20with=20any=20other?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - StartConversationPayload: farmer_id -> recipient_id (any user ID) - start_conversation: check both participant orderings when resuming, correctly increment unread for whichever slot is the non-initiator - No DB migration needed: buyer_id/farmer_id columns now mean initiator/recipient rather than enforcing roles Co-Authored-By: Claude Sonnet 4.6 --- services/message/app/routers/conversations.py | 40 ++++++++++++------- services/message/app/schemas/schemas.py | 2 +- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/services/message/app/routers/conversations.py b/services/message/app/routers/conversations.py index 00b06f1..d5493ef 100644 --- a/services/message/app/routers/conversations.py +++ b/services/message/app/routers/conversations.py @@ -92,41 +92,53 @@ async def start_conversation( user_id: str = Depends(get_current_user_id), db: Session = Depends(get_db), ): - buyer_id = uuid.UUID(user_id) - farmer_id = uuid.UUID(payload.farmer_id) + initiator_id = uuid.UUID(user_id) + recipient_id = uuid.UUID(payload.recipient_id) - if buyer_id == farmer_id: + if initiator_id == recipient_id: raise HTTPException(status_code=400, detail="Cannot message yourself") - # Resume existing conversation if one already exists + # Resume existing conversation if one already exists (check both orderings) existing = db.query(Conversation).filter( - and_( - Conversation.buyer_id == buyer_id, - Conversation.farmer_id == farmer_id, + or_( + and_( + Conversation.buyer_id == initiator_id, + Conversation.farmer_id == recipient_id, + ), + and_( + Conversation.buyer_id == recipient_id, + Conversation.farmer_id == initiator_id, + ), ) ).first() if existing: + is_buyer_slot = str(existing.buyer_id) == user_id msg = Message( conversation_id=existing.id, - sender_id=buyer_id, - sender_name=existing.buyer_name, - sender_initials=existing.buyer_initials, + sender_id=initiator_id, + sender_name=existing.buyer_name if is_buyer_slot else existing.farmer_name, + sender_initials=existing.buyer_initials if is_buyer_slot else existing.farmer_initials, body=payload.first_message, ) db.add(msg) existing.last_message = payload.first_message existing.last_message_at = datetime.now(timezone.utc) - existing.last_sender_id = buyer_id - existing.farmer_unread += 1 + existing.last_sender_id = initiator_id + if is_buyer_slot: + existing.farmer_unread += 1 + else: + existing.buyer_unread += 1 db.commit() db.refresh(existing) return _conversation_response(existing, msg, viewer_id=user_id, is_new=False) - # Fetch user and listing snapshots in parallel + # Fetch user snapshots in parallel + buyer_id = initiator_id + farmer_id = recipient_id buyer, farmer = await asyncio.gather( fetch_user(user_id), - fetch_user(payload.farmer_id), + fetch_user(payload.recipient_id), ) listing_name = None diff --git a/services/message/app/schemas/schemas.py b/services/message/app/schemas/schemas.py index cdf865f..4cbeb5f 100644 --- a/services/message/app/schemas/schemas.py +++ b/services/message/app/schemas/schemas.py @@ -35,7 +35,7 @@ class MessageOut(BaseModel): class StartConversationPayload(BaseModel): - farmer_id: str + recipient_id: str listing_id: Optional[str] = None first_message: str From 72893b68b805a77423cafba48bccc88fe246ca33 Mon Sep 17 00:00:00 2001 From: the-icemann Date: Wed, 20 May 2026 00:39:24 +0300 Subject: [PATCH 14/24] feat: notify recipient when any conversation is initiated (all roles) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First message from start_conversation now triggers a notification to the recipient regardless of role — fixes the gap where only send_message fired a notification, leaving the initial contact silent. Co-Authored-By: Claude Sonnet 4.6 --- services/message/app/routers/conversations.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/services/message/app/routers/conversations.py b/services/message/app/routers/conversations.py index d5493ef..ee69673 100644 --- a/services/message/app/routers/conversations.py +++ b/services/message/app/routers/conversations.py @@ -22,6 +22,24 @@ # ── Internal helpers ────────────────────────────────────────────────────────── +async def _notify_new_message(recipient_id: str, sender_name: str, message_id: str) -> None: + try: + async with httpx.AsyncClient() as client: + await client.post( + f"{settings.NOTIFICATION_SERVICE_URL}/internal/notify", + json={ + "event": "new_message", + "actor_id": recipient_id, + "actor_name": sender_name, + "message_id": message_id, + }, + headers={"x-internal-secret": settings.INTERNAL_SECRET}, + timeout=3.0, + ) + except Exception as e: + logger.warning(f"Notification failed for conversation message {message_id}: {e}") + + async def fetch_user(user_id: str) -> dict: try: async with httpx.AsyncClient() as client: @@ -131,6 +149,11 @@ async def start_conversation( existing.buyer_unread += 1 db.commit() db.refresh(existing) + + notif_recipient = str(existing.farmer_id) if is_buyer_slot else str(existing.buyer_id) + notif_sender = existing.buyer_name if is_buyer_slot else existing.farmer_name + await _notify_new_message(notif_recipient, notif_sender, str(msg.id)) + return _conversation_response(existing, msg, viewer_id=user_id, is_new=False) # Fetch user snapshots in parallel @@ -177,6 +200,8 @@ async def start_conversation( db.refresh(conv) db.refresh(msg) + await _notify_new_message(payload.recipient_id, conv.buyer_name, str(msg.id)) + return _conversation_response(conv, msg, viewer_id=user_id, is_new=True) From c9f37a2b270c69bf3045c021caad7f7f9d861b06 Mon Sep 17 00:00:00 2001 From: the-icemann Date: Wed, 20 May 2026 01:05:57 +0300 Subject: [PATCH 15/24] fix: sync new users to ML feature store immediately on registration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, new users (any role) were invisible to the recommendation service for up to 15 minutes after signing up. A 'both' user would see empty recommendation cards and have no one to Connect with. After registration (email/password) or OAuth profile completion, a background task now calls POST /ingest/user-created, which upserts the user into farmer_features and/or buyer_features and triggers an immediate recommendation-service reload — so matches are available the moment the user hits the home page. Co-Authored-By: Claude Sonnet 4.6 --- services/auth/app/core/config.py | 5 ++-- services/auth/app/routers/auth.py | 43 ++++++++++++++++++++++++++++-- services/auth/app/routers/oauth.py | 16 ++++++++++- 3 files changed, 59 insertions(+), 5 deletions(-) diff --git a/services/auth/app/core/config.py b/services/auth/app/core/config.py index 4acc434..e54af15 100644 --- a/services/auth/app/core/config.py +++ b/services/auth/app/core/config.py @@ -12,8 +12,9 @@ class Settings(BaseSettings): GOOGLE_CLIENT_SECRET: str = "" GOOGLE_REDIRECT_URI: str = "http://localhost/auth/google/callback" FRONTEND_URL: str - USER_SERVICE_URL: str - REFRESH_TOKEN_EXPIRE_DAYS:int = 30 + USER_SERVICE_URL: str + INGEST_SERVICE_URL: str = "http://data-ingestion-service:8004" + REFRESH_TOKEN_EXPIRE_DAYS: int = 30 class Config: env_file = ".env" diff --git a/services/auth/app/routers/auth.py b/services/auth/app/routers/auth.py index 8ccaaeb..ba89f16 100644 --- a/services/auth/app/routers/auth.py +++ b/services/auth/app/routers/auth.py @@ -1,5 +1,5 @@ import logging -from fastapi import APIRouter, Depends, HTTPException, Response, Header +from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Response, Header from sqlalchemy.orm import Session from app.db.session import get_db from app.models.user import AuthCredential, UserRole as DBUserRole @@ -21,8 +21,36 @@ router = APIRouter(tags=["Auth"]) +async def _sync_user_to_ml( + user_id: str, + role: str, + full_name: str, + district: str | None, + specialties: list | None, + interests: list | None, +) -> None: + """Fire-and-forget: sync new user into the ML feature store immediately.""" + try: + async with httpx.AsyncClient() as client: + await client.post( + f"{settings.INGEST_SERVICE_URL}/ingest/user-created", + json={ + "id": user_id, + "role": role, + "full_name": full_name, + "district": district, + "specialties": specialties, + "interests": interests, + }, + headers={"x-internal-secret": settings.INTERNAL_SECRET}, + timeout=8.0, + ) + except Exception as e: + logger.warning(f"ML feature store sync failed for user {user_id}: {e}") + + @router.post("/register", response_model=LoginResponse, status_code=201) -async def register(payload: RegisterPayload, db: Session = Depends(get_db)): +async def register(payload: RegisterPayload, background_tasks: BackgroundTasks, db: Session = Depends(get_db)): # ── 1. Check for existing email if db.query(AuthCredential).filter(AuthCredential.email == payload.email).first(): @@ -81,6 +109,17 @@ async def register(payload: RegisterPayload, db: Session = Depends(get_db)): access_token = create_access_token(str(cred.id), cred.role.value, cred.email) refresh_token = create_refresh_token(str(cred.id)) + # Sync new user to ML feature store so recommendations are available immediately + background_tasks.add_task( + _sync_user_to_ml, + str(cred.id), + cred.role.value, + payload.fullName, + payload.district, + payload.specialties, + payload.interests, + ) + return LoginResponse( tokens=AuthTokens(access_token=access_token, refresh_token=refresh_token), user=AuthUserMinimal(id=str(cred.id), email=cred.email, role=cred.role.value) diff --git a/services/auth/app/routers/oauth.py b/services/auth/app/routers/oauth.py index 9f99827..206c674 100644 --- a/services/auth/app/routers/oauth.py +++ b/services/auth/app/routers/oauth.py @@ -1,5 +1,5 @@ import logging -from fastapi import APIRouter, Cookie, Depends, HTTPException, Request +from fastapi import APIRouter, BackgroundTasks, Cookie, Depends, HTTPException, Request from fastapi.responses import RedirectResponse, JSONResponse from sqlalchemy.orm import Session from authlib.integrations.starlette_client import OAuth @@ -16,6 +16,8 @@ from app.schemas.auth import CompleteProfileRequest import httpx +from .auth import _sync_user_to_ml + logger = logging.getLogger(__name__) router = APIRouter(tags=["OAuth"]) @@ -117,6 +119,7 @@ async def google_callback(request: Request, db: Session = Depends(get_db)): @router.post("/complete-profile") async def complete_profile( body: CompleteProfileRequest, + background_tasks: BackgroundTasks, db: Session = Depends(get_db), setup_token: str | None = Cookie(default=None), ): @@ -185,6 +188,17 @@ async def complete_profile( access_token = create_access_token(str(user.id), user.role.value, user.email) refresh_token = create_refresh_token(str(user.id)) + # Sync new OAuth user to ML feature store so recommendations work immediately + background_tasks.add_task( + _sync_user_to_ml, + str(user.id), + body.role.value, + name, + body.district, + body.specialties, + body.interests, + ) + response = JSONResponse({"message": "Profile complete", "role": user.role.value}) _set_auth_cookies(response, access_token, refresh_token) _clear_setup_cookie(response) From 36eae3db490c5a4b864e0d87adc3fd8ea5336798 Mon Sep 17 00:00:00 2001 From: the-icemann Date: Wed, 20 May 2026 01:13:18 +0300 Subject: [PATCH 16/24] fix: exclude requesting user from their own recommendations A 'both' role user appeared in their own AI-Matched Farmers and AI-Matched Buyers lists because the scorer iterated all profiles with no self-exclusion check. Applies to any role that exists in both feature tables. Co-Authored-By: Claude Sonnet 4.6 --- services/soko-ml/recommendation-service/src/recommender.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/soko-ml/recommendation-service/src/recommender.py b/services/soko-ml/recommendation-service/src/recommender.py index c2ce0dc..5dfb43f 100644 --- a/services/soko-ml/recommendation-service/src/recommender.py +++ b/services/soko-ml/recommendation-service/src/recommender.py @@ -127,6 +127,7 @@ def recommend_farmers_for_buyer(self, buyer_id: str, top_n: int = 5) -> list[dic scored = [ (self._score_farmer_for_buyer(farmer, buyer, buyer_id), farmer) for _, farmer in self.profiles.farmers.iterrows() + if str(farmer["id"]) != buyer_id ] scored.sort(key=lambda x: x[0], reverse=True) @@ -183,6 +184,7 @@ def recommend_buyers_for_farmer(self, farmer_id: str, top_n: int = 5) -> list[di scored = [ (self._score_buyer_for_farmer(buyer, farmer, max_spend), buyer) for _, buyer in self.profiles.buyers.iterrows() + if str(buyer["id"]) != farmer_id ] scored.sort(key=lambda x: x[0], reverse=True) From a3ddb5b4f3a12a91b464fa87aecbadecd8155a6e Mon Sep 17 00:00:00 2001 From: the-icemann Date: Wed, 20 May 2026 01:23:13 +0300 Subject: [PATCH 17/24] feat: add EC2 API origin and per-service cache behaviors to CloudFront Routes all API path prefixes (/auth/*, /message/*, /recommendations/*, etc.) through to EC2 via HTTP while keeping HTTPS for the browser. S3 continues to serve static frontend assets. Uses nip.io so CloudFront has a stable hostname even before a custom domain is wired up. Co-Authored-By: Claude Sonnet 4.6 --- infrastructure/cloudfront.tf | 46 ++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/infrastructure/cloudfront.tf b/infrastructure/cloudfront.tf index d4dea8a..b30a8eb 100644 --- a/infrastructure/cloudfront.tf +++ b/infrastructure/cloudfront.tf @@ -6,6 +6,15 @@ resource "aws_cloudfront_origin_access_control" "frontend" { signing_protocol = "sigv4" } +locals { + # Path prefixes handled by NGINX on EC2 — must match nginx.conf location blocks exactly + api_path_prefixes = [ + "/auth/*", "/oauth/*", "/users/*", "/listings/*", "/orders/*", + "/payments/*", "/webhook/*", "/message/*", "/notifications/*", + "/posts/*", "/ussd/*", "/recommendations/*", "/ml/*", "/health*" + ] +} + resource "aws_cloudfront_distribution" "frontend" { enabled = true is_ipv6_enabled = true @@ -16,12 +25,26 @@ resource "aws_cloudfront_distribution" "frontend" { # depends_on = [aws_acm_certificate_validation.frontend] # uncomment with domain + # S3 origin — serves static frontend assets origin { domain_name = aws_s3_bucket.frontend.bucket_regional_domain_name origin_id = "s3-frontend" origin_access_control_id = aws_cloudfront_origin_access_control.frontend.id } + # EC2 origin — CloudFront talks HTTP to EC2 so the browser always uses HTTPS. + # nip.io maps ".nip.io" → that IP, giving CloudFront a valid hostname. + origin { + domain_name = "${aws_eip.soko.public_ip}.nip.io" + origin_id = "ec2-api" + custom_origin_config { + http_port = 80 + https_port = 443 + origin_protocol_policy = "http-only" + origin_ssl_protocols = ["TLSv1.2"] + } + } + default_cache_behavior { target_origin_id = "s3-frontend" viewer_protocol_policy = "redirect-to-https" @@ -58,6 +81,29 @@ resource "aws_cloudfront_distribution" "frontend" { max_ttl = 31536000 } + # API behaviors — route each service path prefix through to EC2, no caching + dynamic "ordered_cache_behavior" { + for_each = local.api_path_prefixes + content { + path_pattern = ordered_cache_behavior.value + target_origin_id = "ec2-api" + viewer_protocol_policy = "redirect-to-https" + allowed_methods = ["GET", "HEAD", "OPTIONS", "PUT", "PATCH", "POST", "DELETE"] + cached_methods = ["GET", "HEAD"] + compress = false + + forwarded_values { + query_string = true + headers = ["Authorization", "Content-Type", "Accept", "Origin", "X-User-Id", "X-User-Role"] + cookies { forward = "all" } + } + + min_ttl = 0 + default_ttl = 0 + max_ttl = 0 + } + } + # SPA routing: 404 from S3 → serve index.html so React Router handles it custom_error_response { error_code = 404 From ce11d0475776647d4fd684a827f1a7563bdcd4b1 Mon Sep 17 00:00:00 2001 From: the-icemann Date: Wed, 20 May 2026 10:49:16 +0300 Subject: [PATCH 18/24] feat: notify farmer when buyer is interested but no listings exist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit POST /listings/farmer/{id}/request-listing lets the frontend signal that a buyer clicked Connect on a recommended farmer who has no active listings. The endpoint fetches the buyer's name and fires a buyer_interest event to the notification service, which maps it to the system/farmer template and delivers an in-app push routed to the farmer's /sell page. Notification service: added farmer variant to the system template (entity_type=sell) and a buyer_interest handler in internal.py that delivers via event=system, role=farmer — avoids a Postgres enum migration. Co-Authored-By: Claude Sonnet 4.6 --- .../notification/app/helpers/templates.py | 5 +++ services/notification/app/routers/internal.py | 15 +++++++ services/produce/app/core/config.py | 1 + services/produce/app/routers/listings.py | 39 +++++++++++++++++++ 4 files changed, 60 insertions(+) diff --git a/services/notification/app/helpers/templates.py b/services/notification/app/helpers/templates.py index 3660785..e0954a4 100644 --- a/services/notification/app/helpers/templates.py +++ b/services/notification/app/helpers/templates.py @@ -100,6 +100,11 @@ def get_template(event: str, role: str, meta: dict = {}) -> Optional[Notificatio body=message or "Welcome to Soko!", entity_type=None, ), + "farmer": NotificationTemplate( + title="A buyer is looking for your produce!", + body=message or "Someone is interested in what you grow but you have no active listings. Add one now!", + entity_type="sell", + ), }, } diff --git a/services/notification/app/routers/internal.py b/services/notification/app/routers/internal.py index ae447f8..aeaf791 100644 --- a/services/notification/app/routers/internal.py +++ b/services/notification/app/routers/internal.py @@ -150,6 +150,21 @@ async def notify(payload: NotifyPayload, db: Session = Depends(get_db)): meta=meta, ) + # ── Buyer-interest notification (buyer interested in farmer with no listings) + if event == "buyer_interest" and payload.farmer_id: + meta["message"] = ( + f"{payload.actor_name or 'A buyer'} is interested in your produce " + "but you have no active listings. Add one now to connect!" + ) + await deliver( + db=db, + user_id=payload.farmer_id, + event="system", + role="farmer", + entity_id=None, + meta=meta, + ) + # ── System notification (used by USSD welcome SMS) if event == "system": target_id = payload.buyer_id or payload.farmer_id or payload.actor_id diff --git a/services/produce/app/core/config.py b/services/produce/app/core/config.py index 41ea871..7c232a8 100644 --- a/services/produce/app/core/config.py +++ b/services/produce/app/core/config.py @@ -9,6 +9,7 @@ class Settings(BaseSettings): CLOUDINARY_API_KEY: str CLOUDINARY_API_SECRET: str ML_GATEWAY_URL: str = "" # Set to http://ml-gateway:8000 when ML stack is running + NOTIFICATION_SERVICE_URL: str = "http://notification_service:8007" class Config: env_file = ".env" diff --git a/services/produce/app/routers/listings.py b/services/produce/app/routers/listings.py index 013e48e..c291a5b 100644 --- a/services/produce/app/routers/listings.py +++ b/services/produce/app/routers/listings.py @@ -127,6 +127,45 @@ def get_farmer_listings( return result +@router.post("/farmer/{farmer_id}/request-listing", status_code=200) +async def request_listing_from_farmer( + farmer_id: str, + buyer_id: str = Depends(get_current_user_id), + db: Session = Depends(get_db), +): + """Buyer signals interest in a farmer who has no active listings — notifies the farmer.""" + has_listing = db.query(Listing.id).filter( + Listing.farmer_id == uuid.UUID(farmer_id), + Listing.status == ListingStatus.active, + ).first() + + if has_listing: + return {"notified": False, "reason": "farmer_has_listings"} + + try: + buyer_info = await fetch_farmer_snapshot(buyer_id) + buyer_name = buyer_info.get("full_name") or buyer_info.get("name") or "A buyer" + except Exception: + buyer_name = "A buyer" + + try: + async with httpx.AsyncClient() as client: + await client.post( + f"{settings.NOTIFICATION_SERVICE_URL}/internal/notify", + json={ + "event": "buyer_interest", + "farmer_id": farmer_id, + "actor_name": buyer_name, + }, + headers={"x-internal-secret": settings.INTERNAL_SECRET}, + timeout=4.0, + ) + except Exception as e: + logger.warning(f"Failed to notify farmer {farmer_id} of buyer interest: {e}") + + return {"notified": True} + + # ── Farmer — my own listings (all statuses) @router.get("/me", response_model=list[ListingOut]) def get_my_listings( From 0d0a951845a55757f6be359ef6dbeeed9ebb1b8b Mon Sep 17 00:00:00 2001 From: the-icemann Date: Wed, 20 May 2026 13:45:19 +0300 Subject: [PATCH 19/24] fix: decouple fill-envs from seed; add NOTIFICATION_SERVICE_URL to produce MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fill-envs is a local-dev helper — running it automatically during 'make seed' on EC2 is unnecessary (secrets come from AWS Secrets Manager) and confusing. Seed now runs standalone; fill-envs is opt-in for local dev. Also wires NOTIFICATION_SERVICE_URL into the produce service docker-compose environment and fill_envs defaults so the buyer-interest notification endpoint has an explicit URL rather than relying solely on the config default. Co-Authored-By: Claude Sonnet 4.6 --- Makefile | 3 ++- docker-compose.yml | 9 +++++---- scripts/fill_envs.py | 15 ++++++++------- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 4dc86ed..9696fde 100644 --- a/Makefile +++ b/Makefile @@ -635,13 +635,14 @@ port-reference: fill-envs: @python3 scripts/fill_envs.py -seed: fill-envs +seed: @echo "" @echo "╔══════════════════════════════════════════════════════════════════════╗" @echo "║ Soko — Seeding all services ║" @echo "╚══════════════════════════════════════════════════════════════════════╝" @echo "" @echo "Both stacks must be running: make start" + @echo "For local dev, run 'make fill-envs' first if .env files are missing." @echo "" @python3 scripts/seed.py diff --git a/docker-compose.yml b/docker-compose.yml index c54857f..541541a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -160,10 +160,11 @@ services: - "127.0.0.1:8003:8003" env_file: ./services/produce/.env environment: - DATABASE_URL: postgresql://produce_user:${PRODUCE_DB_PASS:-produce_pass}@produce_db:5432/produce_db - REDIS_URL: redis://redis:6379/0 - USER_SERVICE_URL: http://user_service:8002 - ML_GATEWAY_URL: http://ml-gateway:8000 + DATABASE_URL: postgresql://produce_user:${PRODUCE_DB_PASS:-produce_pass}@produce_db:5432/produce_db + REDIS_URL: redis://redis:6379/0 + USER_SERVICE_URL: http://user_service:8002 + ML_GATEWAY_URL: http://ml-gateway:8000 + NOTIFICATION_SERVICE_URL: http://notification_service:8007 depends_on: produce_db: condition: service_healthy diff --git a/scripts/fill_envs.py b/scripts/fill_envs.py index d8ed246..8d205a0 100644 --- a/scripts/fill_envs.py +++ b/scripts/fill_envs.py @@ -36,13 +36,14 @@ "AUTH_SERVICE_URL": "http://auth_service:8001", }, "produce": { - "DATABASE_URL": "postgresql://produce_user:produce_pass@produce_db:5432/produce_db", - "INTERNAL_SECRET": INTERNAL_SECRET, - "USER_SERVICE_URL": "http://user_service:8002", - "REDIS_URL": "redis://redis:6379/0", - "CLOUDINARY_CLOUD_NAME": "", - "CLOUDINARY_API_KEY": "", - "CLOUDINARY_API_SECRET": "", + "DATABASE_URL": "postgresql://produce_user:produce_pass@produce_db:5432/produce_db", + "INTERNAL_SECRET": INTERNAL_SECRET, + "USER_SERVICE_URL": "http://user_service:8002", + "REDIS_URL": "redis://redis:6379/0", + "NOTIFICATION_SERVICE_URL": "http://notification_service:8007", + "CLOUDINARY_CLOUD_NAME": "", + "CLOUDINARY_API_KEY": "", + "CLOUDINARY_API_SECRET": "", }, "order": { "DATABASE_URL": "postgresql://order_user:order_pass@order_db:5432/order_db", From 2d9c3d2b7b89ff5554db9ffeb0132b2de8487284 Mon Sep 17 00:00:00 2001 From: the-icemann Date: Wed, 20 May 2026 17:32:06 +0300 Subject: [PATCH 20/24] fix: resolve service startup failures on local docker compose up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit notification_service: pydantic-settings 2.x defaults to extra='forbid', crashing when .env has stale keys (sendgrid_*) removed from the model. Add extra='ignore' to tolerate them. produce_service + blog_service: CLOUDINARY_* were required str fields with no defaults, crashing startup if the local .env is absent. Default to "" so services start without credentials — image uploads degrade gracefully, production values from Secrets Manager still override. message_service: INTERNAL_SECRET and SECRET_KEY had no defaults, failing without a populated env_file locally. Python-level defaults kick in only when the env var is completely absent; real production values always win. Co-Authored-By: Claude Sonnet 4.6 --- services/blog/app/core/config.py | 6 +++--- services/message/app/core/config.py | 4 ++-- services/notification/app/core/config.py | 1 + services/produce/app/core/config.py | 6 +++--- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/services/blog/app/core/config.py b/services/blog/app/core/config.py index 49fbc44..14df2a7 100644 --- a/services/blog/app/core/config.py +++ b/services/blog/app/core/config.py @@ -8,9 +8,9 @@ class Settings(BaseSettings): USER_SERVICE_URL: str = "http://user_service:8002" # Cloudinary - CLOUDINARY_CLOUD_NAME: str - CLOUDINARY_API_KEY: str - CLOUDINARY_API_SECRET: str + CLOUDINARY_CLOUD_NAME: str = "" + CLOUDINARY_API_KEY: str = "" + CLOUDINARY_API_SECRET: str = "" class Config: env_file = ".env" diff --git a/services/message/app/core/config.py b/services/message/app/core/config.py index 711a555..b11d858 100644 --- a/services/message/app/core/config.py +++ b/services/message/app/core/config.py @@ -3,8 +3,8 @@ class Settings(BaseSettings): DATABASE_URL: str - INTERNAL_SECRET: str - SECRET_KEY: str + INTERNAL_SECRET: str = "internal-secret" + SECRET_KEY: str = "soko-dev-secret-key-2026-change-before-production" USER_SERVICE_URL: str = "http://user-service:8002" PRODUCE_SERVICE_URL: str = "http://produce-service:8003" NOTIFICATION_SERVICE_URL: str = "http://notification-service:8008" diff --git a/services/notification/app/core/config.py b/services/notification/app/core/config.py index 955759c..e34c943 100644 --- a/services/notification/app/core/config.py +++ b/services/notification/app/core/config.py @@ -17,6 +17,7 @@ class Settings(BaseSettings): class Config: env_file = ".env" + extra = "ignore" # tolerate stale keys in .env (e.g. removed sendgrid fields) settings = Settings() \ No newline at end of file diff --git a/services/produce/app/core/config.py b/services/produce/app/core/config.py index 7c232a8..6cd48d7 100644 --- a/services/produce/app/core/config.py +++ b/services/produce/app/core/config.py @@ -5,9 +5,9 @@ class Settings(BaseSettings): INTERNAL_SECRET: str USER_SERVICE_URL: str = "http://user_service:8002" REDIS_URL: str = "redis://redis:6379/0" - CLOUDINARY_CLOUD_NAME: str - CLOUDINARY_API_KEY: str - CLOUDINARY_API_SECRET: str + CLOUDINARY_CLOUD_NAME: str = "" + CLOUDINARY_API_KEY: str = "" + CLOUDINARY_API_SECRET: str = "" ML_GATEWAY_URL: str = "" # Set to http://ml-gateway:8000 when ML stack is running NOTIFICATION_SERVICE_URL: str = "http://notification_service:8007" From 891c8acb0c135450181b53c397c14ed47a005930 Mon Sep 17 00:00:00 2001 From: the-icemann Date: Wed, 20 May 2026 20:30:21 +0300 Subject: [PATCH 21/24] fix: wire up Google Sign-In and fix auth route serving - CloudFront: add explicit S3 behaviors for /auth/sign-in, /auth/sign-up, /auth/complete-profile before the /auth/* EC2 catch-all so SPA pages are served from S3 on direct navigation and page reload - oauth.py: returning Google users now redirect to /auth/complete-profile?access_token= so the frontend zustand store gets the JWT (httpOnly cookie approach was incompatible with the Bearer token auth model used everywhere else) - oauth.py: complete_profile POST now returns access_token in JSON body so the SPA can hydrate the store after new-user profile completion - secrets.tf: add FRONTEND_URL and GOOGLE_REDIRECT_URI as first-class secrets (were incorrectly derived from a non-existent DOMAIN key) - fetch-secrets.sh: read FRONTEND_URL/GOOGLE_REDIRECT_URI directly from Secrets Manager; fix s() helper to return empty string on missing keys Co-Authored-By: Claude Sonnet 4.6 --- infrastructure/cloudfront.tf | 54 ++++++++++++++++++++++++++++++ infrastructure/secrets.tf | 4 +++ scripts/fetch-secrets.sh | 20 +++++++---- services/auth/app/routers/oauth.py | 18 ++++++---- 4 files changed, 82 insertions(+), 14 deletions(-) diff --git a/infrastructure/cloudfront.tf b/infrastructure/cloudfront.tf index b30a8eb..8d49f0d 100644 --- a/infrastructure/cloudfront.tf +++ b/infrastructure/cloudfront.tf @@ -62,6 +62,60 @@ resource "aws_cloudfront_distribution" "frontend" { max_ttl = 31536000 } + # SPA auth pages — these share the /auth/ prefix with backend API routes. + # List them explicitly BEFORE the /auth/* EC2 behavior so CloudFront serves + # them from S3 instead of proxying to NGINX. Deep-links and page-reloads work + # because S3 returns 404→index.html via the custom_error_response below. + ordered_cache_behavior { + path_pattern = "/auth/sign-in" + target_origin_id = "s3-frontend" + viewer_protocol_policy = "redirect-to-https" + allowed_methods = ["GET", "HEAD"] + cached_methods = ["GET", "HEAD"] + compress = true + forwarded_values { + query_string = false + cookies { forward = "none" } + } + min_ttl = 0 + default_ttl = 0 + max_ttl = 0 + } + + ordered_cache_behavior { + path_pattern = "/auth/sign-up" + target_origin_id = "s3-frontend" + viewer_protocol_policy = "redirect-to-https" + allowed_methods = ["GET", "HEAD"] + cached_methods = ["GET", "HEAD"] + compress = true + forwarded_values { + query_string = false + cookies { forward = "none" } + } + min_ttl = 0 + default_ttl = 0 + max_ttl = 0 + } + + # complete-profile: forward query_string so the ?access_token= param (set by + # the backend after Google OAuth for returning users) reaches the SPA. + ordered_cache_behavior { + path_pattern = "/auth/complete-profile" + target_origin_id = "s3-frontend" + viewer_protocol_policy = "redirect-to-https" + allowed_methods = ["GET", "HEAD"] + cached_methods = ["GET", "HEAD"] + compress = true + forwarded_values { + query_string = true + cookies { forward = "none" } + } + min_ttl = 0 + default_ttl = 0 + max_ttl = 0 + } + # Cache JS/CSS/images aggressively (content-hashed filenames by Vite) ordered_cache_behavior { path_pattern = "/assets/*" diff --git a/infrastructure/secrets.tf b/infrastructure/secrets.tf index e55a6b0..94fafc6 100644 --- a/infrastructure/secrets.tf +++ b/infrastructure/secrets.tf @@ -29,6 +29,10 @@ resource "aws_secretsmanager_secret_version" "platform" { INTERNAL_SECRET = "CHANGE_ME" ALGORITHM = "HS256" + # ── App URLs (set to your CloudFront domain until soko-ug.com is wired up) ── + FRONTEND_URL = "CHANGE_ME" + GOOGLE_REDIRECT_URI = "CHANGE_ME/auth/google/callback" + # ── Google OAuth ────────────────────────────────────────────────────────── GOOGLE_CLIENT_ID = "CHANGE_ME" GOOGLE_CLIENT_SECRET = "CHANGE_ME" diff --git a/scripts/fetch-secrets.sh b/scripts/fetch-secrets.sh index 06fc848..5bea8b2 100755 --- a/scripts/fetch-secrets.sh +++ b/scripts/fetch-secrets.sh @@ -20,8 +20,8 @@ RAW=$(aws secretsmanager get-secret-value \ --query "SecretString" \ --output text) -# Helper: extract a key from the JSON -s() { echo "$RAW" | jq -r ".${1}"; } +# Helper: extract a key (returns empty string if null or missing) +s() { echo "$RAW" | jq -r "(.${1} // empty)"; } # ── Root .env (docker-compose substitution) ─────────────────────────────────── # Docker Compose auto-loads this file for ${VAR} substitution @@ -40,14 +40,20 @@ EOF chmod 600 "$REPO_DIR/.env" echo "[fetch-secrets] Root .env written." +# Resolve app base URL — read directly from secrets manager +FRONTEND_URL="$(s FRONTEND_URL)" +GOOGLE_REDIRECT_URI="$(s GOOGLE_REDIRECT_URI)" +# Fall back to deriving GOOGLE_REDIRECT_URI from FRONTEND_URL if not set separately +: "${GOOGLE_REDIRECT_URI:=${FRONTEND_URL}/auth/google/callback}" + # ── Auth Service ────────────────────────────────────────────────────────────── cat > "$REPO_DIR/services/auth/.env" </dev/null || echo "yourdomain.com")/auth/google/callback -FRONTEND_URL=https://$(s DOMAIN 2>/dev/null || echo "yourdomain.com") +GOOGLE_REDIRECT_URI=${GOOGLE_REDIRECT_URI} +FRONTEND_URL=${FRONTEND_URL} INTERNAL_SECRET=$(s INTERNAL_SECRET) USER_SERVICE_URL=http://user_service:8002 EOF @@ -59,12 +65,12 @@ INTERNAL_SECRET=$(s INTERNAL_SECRET) PESAPAL_CONSUMER_KEY=$(s PESAPAL_CONSUMER_KEY) PESAPAL_CONSUMER_SECRET=$(s PESAPAL_CONSUMER_SECRET) PESAPAL_ENV=$(s PESAPAL_ENV) -PESAPAL_IPN_URL=https://$(s DOMAIN 2>/dev/null || echo "yourdomain.com")/payments/webhook/pesapal/ipn -PESAPAL_CALLBACK_URL=https://$(s DOMAIN 2>/dev/null || echo "yourdomain.com")/payments/callback +PESAPAL_IPN_URL=${FRONTEND_URL}/payments/webhook/pesapal/ipn +PESAPAL_CALLBACK_URL=${FRONTEND_URL}/payments/callback ORDER_SERVICE_URL=http://order_service:8004 USER_SERVICE_URL=http://user_service:8002 NOTIFICATION_SERVICE_URL=http://notification_service:8007 -FRONTEND_URL=https://$(s DOMAIN 2>/dev/null || echo "yourdomain.com") +FRONTEND_URL=${FRONTEND_URL} EOF chmod 600 "$REPO_DIR/services/payment/.env" diff --git a/services/auth/app/routers/oauth.py b/services/auth/app/routers/oauth.py index 206c674..a6d6a42 100644 --- a/services/auth/app/routers/oauth.py +++ b/services/auth/app/routers/oauth.py @@ -73,12 +73,12 @@ async def google_callback(request: Request, db: Session = Depends(get_db)): detail="An account with this email already exists. Please log in with your password." ) - # Returning OAuth user — issue real tokens immediately - access_token = create_access_token(str(user.id), user.role.value, user.email) - refresh_token = create_refresh_token(str(user.id)) - response = RedirectResponse(url=f"{settings.FRONTEND_URL}/marketplace") - _set_auth_cookies(response, access_token, refresh_token) - return response + # Returning OAuth user — pass token to SPA via query param so the + # frontend zustand store can pick it up without relying on httpOnly cookies. + access_token = create_access_token(str(user.id), user.role.value, user.email) + return RedirectResponse( + url=f"{settings.FRONTEND_URL}/auth/complete-profile?access_token={access_token}" + ) # ── 3. New user — skeleton credential, no commit yet user = AuthCredential( @@ -199,7 +199,11 @@ async def complete_profile( body.interests, ) - response = JSONResponse({"message": "Profile complete", "role": user.role.value}) + response = JSONResponse({ + "message": "Profile complete", + "role": user.role.value, + "access_token": access_token, + }) _set_auth_cookies(response, access_token, refresh_token) _clear_setup_cookie(response) return response From fcf9c974fc5880aed3b97a16136c090af2ccd0a7 Mon Sep 17 00:00:00 2001 From: the-icemann Date: Thu, 21 May 2026 13:44:10 +0300 Subject: [PATCH 22/24] infra: remove secret version resources from Terraform Terraform was overwriting real secret values with the CHANGE_ME skeleton on terraform apply. Fix: Terraform now only manages the secret container (name, tags, recovery window). All secret values are managed exclusively via AWS Console or CLI and can never be clobbered by a plan/apply. --- infrastructure/secrets.tf | 81 ++++++++------------------------------- 1 file changed, 15 insertions(+), 66 deletions(-) diff --git a/infrastructure/secrets.tf b/infrastructure/secrets.tf index 94fafc6..2f0dfdf 100644 --- a/infrastructure/secrets.tf +++ b/infrastructure/secrets.tf @@ -1,4 +1,18 @@ # ── Platform secrets (all core services) ───────────────────────────────────── +# Terraform owns the secret container only (name, tags, recovery window). +# Secret VALUES are managed exclusively via AWS Console or CLI — never here. +# Run scripts/fetch-secrets.sh on EC2 after updating values. +# +# Keys to populate: +# AUTH_DB_PASS, USER_DB_PASS, PRODUCE_DB_PASS, ORDER_DB_PASS, +# PAYMENT_DB_PASS, MESSAGE_DB_PASS, NOTIFICATION_DB_PASS, +# BLOG_DB_PASS, USSD_DB_PASS, +# SECRET_KEY, INTERNAL_SECRET, ALGORITHM, +# FRONTEND_URL, GOOGLE_REDIRECT_URI, +# GOOGLE_CLIENT_ID, GOOGLE_CLIENT_SECRET, +# PESAPAL_CONSUMER_KEY, PESAPAL_CONSUMER_SECRET, PESAPAL_ENV, +# AT_USERNAME, AT_API_KEY, AT_SENDER_ID, +# SENDGRID_API_KEY, SENDGRID_FROM_EMAIL resource "aws_secretsmanager_secret" "platform" { name = "soko/platform" description = "All Soko core platform secrets" @@ -7,58 +21,8 @@ resource "aws_secretsmanager_secret" "platform" { tags = { Name = "soko-platform-secrets" } } -# Skeleton — set real values via AWS Console or `aws secretsmanager put-secret-value` -# after `terraform apply`. Do NOT put real secrets in this file. -resource "aws_secretsmanager_secret_version" "platform" { - secret_id = aws_secretsmanager_secret.platform.id - - secret_string = jsonencode({ - # ── Database passwords ─────────────────────────────────────────────────── - AUTH_DB_PASS = "CHANGE_ME" - USER_DB_PASS = "CHANGE_ME" - PRODUCE_DB_PASS = "CHANGE_ME" - ORDER_DB_PASS = "CHANGE_ME" - PAYMENT_DB_PASS = "CHANGE_ME" - MESSAGE_DB_PASS = "CHANGE_ME" - NOTIFICATION_DB_PASS = "CHANGE_ME" - BLOG_DB_PASS = "CHANGE_ME" - USSD_DB_PASS = "CHANGE_ME" - - # ── Auth / JWT ──────────────────────────────────────────────────────────── - SECRET_KEY = "CHANGE_ME" - INTERNAL_SECRET = "CHANGE_ME" - ALGORITHM = "HS256" - - # ── App URLs (set to your CloudFront domain until soko-ug.com is wired up) ── - FRONTEND_URL = "CHANGE_ME" - GOOGLE_REDIRECT_URI = "CHANGE_ME/auth/google/callback" - - # ── Google OAuth ────────────────────────────────────────────────────────── - GOOGLE_CLIENT_ID = "CHANGE_ME" - GOOGLE_CLIENT_SECRET = "CHANGE_ME" - - # ── PesaPal (Payments) ──────────────────────────────────────────────────── - PESAPAL_CONSUMER_KEY = "CHANGE_ME" - PESAPAL_CONSUMER_SECRET = "CHANGE_ME" - PESAPAL_ENV = "production" - - # ── Africa's Talking (USSD + SMS) ───────────────────────────────────────── - AT_USERNAME = "CHANGE_ME" - AT_API_KEY = "CHANGE_ME" - AT_SENDER_ID = "SOKO" - - # ── SendGrid / Email ────────────────────────────────────────────────────── - SENDGRID_API_KEY = "CHANGE_ME" - SENDGRID_FROM_EMAIL = "noreply@CHANGE_ME" - }) - - lifecycle { - # Terraform manages the secret skeleton only — real values updated externally - ignore_changes = [secret_string] - } -} - # ── ML stack secrets ────────────────────────────────────────────────────────── +# Keys: ML_DB_PASS, ML_REDIS_PASSWORD, INTERNAL_API_KEY, GOOGLE_MAPS_API_KEY resource "aws_secretsmanager_secret" "ml" { name = "soko/ml" description = "Soko ML layer secrets" @@ -66,18 +30,3 @@ resource "aws_secretsmanager_secret" "ml" { tags = { Name = "soko-ml-secrets" } } - -resource "aws_secretsmanager_secret_version" "ml" { - secret_id = aws_secretsmanager_secret.ml.id - - secret_string = jsonencode({ - ML_DB_PASS = "CHANGE_ME" - ML_REDIS_PASSWORD = "" - INTERNAL_API_KEY = "CHANGE_ME" - GOOGLE_MAPS_API_KEY = "CHANGE_ME" - }) - - lifecycle { - ignore_changes = [secret_string] - } -} From a3c8e351d2f32417f9eb44eb799a8495e69761bd Mon Sep 17 00:00:00 2001 From: the-icemann Date: Thu, 21 May 2026 14:19:58 +0300 Subject: [PATCH 23/24] infra: wire up soko-ug.com domain with ACM + CloudFront + Route 53 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - main.tf: enable us-east-1 provider alias for ACM - acm.tf: rewrite using data source for existing Route 53 zone (Amazon Registrar already created it — avoids duplicate zone); cert covers soko-ug.com + www; alias A records point both to CloudFront - cloudfront.tf: update aliases to soko-ug.com/www, wire ACM cert, enable depends_on cert validation - nginx.conf: add soko-ug.com to CORS allowed origins --- infrastructure/acm.tf | 166 ++++++++++++++++------------------- infrastructure/cloudfront.tf | 14 ++- infrastructure/main.tf | 26 +++--- nginx/nginx.conf | 3 +- 4 files changed, 98 insertions(+), 111 deletions(-) diff --git a/infrastructure/acm.tf b/infrastructure/acm.tf index da3c6f2..2b1a413 100644 --- a/infrastructure/acm.tf +++ b/infrastructure/acm.tf @@ -1,89 +1,79 @@ -# Domain + ACM certificate setup — uncomment when soko-ug.com NS propagation is confirmed. -# To re-enable: set domain_name = "soko-ug.com" in terraform.tfvars and uncomment below. +# ── Domain + ACM certificate ─────────────────────────────────────────────────── +# The domain was registered through Amazon Registrar so Route 53 already has +# a hosted zone — reference it with a data source instead of creating a new one. -# resource "aws_acm_certificate" "frontend" { -# count = var.domain_name != "" ? 1 : 0 -# provider = aws.us_east_1 -# -# domain_name = "app.${var.domain_name}" -# subject_alternative_names = ["${var.domain_name}", "www.${var.domain_name}"] -# validation_method = "DNS" -# -# lifecycle { -# create_before_destroy = true -# } -# -# tags = { Name = "soko-frontend-cert" } -# } -# -# resource "aws_route53_zone" "main" { -# count = var.domain_name != "" ? 1 : 0 -# name = var.domain_name -# tags = { Name = "soko-zone" } -# } -# -# resource "aws_route53_record" "cert_validation" { -# for_each = var.domain_name != "" ? { -# for dvo in aws_acm_certificate.frontend[0].domain_validation_options : dvo.domain_name => { -# name = dvo.resource_record_name -# type = dvo.resource_record_type -# record = dvo.resource_record_value -# } -# } : {} -# -# zone_id = aws_route53_zone.main[0].zone_id -# name = each.value.name -# type = each.value.type -# ttl = 60 -# records = [each.value.record] -# } -# -# resource "aws_acm_certificate_validation" "frontend" { -# count = var.domain_name != "" ? 1 : 0 -# provider = aws.us_east_1 -# certificate_arn = aws_acm_certificate.frontend[0].arn -# validation_record_fqdns = [for r in aws_route53_record.cert_validation : r.fqdn] -# } -# -# resource "aws_route53_record" "root" { -# count = var.domain_name != "" ? 1 : 0 -# zone_id = aws_route53_zone.main[0].zone_id -# name = var.domain_name -# type = "A" -# ttl = 300 -# records = [aws_eip.soko.public_ip] -# } -# -# resource "aws_route53_record" "www" { -# count = var.domain_name != "" ? 1 : 0 -# zone_id = aws_route53_zone.main[0].zone_id -# name = "www.${var.domain_name}" -# type = "A" -# ttl = 300 -# records = [aws_eip.soko.public_ip] -# } -# -# resource "aws_route53_record" "app" { -# count = var.domain_name != "" ? 1 : 0 -# zone_id = aws_route53_zone.main[0].zone_id -# name = "app.${var.domain_name}" -# type = "CNAME" -# ttl = 300 -# records = [aws_cloudfront_distribution.frontend.domain_name] -# } -# -# output "route53_nameservers" { -# description = "Point your domain registrar to these nameservers" -# value = var.domain_name != "" ? aws_route53_zone.main[0].name_servers : [] -# } -# -# output "acm_validation_records" { -# description = "Add these DNS records to validate your ACM certificate" -# value = var.domain_name != "" ? { -# for dvo in aws_acm_certificate.frontend[0].domain_validation_options : dvo.domain_name => { -# name = dvo.resource_record_name -# type = dvo.resource_record_type -# value = dvo.resource_record_value -# } -# } : {} -# } +data "aws_route53_zone" "main" { + count = var.domain_name != "" ? 1 : 0 + name = var.domain_name + private_zone = false +} + +# ACM certificate — must be in us-east-1 for CloudFront +resource "aws_acm_certificate" "frontend" { + count = var.domain_name != "" ? 1 : 0 + provider = aws.us_east_1 + + domain_name = var.domain_name + subject_alternative_names = ["www.${var.domain_name}"] + validation_method = "DNS" + + lifecycle { + create_before_destroy = true + } + + tags = { Name = "soko-frontend-cert" } +} + +# DNS validation records — written into the existing Route 53 zone +resource "aws_route53_record" "cert_validation" { + for_each = var.domain_name != "" ? { + for dvo in aws_acm_certificate.frontend[0].domain_validation_options : dvo.domain_name => { + name = dvo.resource_record_name + type = dvo.resource_record_type + record = dvo.resource_record_value + } + } : {} + + zone_id = data.aws_route53_zone.main[0].zone_id + name = each.value.name + type = each.value.type + ttl = 60 + records = [each.value.record] +} + +resource "aws_acm_certificate_validation" "frontend" { + count = var.domain_name != "" ? 1 : 0 + provider = aws.us_east_1 + certificate_arn = aws_acm_certificate.frontend[0].arn + validation_record_fqdns = [for r in aws_route53_record.cert_validation : r.fqdn] +} + +# ── DNS records ─────────────────────────────────────────────────────────────── + +# Root domain → CloudFront (alias A record — Route 53 supports this for apex domains) +resource "aws_route53_record" "root" { + count = var.domain_name != "" ? 1 : 0 + zone_id = data.aws_route53_zone.main[0].zone_id + name = var.domain_name + type = "A" + + alias { + name = aws_cloudfront_distribution.frontend.domain_name + zone_id = aws_cloudfront_distribution.frontend.hosted_zone_id + evaluate_target_health = false + } +} + +# www → CloudFront +resource "aws_route53_record" "www" { + count = var.domain_name != "" ? 1 : 0 + zone_id = data.aws_route53_zone.main[0].zone_id + name = "www.${var.domain_name}" + type = "A" + + alias { + name = aws_cloudfront_distribution.frontend.domain_name + zone_id = aws_cloudfront_distribution.frontend.hosted_zone_id + evaluate_target_health = false + } +} diff --git a/infrastructure/cloudfront.tf b/infrastructure/cloudfront.tf index 8d49f0d..efc14dc 100644 --- a/infrastructure/cloudfront.tf +++ b/infrastructure/cloudfront.tf @@ -21,9 +21,9 @@ resource "aws_cloudfront_distribution" "frontend" { default_root_object = "index.html" price_class = "PriceClass_200" # includes South Africa edge nodes - aliases = var.domain_name != "" ? ["app.${var.domain_name}"] : [] + aliases = var.domain_name != "" ? [var.domain_name, "www.${var.domain_name}"] : [] - # depends_on = [aws_acm_certificate_validation.frontend] # uncomment with domain + depends_on = [aws_acm_certificate_validation.frontend] # S3 origin — serves static frontend assets origin { @@ -178,12 +178,10 @@ resource "aws_cloudfront_distribution" "frontend" { } viewer_certificate { - cloudfront_default_certificate = true - # Uncomment below and remove line above when domain is ready: - # acm_certificate_arn = aws_acm_certificate_validation.frontend[0].certificate_arn - # ssl_support_method = "sni-only" - # minimum_protocol_version = "TLSv1.2_2021" - # cloudfront_default_certificate = false + cloudfront_default_certificate = var.domain_name == "" + acm_certificate_arn = var.domain_name != "" ? aws_acm_certificate_validation.frontend[0].certificate_arn : null + ssl_support_method = var.domain_name != "" ? "sni-only" : null + minimum_protocol_version = var.domain_name != "" ? "TLSv1.2_2021" : null } tags = { Name = "soko-frontend-cdn" } diff --git a/infrastructure/main.tf b/infrastructure/main.tf index 2dd74fe..602acd2 100644 --- a/infrastructure/main.tf +++ b/infrastructure/main.tf @@ -31,16 +31,16 @@ provider "aws" { } } -# CloudFront ACM certificates must live in us-east-1 — uncomment when adding domain -# provider "aws" { -# alias = "us_east_1" -# region = "us-east-1" -# -# default_tags { -# tags = { -# Project = "soko" -# Environment = var.environment -# ManagedBy = "terraform" -# } -# } -# } +# CloudFront ACM certificates must live in us-east-1 +provider "aws" { + alias = "us_east_1" + region = "us-east-1" + + default_tags { + tags = { + Project = "soko" + Environment = var.environment + ManagedBy = "terraform" + } + } +} diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 6210eae..1ac2b7a 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -82,8 +82,7 @@ http { map $http_origin $cors_origin { ~^https?://localhost:(3000|5173|4173)$ $http_origin; ~^https://[a-z0-9]+\.cloudfront\.net$ $http_origin; - # Uncomment when domain is live: - # https://app.soko-ug.com $http_origin; + ~^https://(www\.)?soko-ug\.com$ $http_origin; default ""; } From 6be27a435d175ec8a64ab60a78e1056479201668 Mon Sep 17 00:00:00 2001 From: the-icemann Date: Thu, 21 May 2026 16:23:14 +0300 Subject: [PATCH 24/24] fixed dns resolution in Route 53 to map domain as soko-ug.com --- infrastructure/acm.tf | 18 ++-- infrastructure/bugs.txt | 212 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 222 insertions(+), 8 deletions(-) create mode 100644 infrastructure/bugs.txt diff --git a/infrastructure/acm.tf b/infrastructure/acm.tf index 2b1a413..3eafe9c 100644 --- a/infrastructure/acm.tf +++ b/infrastructure/acm.tf @@ -52,10 +52,11 @@ resource "aws_acm_certificate_validation" "frontend" { # Root domain → CloudFront (alias A record — Route 53 supports this for apex domains) resource "aws_route53_record" "root" { - count = var.domain_name != "" ? 1 : 0 - zone_id = data.aws_route53_zone.main[0].zone_id - name = var.domain_name - type = "A" + count = var.domain_name != "" ? 1 : 0 + zone_id = data.aws_route53_zone.main[0].zone_id + name = var.domain_name + type = "A" + allow_overwrite = true alias { name = aws_cloudfront_distribution.frontend.domain_name @@ -66,10 +67,11 @@ resource "aws_route53_record" "root" { # www → CloudFront resource "aws_route53_record" "www" { - count = var.domain_name != "" ? 1 : 0 - zone_id = data.aws_route53_zone.main[0].zone_id - name = "www.${var.domain_name}" - type = "A" + count = var.domain_name != "" ? 1 : 0 + zone_id = data.aws_route53_zone.main[0].zone_id + name = "www.${var.domain_name}" + type = "A" + allow_overwrite = true alias { name = aws_cloudfront_distribution.frontend.domain_name diff --git a/infrastructure/bugs.txt b/infrastructure/bugs.txt new file mode 100644 index 0000000..da3eadb --- /dev/null +++ b/infrastructure/bugs.txt @@ -0,0 +1,212 @@ +SOKO PRODUCTION DEPLOYMENT — BUG REPORT +Generated: 2026-05-19 +======================================== + +1. INFRASTRUCTURE & TERRAFORM +============================== + +BUG-01: SSH connection timeout from GitHub Actions + Cause: Security group restricted SSH (port 22) to a single home IP. + GitHub Actions runners use dynamic IPs. + Fix: Set allowed_ssh_cidr = "0.0.0.0/0" in terraform.tfvars. + +BUG-02: EC2 /opt/soko was empty after provisioning + Cause: user_data.sh runs set -e and git clone -b main silently failed + because the working branch (price_prediction_service) wasn't + merged to main yet. + Fix: Manual clone targeting the correct branch: + sudo git clone -b price_prediction_service \ + https://github.com/the-icemann/soko.git /opt/soko + +BUG-03: EC2 IP changed between sessions + Cause: terraform destroy + terraform apply re-allocates a new Elastic IP. + Fix: Updated EC2_HOST GitHub secret in both repos: + 13.244.217.134 -> 13.245.25.42 + +BUG-04: Secrets Manager "scheduled for deletion" error on re-apply + Cause: terraform destroy soft-deletes secrets; re-running apply fails + because the secrets still exist in a pending-deletion state. + Fix: Force-delete before re-applying: + aws secretsmanager delete-secret \ + --force-delete-without-recovery --secret-id soko/platform + aws secretsmanager delete-secret \ + --force-delete-without-recovery --secret-id soko/ml + +BUG-05: ACM certificate validation hung for 18+ minutes + Cause: Domain soko-ug.com was not registered yet — Route 53 validation + records had nowhere to resolve. + Fix: Registered the domain, then later abandoned the domain approach + entirely. Commented out all of acm.tf, disabled the us_east_1 + provider alias, used CloudFront's default certificate, and removed + stale resources from Terraform state: + terraform state rm 'aws_acm_certificate.frontend[0]' + (and related Route 53 / validation resources) + +BUG-06: Two Route 53 hosted zones conflicting + Cause: Domain registration auto-created a second hosted zone. The + original manually-created zone was still in Terraform state. + NS records pointed to the deleted zone -> SERVFAIL. + Fix: Skipped domain setup entirely for now. All domain-related Terraform + is commented out and documented for re-enablement in DEPLOY.md. + +BUG-07: Terraform us_east_1 provider config missing after commenting out ACM + Cause: ACM certificate resource was still in Terraform state but the + provider alias had been commented out, causing a config error. + Fix: terraform state rm to remove all ACM/Route53 resources before + re-applying with the provider commented out. + + +2. GITHUB ACTIONS CI/CD +======================== + +BUG-08: GitHub Actions deploy targeting wrong branch + Cause: deploy.yml had git reset --hard origin/main hardcoded. Pushing + to price_prediction_service would reset the server to main. + Fix: Changed to: + git reset --hard origin/${{ github.ref_name }} + +BUG-09: npm ci failing with ERESOLVE peer dependency error + Cause: eslint-plugin-import@2.32.0 declared a peer dependency on + ESLint <9 but ESLint v10 was installed. + Fix: Added --legacy-peer-deps to npm ci in the frontend workflow. + +BUG-10: package-lock.json out of sync + Cause: Packages were added to the project without running npm install, + leaving package-lock.json stale. npm ci is strict and rejects this. + Fix: Ran npm install --legacy-peer-deps locally and committed the + updated package-lock.json. + + +3. AWS CLI & SECRETS +===================== + +BUG-11: fetch-secrets.sh failing — AWS CLI not found + Cause: Fresh EC2 instance (after destroy/apply) didn't have the AWS CLI + installed. The old instance had it installed manually; user_data + did not include the install step. + Fix: Manually installed AWS CLI v2 on the new instance via the official + install script. + +BUG-12: IAM user missing ACM permissions + Cause: soko_deployment IAM user only had EC2/S3/CloudFront permissions; + ACM calls during terraform apply were denied. + Fix: aws iam attach-user-policy \ + --policy-arn arn:aws:iam::aws:policy/AWSCertificateManagerFullAccess \ + --user-name soko_deployment + + +4. NGINX / CORS +================ + +BUG-13: CORS preflight returning no Access-Control-Allow-Origin + Cause: Multiple failed attempts: + Attempt 1 — add_header inside if blocks in server {} context: + NGINX silently drops headers set inside if blocks. + Attempt 2 — add_header inside if ($request_method = OPTIONS): + NGINX does not allow add_header inside if at server level. + Fix: Used a map directive to resolve the allowed origin dynamically + ($cors_origin), then placed all four add_header ... always + directives directly in the server {} block. OPTIONS preflight uses + a simple if ($request_method = OPTIONS) { return 204; } with no + headers inside the block — server-level headers apply to all + responses including 204. + +BUG-14: NGINX container not picking up updated nginx.conf + Cause: The container was running with the old config in memory. + Editing the file on disk does not affect a running container. + Fix: docker compose up -d --force-recreate nginx + (service is named nginx, not api_gateway) + + +5. FRONTEND BUILD & DEPLOYMENT +================================ + +BUG-15: Wrong Vite environment variable name + Cause: The GitHub Actions workflow and all manual builds used + VITE_API_BASE_URL, but every file in the frontend source (api.ts, + blog.api.ts, etc.) reads import.meta.env.VITE_API_URL. + The variable was never injected — BASE_URL was always undefined. + Fix: Updated workflow env key to VITE_API_URL, renamed the GitHub + Actions variable, and rebuilt with the correct name. + +BUG-16: VITE_API_URL undefined caused requests to go to CloudFront + Cause: fetch("undefinedauth/register", ...) is parsed as a relative URL, + resolved against the current page. CloudFront served index.html + for that path -> browser tried to parse HTML as JSON -> + "Unexpected token '<'". + Fix: Resolved by fixing BUG-15. + +BUG-17: Mixed content — HTTPS frontend calling HTTP backend + Cause: Frontend served over HTTPS (CloudFront) while VITE_API_URL pointed + at http://13.245.25.42. Browsers block HTTP requests from HTTPS + pages entirely. + Fix: Added EC2 as a second CloudFront origin. All API path prefixes now + route through CloudFront (HTTPS) to the EC2 origin (HTTP port 80). + The browser always speaks HTTPS to CloudFront. + +BUG-18: CloudFront rejected bare IP address as origin domain_name + Cause: AWS CloudFront does not accept IP addresses as domain_name for + custom origins — only DNS hostnames. + Fix: Used nip.io — a free wildcard DNS service where .nip.io + resolves to that IP. Terraform config: + domain_name = "${aws_eip.soko.public_ip}.nip.io" + +BUG-19: CloudFront behavior path patterns didn't match NGINX routes + Cause: Initial path list used assumed names that don't match the actual + NGINX location blocks, e.g.: + /user/* should be /users/* + /produce/* should be /listings/* + /order/* should be /orders/* + /payment/* should be /payments/* + /notification/* should be /notifications/* + /blog/* should be /posts/* + /prices/* should be /ml/price/* (covered by /ml/*) + /location/* should be /ml/location/* (covered by /ml/*) + Requests to these paths fell through to the S3 origin and got + index.html instead of the API response. + Fix: Read nginx.conf and corrected the list to match actual location + blocks: + /auth/* /oauth/* /users/* /listings/* /orders/* + /payments/* /webhook/* /message/* /notifications/* + /posts/* /ussd/* /recommendations/* /ml/* /health* + +BUG-20: Missing trailing slash on VITE_API_URL caused broken URL + Cause: api.ts builds URLs as `${BASE_URL}${endpoint}` where endpoints + have no leading slash (e.g. "auth/register"). With: + BASE_URL = "https://d1xay3qieyc2yk.cloudfront.net" + the result is: + "https://d1xay3qieyc2yk.cloudfront.netauth/register" + — a malformed hostname that DNS cannot resolve -> "Failed to fetch". + Fix: Built and deployed with trailing slash: + VITE_API_URL=https://d1xay3qieyc2yk.cloudfront.net/ + Updated the GitHub Actions variable to the same value. + + +SUMMARY TABLE +============= + +# Area Root Cause Severity +--- ------------ -------------------------------------- --------------- +01 Infra SSH locked to home IP Blocked CI +02 Infra user_data cloned wrong branch Blocked first boot +03 Infra EC2 IP changed on destroy/apply Broke CI secret +04 Infra Secrets in pending-deletion state Blocked re-apply +05 Infra Domain not registered before cert 18 min hang +06 Infra Duplicate Route 53 hosted zones DNS SERVFAIL +07 Terraform State had resources with missing Apply crash + provider alias +08 CI Hardcoded branch in deploy script Wrong code on server +09 CI ESLint peer dep conflict Build failure +10 CI Stale package-lock.json Build failure +11 EC2 AWS CLI not installed on fresh inst. Secrets fetch failure +12 IAM Missing ACM policy on deploy user Permission denied +13 NGINX add_header in if block silently All API calls blocked + dropped by CORS +14 Docker Container not restarted after config Old NGINX config served + change +15 Frontend Wrong env var name (BASE_URL/API_URL) All API calls undefined +16 Frontend Relative URL from undefined base JSON parse error +17 Frontend HTTPS page calling HTTP backend Mixed content blocked +18 CloudFront Bare IP rejected as origin Terraform apply error +19 CloudFront Path patterns didn't match NGINX API routes went to S3 +20 Frontend Missing trailing slash on base URL DNS resolution failure