diff --git a/.env b/.env deleted file mode 100644 index 880656a..0000000 --- a/.env +++ /dev/null @@ -1,19 +0,0 @@ -SM_API_ENDPOINT=http://localhost:8080/api/v0 -SM_CURRENCY=USD -SM_DEFAULT_PHONE_COUNTRY_CODE=1 -SM_SERIAL="1234" -SM_SPARKCLOUD_API_KEY=test -SM_HEROKU=false -SM_OFFLINE=true -SM_DEMO_METERS=1234 -INIT_CREATE_DEFAULTS=true -SM_INIT_ADMIN_USERNAME=admin -INIT_ADMIN_USERNAME=admin -INIT_ADMIN_PASSWORD=password -INIT_ADMIN_EMAIL=admin@sparkmeter.io -INIT_GROUND_NAME=Ground -INIT_GROUND_SERIAL=1234 - -HEARTBEAT_PERIOD=1 -SM_SECURITY_PASSWORD_SALT=dev-salt-not-secret -SM_SECRET_KEY=dev-not-secret diff --git a/.gitignore b/.gitignore index a7ba79e..acdf25d 100644 --- a/.gitignore +++ b/.gitignore @@ -79,8 +79,3 @@ _notes/ # Auto-generated by hatch-vcs at build time from the latest git tag. sparkmeter/_version.py - -# Local-only env overrides for site-specific compose runs (e.g. pointing -# symds-ground at a real cloud SymmetricDS endpoint). Use with -# `docker compose --env-file .env.local up -d ...`. -.env.local diff --git a/README.md b/README.md index ed53edb..2b3fa2b 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ uv run flask shell uv run flask status ``` -For Docker deployments, prefix commands with `docker compose exec`: +For Docker deployments, run the same commands inside the webapp container with `docker compose exec`: ```bash docker compose exec ground uv run flask user create ``` @@ -100,69 +100,16 @@ This document is to help you get the development environment up and running. You ### Docker +Dockerized development happens in the **groundbolt-dev workspace metarepo**, which clones this repo and its sibling component repos side by side and runs the whole system — the ground and cloud webapps, their databases, the SymmetricDS sync pair, and the `sparknet-http` metering provider — from a single compose file. See that repo's README for setup. -For a fully Dockerized development environment: - -```bash -$ docker compose up -``` - -Pass `-d` to detach. Run with `--watch` to enable Compose's develop watcher, which rebuilds the `ground` and `cloud` images on file changes (note: full image rebuild, not Flask hot-reload — slow for iterative work, useful for verifying built behavior). - -#### 1. Build docker services - -`docker compose up` builds any missing images automatically, so this step is optional. Run it when you want to force a clean rebuild (e.g. after Dockerfile or dependency changes): - -```bash -$ docker compose build --no-cache -``` - -#### 2. Run the built docker services - -The default Compose profile brings up the ground-side stack: - -```bash -$ docker compose up -d -``` - -After it settles, `docker compose ps` shows: - -``` -ground # the GroundBolt webapp -postgres-ground # its database -metering-provider # the service the ground app calls for meter operations -symds-ground # SymmetricDS node syncing the ground database -``` - -The ground webapp talks to the metering provider over an OpenAPI HTTP+SSE contract, configured via `METERING_PROVIDER_URL` (default `http://localhost:8000`). The Compose file references the provider image by tag. That image must be present locally; build or pull it under that tag before bringing up the ground stack. - -The cloud-side stack lives behind the `cloud` Compose profile. To also bring up the cloud webapp and the cloud-side SymmetricDS node: - -```bash -$ docker compose --profile cloud up -d -``` - -That adds: - -``` -cloud # the ThunderCloud webapp -postgres-cloud # its database -symds-cloud # SymmetricDS node syncing the cloud database -``` - -`symds-ground` and `symds-cloud` together provide bidirectional sync between the ground and cloud databases. With only the ground profile active, `symds-ground` runs but has nothing to sync against until `--profile cloud` brings up `symds-cloud`. - -#### 3. Voilà! Your development environment is ready! - -- The GroundBolt webapp (`ground` container) is at [localhost:8765](http://localhost:8765). -- With `--profile cloud`, the ThunderCloud webapp (`cloud` container) is at [localhost:5010](http://localhost:5010). +This repo's `docker-compose.test.yml` is the self-contained test harness: a throwaway Postgres plus the test-image runner, needing nothing outside this repo. It's what CI runs. #### Useful commands ##### Run unit tests ```bash -$ docker compose --profile test run --rm test +$ docker compose -f docker-compose.test.yml run --rm test ``` ##### Run a subset of unit tests @@ -170,23 +117,25 @@ $ docker compose --profile test run --rm test Override the `test` service's command with a pytest invocation. Any pytest arguments can be passed: ```bash -$ docker compose --profile test run --rm test uv run pytest [::ClassName][::method_name] +$ docker compose -f docker-compose.test.yml run --rm test uv run pytest [::ClassName][::method_name] ``` For example, running the tests for the AddCustomer endpoint in the API: ```bash -docker compose --profile test run --rm test uv run pytest sparkmeter/api/tests/test_customerviews0.py::CustomerAddTest +docker compose -f docker-compose.test.yml run --rm test uv run pytest sparkmeter/api/tests/test_customerviews0.py::CustomerAddTest ``` ##### Create a new database migration -Database schema migrations are managed via Alembic. To get started, you must first create a migration file. +Database schema migrations are managed via Alembic. To get started, you must first create a migration file: ```bash -docker compose exec ground uv run flask database new-revision "" +uv run flask database new-revision "" ``` +(Or run the same command inside the webapp container of a running Docker stack, via `docker compose exec`.) + This will generate a skeleton migration file in `sparkmeter/alembic/versions/`. From there, customize it your liking. ##### Tail the logs of a service diff --git a/docker-compose.test.yml b/docker-compose.test.yml new file mode 100644 index 0000000..3867dd2 --- /dev/null +++ b/docker-compose.test.yml @@ -0,0 +1,48 @@ +# Test harness, not a dev environment: builds the test image from this repo +# alone and runs the suite against a throwaway Postgres. CI +# (scripts/run_coverage.sh) depends on this working with only this repo +# checked out. Invoke with `docker compose -f docker-compose.test.yml ...`. +# +# The local dev stack (ground/cloud webapps, SymmetricDS pair, metering +# provider) lives in the groundbolt-dev workspace metarepo, which clones this +# repo and its siblings side by side and can therefore build across them. + +services: + # Nothing here is named worktree-globally, so two checkouts (a worktree and + # the main checkout, or two worktrees) can run their suites at the same time + # without colliding: + # - No `container_name:`, so re-running the test command after a non-clean + # exit doesn't collide with a leftover container of a fixed name; + # compose auto-names them per-project. + # - No fixed host port. A published `5442:5432` makes the second + # concurrent run die with "Bind for 0.0.0.0:5442 failed: port is + # already allocated". The host doesn't need a fixed port: the test + # container reaches the database over the compose network, and the + # ephemeral publication below still allows poking at it from the host + # via `docker compose -f docker-compose.test.yml port postgres-test 5432`. + postgres-test: + image: library/postgres:18 + environment: + - POSTGRES_USER=spark + - POSTGRES_PASSWORD=spark + - POSTGRES_DB=mydb + ports: + # Host port assigned by docker; container port fixed. + - "5432" + healthcheck: + test: ["CMD", "psql", "-U", "spark", "-d", "mydb", "-c", "SELECT 1;"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + + test: + build: + context: '.' + dockerfile: deploy/Dockerfile + target: test + environment: + - DATABASE_URL=postgresql://spark:spark@postgres-test/mydb + depends_on: + - postgres-test + command: uv run pytest -n auto diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 7b23c61..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,221 +0,0 @@ -services: - ground: - container_name: "ground" - build: - context: '.' - dockerfile: deploy/Dockerfile - target: production - image: sparkmeter - command: /app/run/webapp - ports: - - "8765:5000" - env_file: - - .env - environment: - DATABASE_URL: postgresql://spark:spark@postgres-ground/ground - METERING_PROVIDER_URL: http://metering-provider:8000 - # The ground row created on first boot derives its UUID from `serial` - # via `as_uuid(serial)`. To make this local deployment **be** a - # specific site (so meters/customers/etc. created here are anchored - # to the same ground row the cloud has), set these in `.env.local` - # alongside the SYMDS_* values and run with - # `docker compose --env-file .env.local up -d ...`: - # INIT_GROUND_SERIAL= - # INIT_GROUND_NAME= - # GROUND_SERIAL= # matches INIT_GROUND_SERIAL; drives default-ground lookup - INIT_GROUND_SERIAL: ${INIT_GROUND_SERIAL:-} - INIT_GROUND_NAME: ${INIT_GROUND_NAME:-} - SERIAL: ${GROUND_SERIAL:-} - depends_on: - postgres-ground: - condition: service_started - #metering-provider: - # condition: service_healthy - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:5000/health"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 40s - develop: - watch: - - path: ./ - action: rebuild - - # Vendor-supplied metering provider. - metering-provider: - container_name: "metering-provider" - image: ghcr.io/earthspark/ohm-server:latest - environment: - # SIMULATE_GATEWAY=1 makes the provider run its in-memory gateway - # simulator; unset it to talk to a real gateway over a serial port - # (which then needs to be passed into the container). - SIMULATE_GATEWAY: "1" - ports: - - "8000:8000" - healthcheck: - test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8000/openapi.json >/dev/null 2>&1 || exit 1"] - interval: 5s - timeout: 5s - retries: 5 - start_period: 5s - - # POSSIBLE IMPROVEMENTS: - # - Healthcheck uses psql which requires a full DB connection; could use - # pg_isready -U spark which is lighter weight - # - Could use POSTGRES_HOST_AUTH_METHOD=trust for local dev (no password needed) - # - Environment uses list format (- POSTGRES_USER=spark); could use map format - # (POSTGRES_USER: spark) for consistency with other services - postgres-ground: - container_name: "postgres-ground" - image: library/postgres:18 - environment: - - POSTGRES_USER=spark - - POSTGRES_PASSWORD=spark - - POSTGRES_DB=ground - ports: - - "5440:5432" - healthcheck: - test: ["CMD", "psql", "-U", "spark", "-d", "ground", "-c", "SELECT 1;"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 40s - - # SymmetricDS node on the ground side, syncing the ground database - # against symds-cloud. When the `cloud` profile is not active, sync - # attempts retry until `--profile cloud` brings up symds-cloud. - symds-ground: - container_name: "symds-ground" - image: ghcr.io/earthspark/symmetricds - environment: - DATABASE_URL: postgresql://spark:spark@postgres-ground/ground - ENGINE_NAME: ground-engine - # GROUP_ID must match `sparkmeter.database.sync.SYNC_GROUP_GROUND` - # (= "ground-group"), which is the source side of the configured - # ground-group → cloud-group node_group_link. Without this, the - # cloud SymmetricDS refuses registration with "Registration was - # not open" because no matching link exists. - GROUP_ID: ground-group - # Local-only defaults assume the cloud profile is running side-by-side - # (symds-cloud + postgres-cloud + cloud webapp). To point at a real - # cloud endpoint, populate a gitignored `.env.local` and run with - # `docker compose --env-file .env.local up -d ...`: - # SYMDS_PROTOCOL=https - # SYMDS_REGISTRATION_URL=https://-sync./sync/cloud - # SYMDS_EXTERNAL_ID= - # SYMDS_HTTP_USERNAME=symds - # SYMDS_HTTP_PASSWORD=<...> - EXTERNAL_ID: ${SYMDS_EXTERNAL_ID:-ground} - PROTOCOL: ${SYMDS_PROTOCOL:-http} - REGISTRATION_URL: ${SYMDS_REGISTRATION_URL:-http://symds-cloud:31415/sync/cloud-engine} - SYNC_URL: ${SYMDS_SYNC_URL:-http://symds-ground:31415/sync/ground-engine} - HTTP_USERNAME: ${SYMDS_HTTP_USERNAME:-} - HTTP_PASSWORD: ${SYMDS_HTTP_PASSWORD:-} - depends_on: - postgres-ground: - condition: service_started - - postgres-cloud: - container_name: "postgres-cloud" - image: library/postgres:18 - environment: - - POSTGRES_USER=spark - - POSTGRES_PASSWORD=spark - - POSTGRES_DB=cloud - ports: - - "5441:5432" - healthcheck: - test: ["CMD", "psql", "-U", "spark", "-d", "cloud", "-c", "SELECT 1;"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 40s - profiles: - - cloud - - cloud: - container_name: "cloud" - image: sparkmeter - command: /app/run/webapp - ports: - - "5010:5000" - env_file: - - .env - environment: - DATABASE_URL: postgresql://spark:spark@postgres-cloud/cloud - SM_HEROKU: "true" - depends_on: - - postgres-cloud - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:5000/health"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 40s - develop: - watch: - - path: ./ - action: rebuild - profiles: - - cloud - - # SymmetricDS node on the cloud side, syncing the cloud database. - symds-cloud: - container_name: "symds-cloud" - image: ghcr.io/earthspark/symmetricds - environment: - DATABASE_URL: postgresql://spark:spark@postgres-cloud/cloud - PROTOCOL: http - ENGINE_NAME: cloud-engine - EXTERNAL_ID: cloud - # GROUP_ID must match `sparkmeter.database.sync.SYNC_GROUP_CLOUD` - # (= "cloud-group") so the local symds-cloud is the destination of - # the configured ground-group → cloud-group node_group_link. - GROUP_ID: cloud-group - SYNC_URL: http://symds-cloud:31415/sync/cloud-engine - depends_on: - postgres-cloud: - condition: service_started - ports: - - "31415:31415" - profiles: - - cloud - - # No `container_name:` on the test-profile services so re-running the - # test command after a non-clean exit doesn't collide with a leftover - # container of a fixed name; compose auto-names them per-project. - postgres-test: - image: library/postgres:18 - environment: - - POSTGRES_USER=spark - - POSTGRES_PASSWORD=spark - - POSTGRES_DB=mydb - ports: - - "5442:5432" - healthcheck: - test: ["CMD", "psql", "-U", "spark", "-d", "mydb", "-c", "SELECT 1;"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 40s - profiles: - - test - - test: - build: - context: '.' - dockerfile: deploy/Dockerfile - target: test - environment: - - DATABASE_URL=postgresql://spark:spark@postgres-test/mydb - depends_on: - - postgres-test - profiles: - - test - command: uv run pytest -n auto - - -networks: - default: - name: sparkapp diff --git a/scripts/run_coverage.sh b/scripts/run_coverage.sh index ab8a265..2d27801 100644 --- a/scripts/run_coverage.sh +++ b/scripts/run_coverage.sh @@ -15,7 +15,7 @@ if [ -f .git ]; then version="0.0.0" fi -docker compose --profile test build \ +docker compose -f docker-compose.test.yml build \ --build-arg SETUPTOOLS_SCM_PRETEND_VERSION="$version" \ test @@ -24,7 +24,7 @@ docker compose --profile test build \ # pytest-coverage.txt for the PR report comment. All three are produced in the # container, tarred to stdout, and unpacked on the host. pytest's output is teed # to stderr, not stdout, so stdout carries only the tar stream. -docker compose --profile test run --rm test \ +docker compose -f docker-compose.test.yml run --rm test \ sh -c 'uv run pytest -n auto --junitxml=/app/junit.xml 2>&1 | tee /app/pytest-coverage.txt 1>&2; tar -c -C /app .coverage.xml junit.xml pytest-coverage.txt' \ | tar -x