From 634f68d2bbdd37f7a64a1120a3e62e5d76b4324c Mon Sep 17 00:00:00 2001 From: Tristan Escalada <355457+tescalada@users.noreply.github.com> Date: Mon, 20 Jul 2026 18:30:44 -0400 Subject: [PATCH] Split the dev compose stack out to the workspace metarepo The dev stack spans component repos: it builds the webapp from this repo, builds SymmetricDS from its own, and pulls the sparknet-http image. It cannot be assembled from this repo alone, so it moves to the groundbolt-dev workspace metarepo, which clones the components side by side. What stays here is the test harness, which CI must be able to run with only this repo checked out. It becomes docker-compose.test.yml -- a throwaway Postgres plus the test-image runner -- invoked with `docker compose -f docker-compose.test.yml`, replacing the `--profile test` selection the combined file needed. scripts/run_coverage.sh follows. The harness publishes the test database on an ephemeral host port instead of a fixed 5442, so a second checkout running its suite concurrently does not fail to bind. The test container reaches the database over the compose network, so no fixed port is needed. --- .env | 19 ---- .gitignore | 5 - README.md | 71 ++----------- docker-compose.test.yml | 48 +++++++++ docker-compose.yml | 221 ---------------------------------------- scripts/run_coverage.sh | 4 +- 6 files changed, 60 insertions(+), 308 deletions(-) delete mode 100644 .env create mode 100644 docker-compose.test.yml delete mode 100644 docker-compose.yml diff --git a/.env b/.env deleted file mode 100644 index 880656a..0000000 --- a/.env +++ /dev/null @@ -1,19 +0,0 @@ -SM_API_ENDPOINT=http://localhost:8080/api/v0 -SM_CURRENCY=USD -SM_DEFAULT_PHONE_COUNTRY_CODE=1 -SM_SERIAL="1234" -SM_SPARKCLOUD_API_KEY=test -SM_HEROKU=false -SM_OFFLINE=true -SM_DEMO_METERS=1234 -INIT_CREATE_DEFAULTS=true -SM_INIT_ADMIN_USERNAME=admin -INIT_ADMIN_USERNAME=admin -INIT_ADMIN_PASSWORD=password -INIT_ADMIN_EMAIL=admin@sparkmeter.io -INIT_GROUND_NAME=Ground -INIT_GROUND_SERIAL=1234 - -HEARTBEAT_PERIOD=1 -SM_SECURITY_PASSWORD_SALT=dev-salt-not-secret -SM_SECRET_KEY=dev-not-secret diff --git a/.gitignore b/.gitignore index a7ba79e..acdf25d 100644 --- a/.gitignore +++ b/.gitignore @@ -79,8 +79,3 @@ _notes/ # Auto-generated by hatch-vcs at build time from the latest git tag. sparkmeter/_version.py - -# Local-only env overrides for site-specific compose runs (e.g. pointing -# symds-ground at a real cloud SymmetricDS endpoint). Use with -# `docker compose --env-file .env.local up -d ...`. -.env.local diff --git a/README.md b/README.md index ed53edb..2b3fa2b 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ uv run flask shell uv run flask status ``` -For Docker deployments, prefix commands with `docker compose exec`: +For Docker deployments, run the same commands inside the webapp container with `docker compose exec`: ```bash docker compose exec ground uv run flask user create ``` @@ -100,69 +100,16 @@ This document is to help you get the development environment up and running. You ### Docker +Dockerized development happens in the **groundbolt-dev workspace metarepo**, which clones this repo and its sibling component repos side by side and runs the whole system — the ground and cloud webapps, their databases, the SymmetricDS sync pair, and the `sparknet-http` metering provider — from a single compose file. See that repo's README for setup. -For a fully Dockerized development environment: - -```bash -$ docker compose up -``` - -Pass `-d` to detach. Run with `--watch` to enable Compose's develop watcher, which rebuilds the `ground` and `cloud` images on file changes (note: full image rebuild, not Flask hot-reload — slow for iterative work, useful for verifying built behavior). - -#### 1. Build docker services - -`docker compose up` builds any missing images automatically, so this step is optional. Run it when you want to force a clean rebuild (e.g. after Dockerfile or dependency changes): - -```bash -$ docker compose build --no-cache -``` - -#### 2. Run the built docker services - -The default Compose profile brings up the ground-side stack: - -```bash -$ docker compose up -d -``` - -After it settles, `docker compose ps` shows: - -``` -ground # the GroundBolt webapp -postgres-ground # its database -metering-provider # the service the ground app calls for meter operations -symds-ground # SymmetricDS node syncing the ground database -``` - -The ground webapp talks to the metering provider over an OpenAPI HTTP+SSE contract, configured via `METERING_PROVIDER_URL` (default `http://localhost:8000`). The Compose file references the provider image by tag. That image must be present locally; build or pull it under that tag before bringing up the ground stack. - -The cloud-side stack lives behind the `cloud` Compose profile. To also bring up the cloud webapp and the cloud-side SymmetricDS node: - -```bash -$ docker compose --profile cloud up -d -``` - -That adds: - -``` -cloud # the ThunderCloud webapp -postgres-cloud # its database -symds-cloud # SymmetricDS node syncing the cloud database -``` - -`symds-ground` and `symds-cloud` together provide bidirectional sync between the ground and cloud databases. With only the ground profile active, `symds-ground` runs but has nothing to sync against until `--profile cloud` brings up `symds-cloud`. - -#### 3. Voilà! Your development environment is ready! - -- The GroundBolt webapp (`ground` container) is at [localhost:8765](http://localhost:8765). -- With `--profile cloud`, the ThunderCloud webapp (`cloud` container) is at [localhost:5010](http://localhost:5010). +This repo's `docker-compose.test.yml` is the self-contained test harness: a throwaway Postgres plus the test-image runner, needing nothing outside this repo. It's what CI runs. #### Useful commands ##### Run unit tests ```bash -$ docker compose --profile test run --rm test +$ docker compose -f docker-compose.test.yml run --rm test ``` ##### Run a subset of unit tests @@ -170,23 +117,25 @@ $ docker compose --profile test run --rm test Override the `test` service's command with a pytest invocation. Any pytest arguments can be passed: ```bash -$ docker compose --profile test run --rm test uv run pytest [::ClassName][::method_name] +$ docker compose -f docker-compose.test.yml run --rm test uv run pytest [::ClassName][::method_name] ``` For example, running the tests for the AddCustomer endpoint in the API: ```bash -docker compose --profile test run --rm test uv run pytest sparkmeter/api/tests/test_customerviews0.py::CustomerAddTest +docker compose -f docker-compose.test.yml run --rm test uv run pytest sparkmeter/api/tests/test_customerviews0.py::CustomerAddTest ``` ##### Create a new database migration -Database schema migrations are managed via Alembic. To get started, you must first create a migration file. +Database schema migrations are managed via Alembic. To get started, you must first create a migration file: ```bash -docker compose exec ground uv run flask database new-revision "" +uv run flask database new-revision "" ``` +(Or run the same command inside the webapp container of a running Docker stack, via `docker compose exec`.) + This will generate a skeleton migration file in `sparkmeter/alembic/versions/`. From there, customize it your liking. ##### Tail the logs of a service diff --git a/docker-compose.test.yml b/docker-compose.test.yml new file mode 100644 index 0000000..3867dd2 --- /dev/null +++ b/docker-compose.test.yml @@ -0,0 +1,48 @@ +# Test harness, not a dev environment: builds the test image from this repo +# alone and runs the suite against a throwaway Postgres. CI +# (scripts/run_coverage.sh) depends on this working with only this repo +# checked out. Invoke with `docker compose -f docker-compose.test.yml ...`. +# +# The local dev stack (ground/cloud webapps, SymmetricDS pair, metering +# provider) lives in the groundbolt-dev workspace metarepo, which clones this +# repo and its siblings side by side and can therefore build across them. + +services: + # Nothing here is named worktree-globally, so two checkouts (a worktree and + # the main checkout, or two worktrees) can run their suites at the same time + # without colliding: + # - No `container_name:`, so re-running the test command after a non-clean + # exit doesn't collide with a leftover container of a fixed name; + # compose auto-names them per-project. + # - No fixed host port. A published `5442:5432` makes the second + # concurrent run die with "Bind for 0.0.0.0:5442 failed: port is + # already allocated". The host doesn't need a fixed port: the test + # container reaches the database over the compose network, and the + # ephemeral publication below still allows poking at it from the host + # via `docker compose -f docker-compose.test.yml port postgres-test 5432`. + postgres-test: + image: library/postgres:18 + environment: + - POSTGRES_USER=spark + - POSTGRES_PASSWORD=spark + - POSTGRES_DB=mydb + ports: + # Host port assigned by docker; container port fixed. + - "5432" + healthcheck: + test: ["CMD", "psql", "-U", "spark", "-d", "mydb", "-c", "SELECT 1;"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + + test: + build: + context: '.' + dockerfile: deploy/Dockerfile + target: test + environment: + - DATABASE_URL=postgresql://spark:spark@postgres-test/mydb + depends_on: + - postgres-test + command: uv run pytest -n auto diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 7b23c61..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,221 +0,0 @@ -services: - ground: - container_name: "ground" - build: - context: '.' - dockerfile: deploy/Dockerfile - target: production - image: sparkmeter - command: /app/run/webapp - ports: - - "8765:5000" - env_file: - - .env - environment: - DATABASE_URL: postgresql://spark:spark@postgres-ground/ground - METERING_PROVIDER_URL: http://metering-provider:8000 - # The ground row created on first boot derives its UUID from `serial` - # via `as_uuid(serial)`. To make this local deployment **be** a - # specific site (so meters/customers/etc. created here are anchored - # to the same ground row the cloud has), set these in `.env.local` - # alongside the SYMDS_* values and run with - # `docker compose --env-file .env.local up -d ...`: - # INIT_GROUND_SERIAL= - # INIT_GROUND_NAME= - # GROUND_SERIAL= # matches INIT_GROUND_SERIAL; drives default-ground lookup - INIT_GROUND_SERIAL: ${INIT_GROUND_SERIAL:-} - INIT_GROUND_NAME: ${INIT_GROUND_NAME:-} - SERIAL: ${GROUND_SERIAL:-} - depends_on: - postgres-ground: - condition: service_started - #metering-provider: - # condition: service_healthy - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:5000/health"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 40s - develop: - watch: - - path: ./ - action: rebuild - - # Vendor-supplied metering provider. - metering-provider: - container_name: "metering-provider" - image: ghcr.io/earthspark/ohm-server:latest - environment: - # SIMULATE_GATEWAY=1 makes the provider run its in-memory gateway - # simulator; unset it to talk to a real gateway over a serial port - # (which then needs to be passed into the container). - SIMULATE_GATEWAY: "1" - ports: - - "8000:8000" - healthcheck: - test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8000/openapi.json >/dev/null 2>&1 || exit 1"] - interval: 5s - timeout: 5s - retries: 5 - start_period: 5s - - # POSSIBLE IMPROVEMENTS: - # - Healthcheck uses psql which requires a full DB connection; could use - # pg_isready -U spark which is lighter weight - # - Could use POSTGRES_HOST_AUTH_METHOD=trust for local dev (no password needed) - # - Environment uses list format (- POSTGRES_USER=spark); could use map format - # (POSTGRES_USER: spark) for consistency with other services - postgres-ground: - container_name: "postgres-ground" - image: library/postgres:18 - environment: - - POSTGRES_USER=spark - - POSTGRES_PASSWORD=spark - - POSTGRES_DB=ground - ports: - - "5440:5432" - healthcheck: - test: ["CMD", "psql", "-U", "spark", "-d", "ground", "-c", "SELECT 1;"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 40s - - # SymmetricDS node on the ground side, syncing the ground database - # against symds-cloud. When the `cloud` profile is not active, sync - # attempts retry until `--profile cloud` brings up symds-cloud. - symds-ground: - container_name: "symds-ground" - image: ghcr.io/earthspark/symmetricds - environment: - DATABASE_URL: postgresql://spark:spark@postgres-ground/ground - ENGINE_NAME: ground-engine - # GROUP_ID must match `sparkmeter.database.sync.SYNC_GROUP_GROUND` - # (= "ground-group"), which is the source side of the configured - # ground-group → cloud-group node_group_link. Without this, the - # cloud SymmetricDS refuses registration with "Registration was - # not open" because no matching link exists. - GROUP_ID: ground-group - # Local-only defaults assume the cloud profile is running side-by-side - # (symds-cloud + postgres-cloud + cloud webapp). To point at a real - # cloud endpoint, populate a gitignored `.env.local` and run with - # `docker compose --env-file .env.local up -d ...`: - # SYMDS_PROTOCOL=https - # SYMDS_REGISTRATION_URL=https://-sync./sync/cloud - # SYMDS_EXTERNAL_ID= - # SYMDS_HTTP_USERNAME=symds - # SYMDS_HTTP_PASSWORD=<...> - EXTERNAL_ID: ${SYMDS_EXTERNAL_ID:-ground} - PROTOCOL: ${SYMDS_PROTOCOL:-http} - REGISTRATION_URL: ${SYMDS_REGISTRATION_URL:-http://symds-cloud:31415/sync/cloud-engine} - SYNC_URL: ${SYMDS_SYNC_URL:-http://symds-ground:31415/sync/ground-engine} - HTTP_USERNAME: ${SYMDS_HTTP_USERNAME:-} - HTTP_PASSWORD: ${SYMDS_HTTP_PASSWORD:-} - depends_on: - postgres-ground: - condition: service_started - - postgres-cloud: - container_name: "postgres-cloud" - image: library/postgres:18 - environment: - - POSTGRES_USER=spark - - POSTGRES_PASSWORD=spark - - POSTGRES_DB=cloud - ports: - - "5441:5432" - healthcheck: - test: ["CMD", "psql", "-U", "spark", "-d", "cloud", "-c", "SELECT 1;"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 40s - profiles: - - cloud - - cloud: - container_name: "cloud" - image: sparkmeter - command: /app/run/webapp - ports: - - "5010:5000" - env_file: - - .env - environment: - DATABASE_URL: postgresql://spark:spark@postgres-cloud/cloud - SM_HEROKU: "true" - depends_on: - - postgres-cloud - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:5000/health"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 40s - develop: - watch: - - path: ./ - action: rebuild - profiles: - - cloud - - # SymmetricDS node on the cloud side, syncing the cloud database. - symds-cloud: - container_name: "symds-cloud" - image: ghcr.io/earthspark/symmetricds - environment: - DATABASE_URL: postgresql://spark:spark@postgres-cloud/cloud - PROTOCOL: http - ENGINE_NAME: cloud-engine - EXTERNAL_ID: cloud - # GROUP_ID must match `sparkmeter.database.sync.SYNC_GROUP_CLOUD` - # (= "cloud-group") so the local symds-cloud is the destination of - # the configured ground-group → cloud-group node_group_link. - GROUP_ID: cloud-group - SYNC_URL: http://symds-cloud:31415/sync/cloud-engine - depends_on: - postgres-cloud: - condition: service_started - ports: - - "31415:31415" - profiles: - - cloud - - # No `container_name:` on the test-profile services so re-running the - # test command after a non-clean exit doesn't collide with a leftover - # container of a fixed name; compose auto-names them per-project. - postgres-test: - image: library/postgres:18 - environment: - - POSTGRES_USER=spark - - POSTGRES_PASSWORD=spark - - POSTGRES_DB=mydb - ports: - - "5442:5432" - healthcheck: - test: ["CMD", "psql", "-U", "spark", "-d", "mydb", "-c", "SELECT 1;"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 40s - profiles: - - test - - test: - build: - context: '.' - dockerfile: deploy/Dockerfile - target: test - environment: - - DATABASE_URL=postgresql://spark:spark@postgres-test/mydb - depends_on: - - postgres-test - profiles: - - test - command: uv run pytest -n auto - - -networks: - default: - name: sparkapp diff --git a/scripts/run_coverage.sh b/scripts/run_coverage.sh index ab8a265..2d27801 100644 --- a/scripts/run_coverage.sh +++ b/scripts/run_coverage.sh @@ -15,7 +15,7 @@ if [ -f .git ]; then version="0.0.0" fi -docker compose --profile test build \ +docker compose -f docker-compose.test.yml build \ --build-arg SETUPTOOLS_SCM_PRETEND_VERSION="$version" \ test @@ -24,7 +24,7 @@ docker compose --profile test build \ # pytest-coverage.txt for the PR report comment. All three are produced in the # container, tarred to stdout, and unpacked on the host. pytest's output is teed # to stderr, not stdout, so stdout carries only the tar stream. -docker compose --profile test run --rm test \ +docker compose -f docker-compose.test.yml run --rm test \ sh -c 'uv run pytest -n auto --junitxml=/app/junit.xml 2>&1 | tee /app/pytest-coverage.txt 1>&2; tar -c -C /app .coverage.xml junit.xml pytest-coverage.txt' \ | tar -x