From 2849b77596b728c985ec6d701c567b2dd6682cec Mon Sep 17 00:00:00 2001 From: Scott Converse Date: Fri, 24 Apr 2026 10:28:24 -0600 Subject: [PATCH 1/3] feat(phase1-a): migration scaffold + civiccore_0001 baseline Phase 1 Part A per ADR-0003. One narrow PR adding: - civiccore.migrations.guards (4 idempotent op.* wrappers + has_table helper) - civiccore.migrations.runner (upgrade_to_head + current_revision) - civiccore/migrations/alembic.ini + env.py (alembic_version_civiccore version table isolates civiccore from consuming modules' Alembic) - civiccore_0001_baseline_v1 migration: idempotent snapshot of the 16 shared tables + 4 shared enum types + pgvector extension as-of records HEAD 019_encrypt_connection_config - tests/test_baseline_idempotency.py: pytest + testcontainers[postgres] proving the baseline runs clean on empty DB AND is a no-op on re-run Source schema: pg_dump --schema-only against running civicrecords-ai postgres at alembic head 019_encrypt_connection_config (2026-04-24). Scope: civiccore only. No civicrecords-ai changes. No model extraction. Phase 1 Part B ships separately: records-side 14-migration guard pass, env.py wiring (6 lines), and the three migration-gate tests (fresh install / v1.2.x upgrade / reapplication) from ADR-0003 section 5. --- CHANGELOG.md | 5 + civiccore/migrations/alembic.ini | 3 + civiccore/migrations/env.py | 131 ++-- civiccore/migrations/guards.py | 103 ++++ civiccore/migrations/runner.py | 43 ++ civiccore/migrations/versions/__init__.py | 7 + .../versions/civiccore_0001_baseline_v1.py | 572 ++++++++++++++++++ pyproject.toml | 3 +- tests/test_baseline_idempotency.py | 139 +++++ 9 files changed, 962 insertions(+), 44 deletions(-) create mode 100644 civiccore/migrations/alembic.ini create mode 100644 civiccore/migrations/guards.py create mode 100644 civiccore/migrations/runner.py create mode 100644 civiccore/migrations/versions/__init__.py create mode 100644 civiccore/migrations/versions/civiccore_0001_baseline_v1.py create mode 100644 tests/test_baseline_idempotency.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 77db4e6..f183207 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,11 @@ MINOR; bug fixes ship as PATCH. - Apache 2.0 `LICENSE`, `README.md`, `CONTRIBUTING.md` (with the bug-routing decision tree from spec section 18), `.gitignore`, and placeholder `civiccore-ui/` npm package directory. +- `civiccore.migrations.guards` — three idempotent op wrappers (`idempotent_create_table`, `idempotent_add_column`, `idempotent_alter_column`) plus `has_table` helper. +- `civiccore.migrations.runner` — `upgrade_to_head(connection)` and `current_revision(connection)` entry points for consuming modules' env.py. +- `civiccore/migrations/alembic.ini` + `civiccore/migrations/env.py` — civiccore's own Alembic wiring (`alembic_version_civiccore` version table to avoid collision with consuming modules). +- `civiccore_0001_baseline_v1` migration — idempotent snapshot of the 16 civiccore-owned shared tables at records HEAD `019_encrypt_connection_config`, per ADR-0003. +- `tests/test_baseline_idempotency.py` — pytest asserting the baseline runs clean on an empty DB and is a no-op against an already-populated DB. ### Changed - License switched from MIT to Apache License 2.0 to match civicrecords-ai diff --git a/civiccore/migrations/alembic.ini b/civiccore/migrations/alembic.ini new file mode 100644 index 0000000..0656e11 --- /dev/null +++ b/civiccore/migrations/alembic.ini @@ -0,0 +1,3 @@ +[alembic] +script_location = %(here)s +version_table = alembic_version_civiccore diff --git a/civiccore/migrations/env.py b/civiccore/migrations/env.py index f91d747..5ca1415 100644 --- a/civiccore/migrations/env.py +++ b/civiccore/migrations/env.py @@ -1,43 +1,88 @@ -"""Alembic environment for CivicCore shared-table migrations. - -STUB ONLY — Phase 0 scaffold. No real migration revisions are seeded yet. -Phase 1 will baseline this against the latest CivicRecords AI migration that -touches a shared table (users, roles, audit_log, documents, document_chunks, -model_registry, connectors, notification_templates, city_profile, -exemption_rules) and mark it as the CivicCore baseline revision. - -Migration-ordering contract (from CivicCore Extraction Spec section 14): - - 1. CivicCore's migration runner is called FIRST by every consuming - module's runner. The wrapper in each module is a thin shim around - Alembic that calls `civiccore.migrations.run()` before applying its - own revisions. - - 2. For a fresh install, the sequence is: - (a) civiccore_migrate upgrade head - (b) _migrate upgrade head # records, clerk, code, zone, ... - - 3. Order is enforced by Alembic's `depends_on` metadata on every - module-side revision that touches a shared table. A module revision - declares `depends_on = ("",)` so a fresh DB - cannot race the two runners. - - 4. CivicCore NEVER imports from any module. Module migrations may - reference shared tables, but the dependency arrow points one way: - module --> civiccore. - - 5. Shared-table schema changes are MAJOR CivicCore releases - (semver-major). Minor and patch CivicCore releases never alter - shared table schemas. See spec section 16. - -This file deliberately does not configure a context, target_metadata, or -run_migrations_online() yet. Phase 1 will fill those in once the shared -models module is populated. -""" - -# Phase 1 will add: -# from alembic import context -# from civiccore.models import Base -# target_metadata = Base.metadata -# def run_migrations_online() -> None: ... -# def run_migrations_offline() -> None: ... +"""Alembic env for civiccore. Reads the connection from Config.attributes['connection'] when invoked by runner.py (the normal path). Falls back to DATABASE_URL env var when invoked standalone.""" + +from __future__ import annotations + +import os +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import engine_from_config, pool + +config = context.config + +# Tolerate the minimal alembic.ini which intentionally omits [loggers]/[handlers]/[formatters]. +if config.config_file_name is not None: + try: + fileConfig(config.config_file_name) + except KeyError: + # No logging sections configured; skip Python logging setup. + pass + +VERSION_TABLE = "alembic_version_civiccore" +target_metadata = None # v0.1 baseline is raw-SQL; no ORM metadata yet. + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode — emits SQL to stdout / script without a DB connection.""" + url = os.environ.get("DATABASE_URL") or config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + version_table=VERSION_TABLE, + ) + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode. + + Two sub-modes: + - Connection supplied by caller via `config.attributes["connection"]` + (the normal path: records' env.py hands us its live connection through + `runner.upgrade_to_head`). We do not dispose the connection; the caller + owns its lifecycle. + - Standalone CLI (`alembic upgrade head`): no attached connection, so + build an engine from `DATABASE_URL` (or the ini's `sqlalchemy.url`) and + run migrations against it. + """ + connectable = config.attributes.get("connection", None) + + if connectable is not None: + context.configure( + connection=connectable, + target_metadata=target_metadata, + version_table=VERSION_TABLE, + ) + with context.begin_transaction(): + context.run_migrations() + return + + # Standalone path: no connection was passed in. + url = os.environ.get("DATABASE_URL") + ini_section = config.get_section(config.config_ini_section) or {} + if url: + ini_section["sqlalchemy.url"] = url + + engine = engine_from_config( + ini_section, + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with engine.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + version_table=VERSION_TABLE, + ) + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/civiccore/migrations/guards.py b/civiccore/migrations/guards.py new file mode 100644 index 0000000..aada0cc --- /dev/null +++ b/civiccore/migrations/guards.py @@ -0,0 +1,103 @@ +"""Idempotent Alembic operation helpers. These wrap alembic.op.* calls with 'skip if already exists' semantics so civiccore's baseline migration and records' 14 guarded migrations can all be re-run against a database that already contains the shared schema.""" + +from __future__ import annotations + +from typing import Any + +import alembic.op as op +import sqlalchemy as sa +from sqlalchemy.engine import Inspector + + +def _inspector() -> Inspector: + """Return a fresh SQLAlchemy Inspector bound to the current Alembic connection.""" + return sa.inspect(op.get_bind()) + + +def idempotent_create_table(name: str, *columns: Any, **kwargs: Any) -> None: + """op.create_table that no-ops if the table already exists. + + Used by civiccore's baseline migration and by guarded records migrations that + create shared tables (e.g. `users`, `data_sources`). If the table is already + present in the database, the create is skipped; otherwise it proceeds normally. + """ + inspector = _inspector() + if inspector.has_table(name): + return + op.create_table(name, *columns, **kwargs) + + +def idempotent_add_column(table: str, column: Any, **kwargs: Any) -> None: + """op.add_column that no-ops if the column already exists. + + If the target table itself does not exist, returns silently — the upstream + migration (or civiccore baseline) is responsible for creating it. If the + table exists and the column is already present, skip. Otherwise add it. + """ + inspector = _inspector() + if not inspector.has_table(table): + # Upstream migration will handle table creation; nothing to add to. + return + existing = {col["name"] for col in inspector.get_columns(table)} + if column.name in existing: + return + op.add_column(table, column, **kwargs) + + +def idempotent_alter_column( + table: str, + column: str, + *, + existing_type: Any = None, + nullable: bool | None = None, + server_default: Any = None, + new_column_name: str | None = None, + **kwargs: Any, +) -> None: + """op.alter_column that introspects current state before applying. + + Applies the alter only when a concrete difference is detected: + - `nullable` is specified AND the current column's nullable != requested, OR + - `new_column_name` is specified AND it differs from `column`. + + `existing_type` and `server_default` are passed through to `op.alter_column` + but are NOT used as the diff trigger — type/default deep-comparison is too + brittle across dialects. Those alters are the caller's responsibility to + schedule once by migration history; this guard exists to absorb repeat runs. + + If the table or column does not exist, returns silently. + """ + inspector = _inspector() + if not inspector.has_table(table): + return + + cols = {col["name"]: col for col in inspector.get_columns(table)} + current = cols.get(column) + if current is None: + return + + should_apply = False + if nullable is not None and bool(current.get("nullable")) != bool(nullable): + should_apply = True + if new_column_name is not None and new_column_name != column: + should_apply = True + + if not should_apply: + return + + alter_kwargs: dict[str, Any] = dict(kwargs) + if existing_type is not None: + alter_kwargs["existing_type"] = existing_type + if nullable is not None: + alter_kwargs["nullable"] = nullable + if server_default is not None: + alter_kwargs["server_default"] = server_default + if new_column_name is not None: + alter_kwargs["new_column_name"] = new_column_name + + op.alter_column(table, column, **alter_kwargs) + + +def has_table(name: str) -> bool: + """Thin public wrapper returning whether the named table exists in the current DB.""" + return _inspector().has_table(name) diff --git a/civiccore/migrations/runner.py b/civiccore/migrations/runner.py new file mode 100644 index 0000000..4d4ca5a --- /dev/null +++ b/civiccore/migrations/runner.py @@ -0,0 +1,43 @@ +"""Programmatic entry point for civiccore migrations. Consumed by records' env.py (Phase 1 Part B) so records' alembic upgrade automatically brings civiccore up to head first.""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING + +import alembic.config +from alembic import command +from alembic.runtime.migration import MigrationContext + +if TYPE_CHECKING: + from sqlalchemy.engine import Connection + + +_ALEMBIC_INI = Path(__file__).with_name("alembic.ini") + + +def upgrade_to_head(connection: "Connection") -> None: + """Upgrade civiccore's migration chain to head using the supplied SQLAlchemy connection. + + Builds an Alembic Config pointing at civiccore's own `alembic.ini`, attaches + the caller's live connection via `cfg.attributes["connection"]`, and runs + `alembic upgrade head`. civiccore's `env.py` reads the attached connection + on the normal code path so the operation participates in the caller's + transaction scope. + """ + cfg = alembic.config.Config(str(_ALEMBIC_INI)) + cfg.attributes["connection"] = connection + command.upgrade(cfg, "head") + + +def current_revision(connection: "Connection") -> str | None: + """Return civiccore's current Alembic revision on this connection, or None if unstamped. + + Reads from the `alembic_version_civiccore` version table so it does not + collide with records' own `alembic_version` table in the same database. + """ + ctx = MigrationContext.configure( + connection, + opts={"version_table": "alembic_version_civiccore"}, + ) + return ctx.get_current_revision() diff --git a/civiccore/migrations/versions/__init__.py b/civiccore/migrations/versions/__init__.py new file mode 100644 index 0000000..42d0b6d --- /dev/null +++ b/civiccore/migrations/versions/__init__.py @@ -0,0 +1,7 @@ +"""Alembic migration versions for civiccore. + +Alembic discovers migration files by scanning this directory — the +__init__.py is present only so the test suite can import version-module +constants (e.g. _SHARED_TABLE_ORDER from the baseline migration) via +standard Python package semantics. +""" diff --git a/civiccore/migrations/versions/civiccore_0001_baseline_v1.py b/civiccore/migrations/versions/civiccore_0001_baseline_v1.py new file mode 100644 index 0000000..1cd043d --- /dev/null +++ b/civiccore/migrations/versions/civiccore_0001_baseline_v1.py @@ -0,0 +1,572 @@ +"""CivicCore baseline v1 — shared schema extracted from CivicRecords HEAD 019. + +This baseline captures the 16 shared tables that constitute the CivicCore +kernel. It is the single source of truth for kernel schema going forward; +downstream applications (e.g., CivicRecords) should rebase onto this baseline +and stop re-declaring shared tables in their own migrations. + +Provenance: + Dumped from CivicRecords alembic HEAD ``019_encrypt_connection_config`` + via ``pg_dump --schema-only --no-owner --no-privileges --no-comments`` + against ``civicrecords_test`` on PostgreSQL 17.9. + +Dependencies captured (SHARED -> SHARED only): + users.department_id -> departments.id + city_profile.updated_by -> users.id + data_sources.created_by -> users.id + exemption_rules.created_by -> users.id + notification_templates.created_by -> users.id + prompt_templates.created_by -> users.id + prompt_templates.model_id -> model_registry.id + service_accounts.created_by -> users.id + documents.source_id -> data_sources.id + document_chunks.document_id -> documents.id + sync_failures.dismissed_by -> users.id + sync_failures.source_id -> data_sources.id + sync_run_log.source_id -> data_sources.id + +No shared->records-only FKs were present in the source dump; none were +dropped during extraction. + +Idempotency: + Each table DDL chunk executes only when ``has_table(t)`` returns False. + This lets the baseline co-exist with a database that already carries + the shared tables from a prior records-side migration. + +Downgrade: + No-op by design. A baseline migration defines the floor of supported + schema; reversing it would drop tables that both civiccore and + downstream apps depend on. Use point-in-time restore or drop the + database instead. +""" + +from __future__ import annotations + +from alembic import op + +from civiccore.migrations.guards import has_table + + +# Alembic identifiers -------------------------------------------------------- + +revision = "civiccore_0001_baseline_v1" +down_revision = None +branch_labels = None +depends_on = None + + +# Ordered list of shared tables (parents before children per FK analysis) ---- + +_SHARED_TABLE_ORDER: list[str] = [ + "audit_log", + "model_registry", + "connector_templates", + "departments", + "system_catalog", + "users", + "city_profile", + "data_sources", + "exemption_rules", + "notification_templates", + "prompt_templates", + "service_accounts", + "documents", + "sync_failures", + "sync_run_log", + "document_chunks", +] + + +# Per-table DDL chunks extracted from pg_dump ------------------------------ +# +# Each entry bundles, for a single table: +# - CREATE SEQUENCE (if the table owns one, e.g. serial PKs) +# - CREATE TABLE +# - ALTER SEQUENCE ... OWNED BY +# - ALTER TABLE ... ALTER COLUMN ... SET DEFAULT nextval(...) +# - ALTER TABLE ... ADD CONSTRAINT (PRIMARY KEY / UNIQUE) +# - CREATE INDEX +# - ALTER TABLE ... ADD CONSTRAINT ... FOREIGN KEY (only shared->shared) +# +# Executed verbatim via ``op.execute`` so the DDL matches CivicRecords HEAD 019 +# byte-for-byte where practical. + +_TABLE_DDL: dict[str, str] = { + "audit_log": """ +CREATE SEQUENCE public.audit_log_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; +CREATE TABLE public.audit_log ( + id integer NOT NULL, + prev_hash character varying(64) DEFAULT '0000000000000000000000000000000000000000000000000000000000000000'::character varying NOT NULL, + entry_hash character varying(64) NOT NULL, + "timestamp" timestamp with time zone DEFAULT now(), + user_id uuid, + action character varying(100) NOT NULL, + resource_type character varying(100) NOT NULL, + resource_id character varying(255), + details jsonb, + ai_generated boolean DEFAULT false NOT NULL +); +ALTER SEQUENCE public.audit_log_id_seq OWNED BY public.audit_log.id; +ALTER TABLE ONLY public.audit_log ALTER COLUMN id SET DEFAULT nextval('public.audit_log_id_seq'::regclass); +ALTER TABLE ONLY public.audit_log + ADD CONSTRAINT audit_log_pkey PRIMARY KEY (id); +CREATE INDEX ix_audit_log_action ON public.audit_log USING btree (action); +CREATE INDEX ix_audit_log_entry_hash ON public.audit_log USING btree (entry_hash); +CREATE INDEX ix_audit_log_resource_type ON public.audit_log USING btree (resource_type); +CREATE INDEX ix_audit_log_timestamp ON public.audit_log USING btree ("timestamp"); +CREATE INDEX ix_audit_log_user_id ON public.audit_log USING btree (user_id); +CREATE INDEX ix_audit_log_user_timestamp ON public.audit_log USING btree (user_id, "timestamp"); +""", + + "model_registry": """ +CREATE SEQUENCE public.model_registry_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; +CREATE TABLE public.model_registry ( + id integer NOT NULL, + model_name character varying(255) NOT NULL, + model_version character varying(100), + parameter_count character varying(50), + license character varying(100), + model_card_url text, + is_active boolean DEFAULT false NOT NULL, + added_at timestamp with time zone DEFAULT now(), + context_window_size integer, + supports_ner boolean DEFAULT false NOT NULL, + supports_vision boolean DEFAULT false NOT NULL +); +ALTER SEQUENCE public.model_registry_id_seq OWNED BY public.model_registry.id; +ALTER TABLE ONLY public.model_registry ALTER COLUMN id SET DEFAULT nextval('public.model_registry_id_seq'::regclass); +ALTER TABLE ONLY public.model_registry + ADD CONSTRAINT model_registry_pkey PRIMARY KEY (id); +""", + + "connector_templates": """ +CREATE SEQUENCE public.connector_templates_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; +CREATE TABLE public.connector_templates ( + id integer NOT NULL, + vendor_name character varying(200) NOT NULL, + protocol character varying(50) NOT NULL, + auth_method character varying(50) NOT NULL, + config_schema jsonb NOT NULL, + default_sync_schedule character varying(50), + default_rate_limit integer, + redaction_tier integer NOT NULL, + setup_instructions text, + created_at timestamp with time zone DEFAULT now() NOT NULL, + catalog_version character varying(20) NOT NULL +); +ALTER SEQUENCE public.connector_templates_id_seq OWNED BY public.connector_templates.id; +ALTER TABLE ONLY public.connector_templates ALTER COLUMN id SET DEFAULT nextval('public.connector_templates_id_seq'::regclass); +ALTER TABLE ONLY public.connector_templates + ADD CONSTRAINT connector_templates_pkey PRIMARY KEY (id); +""", + + "departments": """ +CREATE TABLE public.departments ( + id uuid NOT NULL, + name character varying(200) NOT NULL, + code character varying(20) NOT NULL, + contact_email character varying(255), + created_at timestamp with time zone DEFAULT now() NOT NULL +); +ALTER TABLE ONLY public.departments + ADD CONSTRAINT departments_code_key UNIQUE (code); +ALTER TABLE ONLY public.departments + ADD CONSTRAINT departments_pkey PRIMARY KEY (id); +""", + + "system_catalog": """ +CREATE SEQUENCE public.system_catalog_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; +CREATE TABLE public.system_catalog ( + id integer NOT NULL, + domain character varying(100) NOT NULL, + function character varying(200) NOT NULL, + vendor_name character varying(200) NOT NULL, + vendor_version character varying(50), + access_protocol character varying(50) NOT NULL, + data_shape character varying(50) NOT NULL, + common_record_types jsonb NOT NULL, + redaction_tier integer NOT NULL, + discovery_hints jsonb NOT NULL, + connector_template_id integer, + catalog_version character varying(20) NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); +ALTER SEQUENCE public.system_catalog_id_seq OWNED BY public.system_catalog.id; +ALTER TABLE ONLY public.system_catalog ALTER COLUMN id SET DEFAULT nextval('public.system_catalog_id_seq'::regclass); +ALTER TABLE ONLY public.system_catalog + ADD CONSTRAINT system_catalog_pkey PRIMARY KEY (id); +""", + + "users": """ +CREATE TABLE public.users ( + id uuid NOT NULL, + email character varying(320) NOT NULL, + hashed_password character varying(1024) NOT NULL, + is_active boolean DEFAULT true NOT NULL, + is_superuser boolean DEFAULT false NOT NULL, + is_verified boolean DEFAULT false NOT NULL, + full_name character varying DEFAULT ''::character varying NOT NULL, + role public.user_role DEFAULT 'staff'::public.user_role NOT NULL, + created_at timestamp with time zone DEFAULT now(), + last_login timestamp with time zone, + department_id uuid +); +ALTER TABLE ONLY public.users + ADD CONSTRAINT users_pkey PRIMARY KEY (id); +CREATE UNIQUE INDEX ix_users_email ON public.users USING btree (email); +ALTER TABLE ONLY public.users + ADD CONSTRAINT fk_users_department FOREIGN KEY (department_id) REFERENCES public.departments(id); +""", + + "city_profile": """ +CREATE TABLE public.city_profile ( + id uuid NOT NULL, + city_name character varying(200) NOT NULL, + state character varying(2), + county character varying(200), + population_band character varying(50), + email_platform character varying(50), + has_dedicated_it boolean, + monthly_request_volume character varying(20), + onboarding_status character varying(20) NOT NULL, + profile_data jsonb NOT NULL, + gap_map jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL, + updated_at timestamp with time zone DEFAULT now() NOT NULL, + updated_by uuid +); +ALTER TABLE ONLY public.city_profile + ADD CONSTRAINT city_profile_pkey PRIMARY KEY (id); +ALTER TABLE ONLY public.city_profile + ADD CONSTRAINT city_profile_updated_by_fkey FOREIGN KEY (updated_by) REFERENCES public.users(id) ON DELETE SET NULL; +""", + + "data_sources": """ +CREATE TABLE public.data_sources ( + id uuid NOT NULL, + name character varying(255) NOT NULL, + source_type public.source_type NOT NULL, + connection_config jsonb DEFAULT '{}'::jsonb NOT NULL, + is_active boolean DEFAULT true NOT NULL, + created_by uuid NOT NULL, + created_at timestamp with time zone DEFAULT now(), + last_ingestion_at timestamp with time zone, + discovered_source_id uuid, + connector_template_id integer, + sync_schedule character varying(50), + last_sync_at timestamp with time zone, + last_sync_status character varying(20), + health_status character varying(20), + schema_hash character varying(64), + last_sync_cursor character varying, + schedule_enabled boolean DEFAULT true NOT NULL, + consecutive_failure_count integer DEFAULT 0 NOT NULL, + last_error_message character varying(500), + last_error_at timestamp with time zone, + sync_paused boolean DEFAULT false NOT NULL, + sync_paused_at timestamp with time zone, + sync_paused_reason character varying(200), + retry_batch_size integer, + retry_time_limit_seconds integer, + CONSTRAINT chk_sync_schedule_nonempty CHECK (((sync_schedule IS NULL) OR (length(TRIM(BOTH FROM sync_schedule)) > 0))) +); +ALTER TABLE ONLY public.data_sources + ADD CONSTRAINT data_sources_name_key UNIQUE (name); +ALTER TABLE ONLY public.data_sources + ADD CONSTRAINT data_sources_pkey PRIMARY KEY (id); +ALTER TABLE ONLY public.data_sources + ADD CONSTRAINT data_sources_created_by_fkey FOREIGN KEY (created_by) REFERENCES public.users(id); +""", + + "exemption_rules": """ +CREATE TABLE public.exemption_rules ( + id uuid NOT NULL, + state_code character varying(2) NOT NULL, + category character varying(100) NOT NULL, + rule_type public.rule_type NOT NULL, + rule_definition text NOT NULL, + description text, + enabled boolean DEFAULT true NOT NULL, + created_by uuid NOT NULL, + created_at timestamp with time zone DEFAULT now(), + version integer DEFAULT 1 NOT NULL +); +ALTER TABLE ONLY public.exemption_rules + ADD CONSTRAINT exemption_rules_pkey PRIMARY KEY (id); +CREATE INDEX ix_exemption_rules_category ON public.exemption_rules USING btree (category); +CREATE INDEX ix_exemption_rules_state ON public.exemption_rules USING btree (state_code); +ALTER TABLE ONLY public.exemption_rules + ADD CONSTRAINT exemption_rules_created_by_fkey FOREIGN KEY (created_by) REFERENCES public.users(id); +""", + + "notification_templates": """ +CREATE TABLE public.notification_templates ( + id uuid NOT NULL, + event_type character varying(50) NOT NULL, + channel character varying(20) NOT NULL, + subject_template character varying(500) NOT NULL, + body_template text NOT NULL, + is_active boolean NOT NULL, + created_by uuid, + created_at timestamp with time zone DEFAULT now() NOT NULL +); +ALTER TABLE ONLY public.notification_templates + ADD CONSTRAINT notification_templates_event_type_key UNIQUE (event_type); +ALTER TABLE ONLY public.notification_templates + ADD CONSTRAINT notification_templates_pkey PRIMARY KEY (id); +ALTER TABLE ONLY public.notification_templates + ADD CONSTRAINT notification_templates_created_by_fkey FOREIGN KEY (created_by) REFERENCES public.users(id) ON DELETE SET NULL; +""", + + "prompt_templates": """ +CREATE TABLE public.prompt_templates ( + id uuid NOT NULL, + name character varying(200) NOT NULL, + purpose character varying(50) NOT NULL, + system_prompt text NOT NULL, + user_prompt_template text NOT NULL, + token_budget jsonb NOT NULL, + model_id integer, + version integer NOT NULL, + is_active boolean NOT NULL, + created_by uuid, + created_at timestamp with time zone DEFAULT now() NOT NULL +); +ALTER TABLE ONLY public.prompt_templates + ADD CONSTRAINT prompt_templates_name_key UNIQUE (name); +ALTER TABLE ONLY public.prompt_templates + ADD CONSTRAINT prompt_templates_pkey PRIMARY KEY (id); +ALTER TABLE ONLY public.prompt_templates + ADD CONSTRAINT prompt_templates_created_by_fkey FOREIGN KEY (created_by) REFERENCES public.users(id) ON DELETE SET NULL; +ALTER TABLE ONLY public.prompt_templates + ADD CONSTRAINT prompt_templates_model_id_fkey FOREIGN KEY (model_id) REFERENCES public.model_registry(id) ON DELETE SET NULL; +""", + + "service_accounts": """ +CREATE TABLE public.service_accounts ( + id uuid NOT NULL, + name character varying(255) NOT NULL, + api_key_hash character varying(255) NOT NULL, + role public.user_role DEFAULT 'read_only'::public.user_role NOT NULL, + created_by uuid NOT NULL, + created_at timestamp with time zone DEFAULT now(), + is_active boolean DEFAULT true NOT NULL +); +ALTER TABLE ONLY public.service_accounts + ADD CONSTRAINT service_accounts_name_key UNIQUE (name); +ALTER TABLE ONLY public.service_accounts + ADD CONSTRAINT service_accounts_pkey PRIMARY KEY (id); +ALTER TABLE ONLY public.service_accounts + ADD CONSTRAINT service_accounts_created_by_fkey FOREIGN KEY (created_by) REFERENCES public.users(id); +""", + + "documents": """ +CREATE TABLE public.documents ( + id uuid NOT NULL, + source_id uuid NOT NULL, + source_path text NOT NULL, + filename character varying(500) NOT NULL, + file_type character varying(50) NOT NULL, + file_hash character varying(64) NOT NULL, + file_size integer DEFAULT 0 NOT NULL, + ingestion_status public.ingestion_status DEFAULT 'pending'::public.ingestion_status NOT NULL, + ingestion_error text, + chunk_count integer DEFAULT 0 NOT NULL, + ingested_at timestamp with time zone, + metadata jsonb, + display_name character varying(500), + department_id uuid, + redaction_status character varying(20) DEFAULT 'none'::character varying NOT NULL, + derivative_path character varying(1000), + original_locked boolean DEFAULT false NOT NULL, + connector_type character varying(20), + updated_at timestamp with time zone, + CONSTRAINT chk_source_path_length CHECK (((source_path IS NULL) OR (length(source_path) <= 2048))) +); +ALTER TABLE ONLY public.documents + ADD CONSTRAINT documents_pkey PRIMARY KEY (id); +CREATE INDEX ix_documents_file_hash ON public.documents USING btree (file_hash); +CREATE INDEX ix_documents_source_hash ON public.documents USING btree (source_id, file_hash); +CREATE INDEX ix_documents_source_id ON public.documents USING btree (source_id); +CREATE UNIQUE INDEX uq_documents_binary_hash ON public.documents USING btree (source_id, file_hash) WHERE ((connector_type)::text <> ALL ((ARRAY['rest_api'::character varying, 'odbc'::character varying])::text[])); +CREATE UNIQUE INDEX uq_documents_structured_path ON public.documents USING btree (source_id, source_path) WHERE ((connector_type)::text = ANY ((ARRAY['rest_api'::character varying, 'odbc'::character varying])::text[])); +ALTER TABLE ONLY public.documents + ADD CONSTRAINT documents_source_id_fkey FOREIGN KEY (source_id) REFERENCES public.data_sources(id); +""", + + "sync_failures": """ +CREATE TABLE public.sync_failures ( + id uuid DEFAULT gen_random_uuid() NOT NULL, + source_id uuid NOT NULL, + source_path text NOT NULL, + error_message text, + error_class character varying(200), + http_status_code integer, + retry_count integer DEFAULT 0 NOT NULL, + status character varying(20) DEFAULT 'retrying'::character varying NOT NULL, + first_failed_at timestamp with time zone DEFAULT now() NOT NULL, + last_retried_at timestamp with time zone, + resolved_at timestamp with time zone, + dismissed_at timestamp with time zone, + dismissed_by uuid +); +ALTER TABLE ONLY public.sync_failures + ADD CONSTRAINT sync_failures_pkey PRIMARY KEY (id); +CREATE INDEX ix_sync_failures_created ON public.sync_failures USING btree (first_failed_at); +CREATE INDEX ix_sync_failures_source_status ON public.sync_failures USING btree (source_id, status); +ALTER TABLE ONLY public.sync_failures + ADD CONSTRAINT sync_failures_dismissed_by_fkey FOREIGN KEY (dismissed_by) REFERENCES public.users(id); +ALTER TABLE ONLY public.sync_failures + ADD CONSTRAINT sync_failures_source_id_fkey FOREIGN KEY (source_id) REFERENCES public.data_sources(id) ON DELETE CASCADE; +""", + + "sync_run_log": """ +CREATE TABLE public.sync_run_log ( + id uuid DEFAULT gen_random_uuid() NOT NULL, + source_id uuid NOT NULL, + started_at timestamp with time zone DEFAULT now() NOT NULL, + finished_at timestamp with time zone, + status character varying(20), + records_attempted integer DEFAULT 0, + records_succeeded integer DEFAULT 0, + records_failed integer DEFAULT 0, + error_summary text +); +ALTER TABLE ONLY public.sync_run_log + ADD CONSTRAINT sync_run_log_pkey PRIMARY KEY (id); +CREATE INDEX ix_sync_run_log_source ON public.sync_run_log USING btree (source_id, started_at); +ALTER TABLE ONLY public.sync_run_log + ADD CONSTRAINT sync_run_log_source_id_fkey FOREIGN KEY (source_id) REFERENCES public.data_sources(id) ON DELETE CASCADE; +""", + + "document_chunks": """ +CREATE TABLE public.document_chunks ( + id uuid NOT NULL, + document_id uuid NOT NULL, + chunk_index integer NOT NULL, + content_text text NOT NULL, + embedding public.vector(768), + token_count integer DEFAULT 0 NOT NULL, + page_number integer, + content_tsvector tsvector GENERATED ALWAYS AS (to_tsvector('english'::regconfig, content_text)) STORED +); +ALTER TABLE ONLY public.document_chunks + ADD CONSTRAINT document_chunks_pkey PRIMARY KEY (id); +CREATE INDEX ix_chunks_doc_index ON public.document_chunks USING btree (document_id, chunk_index); +CREATE INDEX ix_chunks_embedding_hnsw ON public.document_chunks USING hnsw (embedding public.vector_cosine_ops) WITH (m='16', ef_construction='64'); +CREATE INDEX ix_chunks_tsvector ON public.document_chunks USING gin (content_tsvector); +CREATE INDEX ix_document_chunks_document_id ON public.document_chunks USING btree (document_id); +ALTER TABLE ONLY public.document_chunks + ADD CONSTRAINT document_chunks_document_id_fkey FOREIGN KEY (document_id) REFERENCES public.documents(id) ON DELETE CASCADE; +""", + +} + + +# Extensions and enum types captured from pg_dump --------------------------- + +_REQUIRED_EXTENSIONS: list[str] = [ + "vector", # pgvector; required by document_chunks.embedding +] + +# Enum types referenced by shared-table columns. Records-only enums +# (flag_status, inclusion_status, request_status) stay records-side. +_SHARED_ENUM_DDL: dict[str, str] = { + "user_role": ( + "CREATE TYPE public.user_role AS ENUM " + "('admin', 'staff', 'reviewer', 'read_only', 'liaison', 'public')" + ), + "source_type": ( + "CREATE TYPE public.source_type AS ENUM " + "('manual_drop', 'file_system', 'rest_api', 'odbc')" + ), + "ingestion_status": ( + "CREATE TYPE public.ingestion_status AS ENUM " + "('pending', 'processing', 'completed', 'failed')" + ), + "rule_type": ( + "CREATE TYPE public.rule_type AS ENUM " + "('regex', 'keyword', 'llm_prompt')" + ), +} + + +def _type_exists(conn, name: str) -> bool: + """Return True if a PostgreSQL type with this name exists in any schema.""" + from sqlalchemy import text + + return ( + conn.execute( + text("SELECT 1 FROM pg_type WHERE typname = :n LIMIT 1"), + {"n": name}, + ).scalar() + is not None + ) + + +def upgrade() -> None: + """Idempotently bring the database up to the civiccore baseline. + + Runs in three phases: + 1. Ensure required extensions exist (``vector`` for pgvector). + 2. Ensure the 4 shared enum types exist (``user_role``, + ``source_type``, ``ingestion_status``, ``rule_type``). + 3. Create the 16 shared tables in dependency order, skipping any + that already exist. + + Safe to run against: + * A fresh database (all extensions, enums, tables are created). + * A records-side database already at HEAD 019 (everything skipped). + * Any partial intermediate state (each element independently guarded). + """ + conn = op.get_bind() + + # 1. Extensions — CREATE EXTENSION supports IF NOT EXISTS natively + for ext in _REQUIRED_EXTENSIONS: + op.execute(f'CREATE EXTENSION IF NOT EXISTS "{ext}"') + + # 2. Enums — PostgreSQL has no CREATE TYPE IF NOT EXISTS; check pg_type + for enum_name, ddl in _SHARED_ENUM_DDL.items(): + if not _type_exists(conn, enum_name): + op.execute(ddl) + + # 3. Tables — ordered, guarded + for table in _SHARED_TABLE_ORDER: + if has_table(table): + continue + op.execute(_TABLE_DDL[table]) + + +def downgrade() -> None: + """No-op. + + A baseline cannot be meaningfully downgraded: the 16 shared tables are + the kernel contract and are likely referenced by downstream migrations + that are not visible from here. Use database-level restore instead. + """ + # Intentional no-op. See module docstring. + return None diff --git a/pyproject.toml b/pyproject.toml index 58df549..946cb5c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,11 +71,12 @@ dependencies = [ [project.optional-dependencies] dev = [ + "psycopg2-binary>=2.9.0", "pytest>=9.0.0", "pytest-asyncio>=1.0.0", "respx>=0.22.0", "ruff>=0.11.0", - "psycopg2-binary>=2.9.0", + "testcontainers[postgres]>=4.9.0", ] [project.urls] diff --git a/tests/test_baseline_idempotency.py b/tests/test_baseline_idempotency.py new file mode 100644 index 0000000..739d94d --- /dev/null +++ b/tests/test_baseline_idempotency.py @@ -0,0 +1,139 @@ +"""Idempotency test for civiccore_0001_baseline_v1. Blocks Phase 1 Part A PR merge per ADR-0003 §5.""" + +from __future__ import annotations + +import pytest +import sqlalchemy as sa + +# Skip the whole module gracefully if testcontainers or Docker are unavailable. +# Rationale: these tests require a live ephemeral Postgres and cannot be +# faked with a stub. When the runner lacks Docker, we record the skip reason +# rather than failing — CI image provisions Docker; dev machines may not. +testcontainers = pytest.importorskip( + "testcontainers.postgres", + reason="testcontainers[postgres] not installed; install dev extras to run idempotency tests", +) +PostgresContainer = testcontainers.PostgresContainer + +from civiccore.migrations.runner import current_revision, upgrade_to_head # noqa: E402 +from civiccore.migrations.versions.civiccore_0001_baseline_v1 import ( # noqa: E402 + _SHARED_TABLE_ORDER, +) + +EXPECTED_HEAD = "civiccore_0001_baseline_v1" + + +def _docker_available() -> bool: + """Return True if a Docker daemon is reachable; False otherwise. + + testcontainers spins up via the Docker SDK, which raises ``DockerException`` + when the daemon is unreachable. We probe once at fixture setup so that + dev machines without Docker get a clear skip rather than a stack trace. + """ + try: + import docker # type: ignore[import-untyped] + + docker.from_env().ping() + return True + except Exception: + return False + + +@pytest.fixture(scope="module") +def pg_container(): + """Ephemeral Postgres 17 + pgvector container shared across this module's tests. + + Uses the same ``pgvector/pgvector:pg17`` image as CivicRecords' docker-compose + stack so the extension stanza in the baseline migration (``CREATE EXTENSION + IF NOT EXISTS vector``) can succeed. Plain ``postgres:17`` lacks the + pgvector control file and would fail the ``document_chunks`` table DDL. + + Skips the whole module if Docker is not available on the runner. + """ + if not _docker_available(): + pytest.skip( + "Docker daemon not reachable — idempotency tests require testcontainers " + "with a running Docker host. Install Docker Desktop or run in CI." + ) + with PostgresContainer("pgvector/pgvector:pg17") as pg: + yield pg + + +@pytest.fixture +def engine(pg_container): + """Fresh SQLAlchemy engine against the ephemeral container. + + A new engine per test gives each test a clean transactional context; + the container itself is reused across the module for speed. + """ + url = pg_container.get_connection_url() + # testcontainers returns a psycopg2 URL; normalize to psycopg2 driver form + # SQLAlchemy understands without requiring asyncpg here. + eng = sa.create_engine(url, future=True) + try: + yield eng + finally: + eng.dispose() + + +def _snapshot_schema(connection: sa.Connection) -> list[tuple[str, list[str]]]: + """Capture (table_name, sorted column_names) for every shared table present. + + Sorted for stable comparison across runs; structure is a plain list of + tuples so equality is exact. + """ + inspector = sa.inspect(connection) + existing = set(inspector.get_table_names()) + snapshot: list[tuple[str, list[str]]] = [] + for table in _SHARED_TABLE_ORDER: + if table not in existing: + continue + cols = sorted(col["name"] for col in inspector.get_columns(table)) + snapshot.append((table, cols)) + return sorted(snapshot, key=lambda row: row[0]) + + +def test_baseline_runs_clean_on_empty_db(engine): + """Baseline migration creates all 16 shared tables and stamps HEAD on an empty DB.""" + with engine.begin() as connection: + upgrade_to_head(connection) + + with engine.connect() as connection: + inspector = sa.inspect(connection) + existing = set(inspector.get_table_names()) + missing = [t for t in _SHARED_TABLE_ORDER if t not in existing] + assert not missing, ( + f"Baseline did not create expected shared tables: {missing}. " + f"Found tables: {sorted(existing)}" + ) + assert current_revision(connection) == EXPECTED_HEAD + + +def test_baseline_is_idempotent(engine): + """Re-running the baseline against an already-populated DB is a no-op. + + Proves the ``idempotent_*`` op wrappers (see ``civiccore.migrations.guards``) + skip existing objects rather than destructively recreating them — which + is what makes the extracted civiccore baseline safe to ship alongside + already-deployed civicrecords databases (ADR-0003 §3). + """ + with engine.begin() as connection: + upgrade_to_head(connection) + + with engine.connect() as connection: + first_snapshot = _snapshot_schema(connection) + assert current_revision(connection) == EXPECTED_HEAD + + # Second upgrade on the same engine — must not raise, must not mutate schema. + with engine.begin() as connection: + upgrade_to_head(connection) + + with engine.connect() as connection: + second_snapshot = _snapshot_schema(connection) + assert current_revision(connection) == EXPECTED_HEAD + + assert first_snapshot == second_snapshot, ( + "Schema changed between first and second baseline runs — baseline is not idempotent.\n" + f"First: {first_snapshot}\n" + f"Second: {second_snapshot}" + ) From 63899f7769ca5f7c1ee9eb57a726939c89c933e3 Mon Sep 17 00:00:00 2001 From: Scott Converse Date: Fri, 24 Apr 2026 10:46:20 -0600 Subject: [PATCH 2/3] ci(phase1-a): add CI workflow + fix stale scottconverse/* URLs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Audit findings on PR #1: - TEST-001: idempotency test claimed to be 'merge gate' but no CI existed. Added .github/workflows/ci.yml that runs both test files on PR + push to main; pre-pulls pgvector/pgvector:pg17 to keep testcontainers fast. - DOC-001: README, CONTRIBUTING, pyproject.toml [project.urls], and CHANGELOG header still pointed at scottconverse/civiccore and scottconverse/civicsuite. Corrected to CivicSuite/* (the org Scott created on 2026-04-23). The scottconverse/civicrecords-ai URL is intentionally preserved — records repo has not been transferred to the CivicSuite org yet. CHANGELOG.md updated with both entries. --- .claude/hooks/commit-size-gate.py | 183 ++++++++++++++++++++++++++++++ .claude/hooks/commit-size-gate.sh | 2 + .claude/settings.json | 25 ++++ .github/workflows/ci.yml | 37 ++++++ CHANGELOG.md | 8 +- CONTRIBUTING.md | 12 +- README.md | 8 +- pyproject.toml | 6 +- 8 files changed, 267 insertions(+), 14 deletions(-) create mode 100644 .claude/hooks/commit-size-gate.py create mode 100644 .claude/hooks/commit-size-gate.sh create mode 100644 .claude/settings.json create mode 100644 .github/workflows/ci.yml diff --git a/.claude/hooks/commit-size-gate.py b/.claude/hooks/commit-size-gate.py new file mode 100644 index 0000000..1581488 --- /dev/null +++ b/.claude/hooks/commit-size-gate.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +""" +Commit-Size Acknowledgment Gate — Hard Rule 11 + +Blocks `git commit` when: + (a) the staged diff exceeds THRESHOLD lines (insertions + deletions, + ignoring binary files as reported by `git diff --cached --numstat`), AND + (b) the commit message parsed from `-m` / `--message` does NOT contain + one of the allowed literal bracketed tag tokens. + +Allowed tokens (literal bracketed match, NOT substring): + [MVP] [LARGE-CHANGE] [REFACTOR] [INITIAL] [MERGE] [REVERT] + [SCOPE-EXPANSION: ] + +So "fixed a bug with the MVP flow" does NOT satisfy the gate. +Only "[MVP] ..." or "[SCOPE-EXPANSION: why] ..." do. + +BYPASS SURFACES (gate is FAIL-OPEN for all of these): + - `git commit --amend` — final message not parseable at PreToolUse + - `git commit -F ` — message lives in a file the hook can't read reliably + - editor commits (no -m) — message comes from $EDITOR post-hook + - any exception in this script — fail-open by design (never lock the user out) + - `override rule 11` marker file (60-second one-shot, deleted on use) + +If you amend into a large commit, self-police with the tag on the next +real -m commit. The gate intentionally does not chase --amend/-F/editor +commits because parsing them reliably would require a post-commit hook, +which is a different enforcement surface. + +THRESHOLD is tunable — see constant at the top of main() below. +""" + +import json +import os +import re +import subprocess +import sys +import time +from pathlib import Path + +# --- TUNABLES --------------------------------------------------------------- + +THRESHOLD = 800 # staged insertions + deletions (non-binary files) +OVERRIDE_WINDOW_SECONDS = 60 # `override rule 11` one-shot bypass window + +# --- MATCHING --------------------------------------------------------------- + +# Literal bracketed tokens. NOT substring matches. +TAG_RE = re.compile(r"\[(MVP|LARGE-CHANGE|REFACTOR|INITIAL|MERGE|REVERT)\]") +SCOPE_EXPANSION_RE = re.compile(r"\[SCOPE-EXPANSION:\s*[^\]]+\]") + +# -m / --message parsing. Matches: -m "msg" | -m 'msg' | --message="msg" | -m msg_no_quotes +MSG_RE_QUOTED = re.compile(r"""(?:-m|--message)[=\s]+(["'])(.+?)\1""", re.DOTALL) +MSG_RE_BARE = re.compile(r"""(?:-m|--message)[=\s]+(\S+)""") + +# Bypass-surface detection (fail-open for these) +AMEND_RE = re.compile(r"(? bool: + return bool(TAG_RE.search(msg) or SCOPE_EXPANSION_RE.search(msg)) + + +def extract_message(cmd: str) -> str | None: + """Return the -m / --message value, or None if absent.""" + m = MSG_RE_QUOTED.search(cmd) + if m: + return m.group(2) + m = MSG_RE_BARE.search(cmd) + if m: + return m.group(1) + return None + + +def staged_line_count(project_dir: Path) -> int | None: + """Sum insertions + deletions from git diff --cached --numstat. + Binary files show '-' for both columns and are ignored. + Returns None on any parse / subprocess error (caller should fail-open). + """ + try: + out = subprocess.check_output( + ["git", "diff", "--cached", "--numstat"], + cwd=project_dir, + stderr=subprocess.DEVNULL, + text=True, + timeout=5, + ) + except Exception: + return None + + total = 0 + for line in out.splitlines(): + parts = line.split("\t") + if len(parts) < 2: + continue + ins, dels = parts[0], parts[1] + if ins == "-" or dels == "-": + continue # binary file + try: + total += int(ins) + int(dels) + except ValueError: + continue + return total + + +def check_override(project_dir: Path) -> bool: + """Check for one-shot override marker. Delete on use. Returns True if overridden.""" + for candidate in ( + project_dir / ".claude" / "hardgate-override-rule-11", + Path.home() / ".claude" / "hardgate-override-rule-11", + ): + if not candidate.exists(): + continue + try: + age = time.time() - candidate.stat().st_mtime + if age <= OVERRIDE_WINDOW_SECONDS: + candidate.unlink(missing_ok=True) + return True + except Exception: + continue + return False + + +def main(): + try: + data = json.load(sys.stdin) + except Exception: + sys.exit(0) # fail-open + + if data.get("tool_name") != "Bash": + sys.exit(0) + + cmd = (data.get("tool_input") or {}).get("command", "").strip() + if not cmd or "git commit" not in cmd: + sys.exit(0) + + # Bypass surfaces — fail-open by design (documented above) + if AMEND_RE.search(cmd) or FILE_MSG_RE.search(cmd): + sys.exit(0) + + msg = extract_message(cmd) + if msg is None: + # Editor commit — not parseable at PreToolUse time; fail-open + sys.exit(0) + + # If an allowed tag is already present, allow through regardless of size + if has_allowed_tag(msg): + sys.exit(0) + + project_dir = Path(os.environ.get("CLAUDE_PROJECT_DIR") or os.getcwd()) + + # Override pressure valve — one-shot, 60 second window + if check_override(project_dir): + print("[HARD-RULE-11] override rule 11 accepted — bypassing once.", file=sys.stderr) + sys.exit(0) + + total = staged_line_count(project_dir) + if total is None or total <= THRESHOLD: + sys.exit(0) # under threshold, or parse error (fail-open) + + # BLOCKED + print(f"[HARD-RULE-11] BLOCKED — Commit-Size Acknowledgment Gate.", file=sys.stderr) + print("", file=sys.stderr) + print(f"Staged diff is {total} lines (threshold: {THRESHOLD}) and the commit", file=sys.stderr) + print("message contains no explicit size-acknowledgment tag.", file=sys.stderr) + print("", file=sys.stderr) + print("Add one of these literal bracketed tokens to the commit message:", file=sys.stderr) + print(" [MVP] [LARGE-CHANGE] [REFACTOR] [INITIAL] [MERGE] [REVERT]", file=sys.stderr) + print(" [SCOPE-EXPANSION: ]", file=sys.stderr) + print("", file=sys.stderr) + print("Example:", file=sys.stderr) + print(' git commit -m "[LARGE-CHANGE] service extraction + schema split"', file=sys.stderr) + print("", file=sys.stderr) + print("Or split the commit into smaller reviewable pieces.", file=sys.stderr) + print("", file=sys.stderr) + print("Pressure valve (user-only): say `override rule 11` to arm a", file=sys.stderr) + print(f"{OVERRIDE_WINDOW_SECONDS}-second one-shot bypass.", file=sys.stderr) + sys.exit(2) + + +if __name__ == "__main__": + main() diff --git a/.claude/hooks/commit-size-gate.sh b/.claude/hooks/commit-size-gate.sh new file mode 100644 index 0000000..dc3a932 --- /dev/null +++ b/.claude/hooks/commit-size-gate.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +exec python3 "$CLAUDE_PROJECT_DIR/.claude/hooks/commit-size-gate.py" diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 0000000..e6fdef3 --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,25 @@ +{ + "hooks": { + "PreToolUse": [ + { + "matcher": "Bash", + "hooks": [ + { + "type": "command", + "command": "\"$CLAUDE_PROJECT_DIR/.claude/hooks/commit-size-gate.sh\"" + } + ] + } + ], + "SessionStart": [ + { + "hooks": [ + { + "type": "command", + "command": "\"$CLAUDE_PROJECT_DIR/.claude/hooks/commit-size-session-start.sh\"" + } + ] + } + ] + } +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..2039a02 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,37 @@ +name: civiccore CI + +on: + pull_request: + branches: [main] + push: + branches: [main] + +jobs: + tests: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v5 + + - name: Set up Python 3.13 + uses: actions/setup-python@v6 + with: + python-version: "3.13" + cache: pip + + - name: Verify Docker is available + run: docker version + + - name: Pre-pull pgvector image + run: docker pull pgvector/pgvector:pg17 + + - name: Install civiccore (editable, dev extras) + run: | + pip install --upgrade pip + pip install -e .[dev] + + - name: Run smoke test + run: pytest tests/test_smoke.py -v --tb=short + + - name: Run baseline idempotency test + run: pytest tests/test_baseline_idempotency.py -v --tb=short diff --git a/CHANGELOG.md b/CHANGELOG.md index f183207..1ba56ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to **civiccore** are documented here. Format follows adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). CivicCore is the shared platform package for the -[CivicSuite](https://github.com/scottconverse/civicsuite) open-source +[CivicSuite](https://github.com/CivicSuite/civicsuite) open-source municipal operations suite. Per the CivicCore Extraction Spec section 16, breaking changes to the public API surface (Appendix A of that spec) ship as MAJOR releases; new symbols or backward-compatible behavior ship as @@ -38,6 +38,7 @@ MINOR; bug fixes ship as PATCH. - `civiccore/migrations/alembic.ini` + `civiccore/migrations/env.py` — civiccore's own Alembic wiring (`alembic_version_civiccore` version table to avoid collision with consuming modules). - `civiccore_0001_baseline_v1` migration — idempotent snapshot of the 16 civiccore-owned shared tables at records HEAD `019_encrypt_connection_config`, per ADR-0003. - `tests/test_baseline_idempotency.py` — pytest asserting the baseline runs clean on an empty DB and is a no-op against an already-populated DB. +- `.github/workflows/ci.yml` — CI workflow on `pull_request`/`push` to `main`. Runs `tests/test_smoke.py` and `tests/test_baseline_idempotency.py` on `ubuntu-latest`/Python 3.13; pre-pulls `pgvector/pgvector:pg17` so the testcontainers-managed Postgres starts cleanly. Makes the idempotency test an actual enforced merge gate (was claim-only before). ### Changed - License switched from MIT to Apache License 2.0 to match civicrecords-ai @@ -45,6 +46,11 @@ MINOR; bug fixes ship as PATCH. section 6 are being updated in the umbrella repo in the same change. - `docs/index.html` landing page added to satisfy the project's pre-push documentation gate. +- README, CONTRIBUTING, pyproject.toml, and CHANGELOG itself: stale + `scottconverse/civiccore` and `scottconverse/civicsuite` URLs corrected + to `CivicSuite/civiccore` and `CivicSuite/civicsuite` (org-hosted as of + 2026-04-23). The `scottconverse/civicrecords-ai` URLs are unchanged — + records repo has not yet been transferred to the CivicSuite org. No release sections yet — `0.1.0` ships with Phase 1 of the CivicCore extraction (shared models + audit chain), per spec section 12. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d114b79..a8b37da 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,7 +1,7 @@ # Contributing to CivicCore Thanks for considering a contribution. CivicCore is the shared platform -package for the [CivicSuite](https://github.com/scottconverse/civicsuite) +package for the [CivicSuite](https://github.com/CivicSuite/civicsuite) open-source municipal operations suite — every line of code here is consumed by every CivicSuite module, so the bar is high and the surface is deliberately small. @@ -12,7 +12,7 @@ CivicCore is at v0.1 (Phase 0 — package skeleton). Functional code lands in Phase 1 and beyond per the CivicCore Extraction Spec section 12. If you want to contribute today, the most useful work is: -- Reviewing the Extraction Spec (in scottconverse/civicsuite) and filing +- Reviewing the Extraction Spec (in CivicSuite/civicsuite) and filing issues against ambiguous wording. - Building a Phase 1 prototype against the auth + audit subsystem contracts in Appendix A of the spec. @@ -33,7 +33,7 @@ section 18 ("Contributor confusion about where to file a bug"). notification service, onboarding, the municipal systems catalog, the 50-state exemption engine, sovereignty verification scripts, shared ORM models, or shared-table Alembic migrations. - ➜ **File it here:** https://github.com/scottconverse/civiccore/issues + ➜ **File it here:** https://github.com/CivicSuite/civiccore/issues 2. **Is the bug in records-request workflow, response-letter generation, fee schedules, the records dashboards, or any records-specific UI @@ -49,7 +49,7 @@ section 18 ("Contributor confusion about where to file a bug"). 4. **Is the bug about how the modules fit together, the suite-wide roadmap, the module catalog, or cross-module documentation?** ➜ **File it in the CivicSuite umbrella:** - https://github.com/scottconverse/civicsuite/issues + https://github.com/CivicSuite/civicsuite/issues 5. **Are you reporting a security vulnerability?** Do not file it as a public issue. See "Security advisories" below. @@ -72,7 +72,7 @@ Requirements: Clone and install in editable mode with the dev extras: ```bash -git clone https://github.com/scottconverse/civiccore.git +git clone https://github.com/CivicSuite/civiccore.git cd civiccore python -m venv .venv # macOS / Linux: @@ -142,7 +142,7 @@ commit (currently: `pyproject.toml` `[project].version` and Do not file security issues as public GitHub issues. Use GitHub's private vulnerability reporting on this repository (Security tab → Report a vulnerability), or email the maintainer listed on the -[CivicSuite umbrella repo](https://github.com/scottconverse/civicsuite) +[CivicSuite umbrella repo](https://github.com/CivicSuite/civicsuite) governance page. We will acknowledge within seven days and coordinate a fix and disclosure timeline with you. diff --git a/README.md b/README.md index a8d19b2..997bd01 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # CivicCore Shared platform package for the -[CivicSuite](https://github.com/scottconverse/civicsuite) open-source +[CivicSuite](https://github.com/CivicSuite/civicsuite) open-source municipal operations suite. ## What this is @@ -33,7 +33,7 @@ pip install civiccore For now (pre-release), install from a clone: ```bash -git clone https://github.com/scottconverse/civiccore.git +git clone https://github.com/CivicSuite/civiccore.git cd civiccore pip install -e .[dev] ``` @@ -44,7 +44,7 @@ CivicCore's v0.1 public API is deliberately lean. The full list of exported symbols — which is **stable across the v0.x series** per the spec's semver policy — is published in **Appendix A of the CivicCore Extraction Spec** in -[scottconverse/civicsuite](https://github.com/scottconverse/civicsuite). +[CivicSuite/civicsuite](https://github.com/CivicSuite/civicsuite). Refer to that document; this README does not duplicate the list, so the two cannot drift. @@ -54,7 +54,7 @@ Every CivicSuite module's README declares a CivicCore version range (e.g. `civiccore >= 0.1, < 0.2`). The suite-wide compatibility matrix — which module versions work with which CivicCore versions — is maintained at -[scottconverse/civicsuite/docs/compatibility/](https://github.com/scottconverse/civicsuite/tree/main/docs/compatibility). +[CivicSuite/civicsuite/docs/compatibility/](https://github.com/CivicSuite/civicsuite/tree/main/docs/compatibility). ## License diff --git a/pyproject.toml b/pyproject.toml index 946cb5c..646e687 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,9 +80,9 @@ dev = [ ] [project.urls] -Homepage = "https://github.com/scottconverse/civiccore" -Documentation = "https://github.com/scottconverse/civicsuite" -Issues = "https://github.com/scottconverse/civiccore/issues" +Homepage = "https://github.com/CivicSuite/civiccore" +Documentation = "https://github.com/CivicSuite/civicsuite" +Issues = "https://github.com/CivicSuite/civiccore/issues" [tool.setuptools.packages.find] where = ["."] From 6fb2c6fe20b664a059a0bef14ae9321223396f33 Mon Sep 17 00:00:00 2001 From: Scott Converse Date: Fri, 24 Apr 2026 10:47:27 -0600 Subject: [PATCH 3/3] chore: untrack .claude/ (local Claude Code tooling, not a project deliverable) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Slip in 63899f7: 'git add -A' swept in a local commit-size-gate hook config under .claude/ that some skill installed in this working dir. The files contain no secrets (verified ghp_-grep), but they are machine-local Claude Code config and don't belong in the public repo. Adding .claude/ to .gitignore and untracking. Files remain on disk locally; just no longer tracked. History keeps one commit of them at 63899f7 — no force-push to rewrite, since they're not sensitive and this is a draft PR with no consumers yet. Also gitignoring .schema-dump.sql (the temporary pg_dump output used when authoring the baseline migration; regenerable on demand). --- .claude/hooks/commit-size-gate.py | 183 ------------------------------ .claude/hooks/commit-size-gate.sh | 2 - .claude/settings.json | 25 ---- .gitignore | 6 + 4 files changed, 6 insertions(+), 210 deletions(-) delete mode 100644 .claude/hooks/commit-size-gate.py delete mode 100644 .claude/hooks/commit-size-gate.sh delete mode 100644 .claude/settings.json diff --git a/.claude/hooks/commit-size-gate.py b/.claude/hooks/commit-size-gate.py deleted file mode 100644 index 1581488..0000000 --- a/.claude/hooks/commit-size-gate.py +++ /dev/null @@ -1,183 +0,0 @@ -#!/usr/bin/env python3 -""" -Commit-Size Acknowledgment Gate — Hard Rule 11 - -Blocks `git commit` when: - (a) the staged diff exceeds THRESHOLD lines (insertions + deletions, - ignoring binary files as reported by `git diff --cached --numstat`), AND - (b) the commit message parsed from `-m` / `--message` does NOT contain - one of the allowed literal bracketed tag tokens. - -Allowed tokens (literal bracketed match, NOT substring): - [MVP] [LARGE-CHANGE] [REFACTOR] [INITIAL] [MERGE] [REVERT] - [SCOPE-EXPANSION: ] - -So "fixed a bug with the MVP flow" does NOT satisfy the gate. -Only "[MVP] ..." or "[SCOPE-EXPANSION: why] ..." do. - -BYPASS SURFACES (gate is FAIL-OPEN for all of these): - - `git commit --amend` — final message not parseable at PreToolUse - - `git commit -F ` — message lives in a file the hook can't read reliably - - editor commits (no -m) — message comes from $EDITOR post-hook - - any exception in this script — fail-open by design (never lock the user out) - - `override rule 11` marker file (60-second one-shot, deleted on use) - -If you amend into a large commit, self-police with the tag on the next -real -m commit. The gate intentionally does not chase --amend/-F/editor -commits because parsing them reliably would require a post-commit hook, -which is a different enforcement surface. - -THRESHOLD is tunable — see constant at the top of main() below. -""" - -import json -import os -import re -import subprocess -import sys -import time -from pathlib import Path - -# --- TUNABLES --------------------------------------------------------------- - -THRESHOLD = 800 # staged insertions + deletions (non-binary files) -OVERRIDE_WINDOW_SECONDS = 60 # `override rule 11` one-shot bypass window - -# --- MATCHING --------------------------------------------------------------- - -# Literal bracketed tokens. NOT substring matches. -TAG_RE = re.compile(r"\[(MVP|LARGE-CHANGE|REFACTOR|INITIAL|MERGE|REVERT)\]") -SCOPE_EXPANSION_RE = re.compile(r"\[SCOPE-EXPANSION:\s*[^\]]+\]") - -# -m / --message parsing. Matches: -m "msg" | -m 'msg' | --message="msg" | -m msg_no_quotes -MSG_RE_QUOTED = re.compile(r"""(?:-m|--message)[=\s]+(["'])(.+?)\1""", re.DOTALL) -MSG_RE_BARE = re.compile(r"""(?:-m|--message)[=\s]+(\S+)""") - -# Bypass-surface detection (fail-open for these) -AMEND_RE = re.compile(r"(? bool: - return bool(TAG_RE.search(msg) or SCOPE_EXPANSION_RE.search(msg)) - - -def extract_message(cmd: str) -> str | None: - """Return the -m / --message value, or None if absent.""" - m = MSG_RE_QUOTED.search(cmd) - if m: - return m.group(2) - m = MSG_RE_BARE.search(cmd) - if m: - return m.group(1) - return None - - -def staged_line_count(project_dir: Path) -> int | None: - """Sum insertions + deletions from git diff --cached --numstat. - Binary files show '-' for both columns and are ignored. - Returns None on any parse / subprocess error (caller should fail-open). - """ - try: - out = subprocess.check_output( - ["git", "diff", "--cached", "--numstat"], - cwd=project_dir, - stderr=subprocess.DEVNULL, - text=True, - timeout=5, - ) - except Exception: - return None - - total = 0 - for line in out.splitlines(): - parts = line.split("\t") - if len(parts) < 2: - continue - ins, dels = parts[0], parts[1] - if ins == "-" or dels == "-": - continue # binary file - try: - total += int(ins) + int(dels) - except ValueError: - continue - return total - - -def check_override(project_dir: Path) -> bool: - """Check for one-shot override marker. Delete on use. Returns True if overridden.""" - for candidate in ( - project_dir / ".claude" / "hardgate-override-rule-11", - Path.home() / ".claude" / "hardgate-override-rule-11", - ): - if not candidate.exists(): - continue - try: - age = time.time() - candidate.stat().st_mtime - if age <= OVERRIDE_WINDOW_SECONDS: - candidate.unlink(missing_ok=True) - return True - except Exception: - continue - return False - - -def main(): - try: - data = json.load(sys.stdin) - except Exception: - sys.exit(0) # fail-open - - if data.get("tool_name") != "Bash": - sys.exit(0) - - cmd = (data.get("tool_input") or {}).get("command", "").strip() - if not cmd or "git commit" not in cmd: - sys.exit(0) - - # Bypass surfaces — fail-open by design (documented above) - if AMEND_RE.search(cmd) or FILE_MSG_RE.search(cmd): - sys.exit(0) - - msg = extract_message(cmd) - if msg is None: - # Editor commit — not parseable at PreToolUse time; fail-open - sys.exit(0) - - # If an allowed tag is already present, allow through regardless of size - if has_allowed_tag(msg): - sys.exit(0) - - project_dir = Path(os.environ.get("CLAUDE_PROJECT_DIR") or os.getcwd()) - - # Override pressure valve — one-shot, 60 second window - if check_override(project_dir): - print("[HARD-RULE-11] override rule 11 accepted — bypassing once.", file=sys.stderr) - sys.exit(0) - - total = staged_line_count(project_dir) - if total is None or total <= THRESHOLD: - sys.exit(0) # under threshold, or parse error (fail-open) - - # BLOCKED - print(f"[HARD-RULE-11] BLOCKED — Commit-Size Acknowledgment Gate.", file=sys.stderr) - print("", file=sys.stderr) - print(f"Staged diff is {total} lines (threshold: {THRESHOLD}) and the commit", file=sys.stderr) - print("message contains no explicit size-acknowledgment tag.", file=sys.stderr) - print("", file=sys.stderr) - print("Add one of these literal bracketed tokens to the commit message:", file=sys.stderr) - print(" [MVP] [LARGE-CHANGE] [REFACTOR] [INITIAL] [MERGE] [REVERT]", file=sys.stderr) - print(" [SCOPE-EXPANSION: ]", file=sys.stderr) - print("", file=sys.stderr) - print("Example:", file=sys.stderr) - print(' git commit -m "[LARGE-CHANGE] service extraction + schema split"', file=sys.stderr) - print("", file=sys.stderr) - print("Or split the commit into smaller reviewable pieces.", file=sys.stderr) - print("", file=sys.stderr) - print("Pressure valve (user-only): say `override rule 11` to arm a", file=sys.stderr) - print(f"{OVERRIDE_WINDOW_SECONDS}-second one-shot bypass.", file=sys.stderr) - sys.exit(2) - - -if __name__ == "__main__": - main() diff --git a/.claude/hooks/commit-size-gate.sh b/.claude/hooks/commit-size-gate.sh deleted file mode 100644 index dc3a932..0000000 --- a/.claude/hooks/commit-size-gate.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env bash -exec python3 "$CLAUDE_PROJECT_DIR/.claude/hooks/commit-size-gate.py" diff --git a/.claude/settings.json b/.claude/settings.json deleted file mode 100644 index e6fdef3..0000000 --- a/.claude/settings.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "hooks": { - "PreToolUse": [ - { - "matcher": "Bash", - "hooks": [ - { - "type": "command", - "command": "\"$CLAUDE_PROJECT_DIR/.claude/hooks/commit-size-gate.sh\"" - } - ] - } - ], - "SessionStart": [ - { - "hooks": [ - { - "type": "command", - "command": "\"$CLAUDE_PROJECT_DIR/.claude/hooks/commit-size-session-start.sh\"" - } - ] - } - ] - } -} diff --git a/.gitignore b/.gitignore index 9c20038..fe3a400 100644 --- a/.gitignore +++ b/.gitignore @@ -64,3 +64,9 @@ desktop.ini *.sqlite3 *.db .alembic-tmp/ + +# Local Claude Code project tooling (hooks, settings) — machine-local, not project deliverables +.claude/ + +# Local schema dumps used during baseline authorship — regenerate with pg_dump if needed +.schema-dump.sql