feat(phase1-b): civiccore integration (env.py + 14 guarded migrations + 3 gate tests)

CHANGELOG.md

@@ -9,6 +9,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 Post-v1.2.0 commits on `master`. No version bump yet.
 
+### Added
+- New external dependency on `civiccore` (pinned to `git+https://github.com/CivicSuite/civiccore.git@e7c5570` during Phase 1; will become a versioned wheel pin before v1.3.0).
+- Backend migrations now run the `civiccore` shared-schema baseline before records' own chain via `backend/alembic/env.py`. See [ADR-0003](https://github.com/CivicSuite/civicsuite/blob/main/docs/architecture/ADR-0003-civiccore-alembic-baseline-strategy.md) for the rationale and gate contract.
+- 3 migration gate tests at `backend/tests/test_civiccore_migration_gates.py` covering fresh-install, v1.2.x upgrade, and reapplication idempotency scenarios.
+
+### Changed
+- `Dockerfile.backend` now installs `git` during image build so pip can resolve the temporary `git+https` civiccore dependency.
+- 14 records migrations updated to use `civiccore.migrations.guards.idempotent_*` helpers, making them safe to re-apply on databases where the civiccore baseline has already created the shared tables (users, service_accounts, audit_log, data_sources, documents, document_chunks, model_registry, exemption_rules, connector_templates, departments, system_catalog, city_profile, notification_templates, prompt_templates, sync_run_log, sync_failures).
+
 ## [1.2.0] - 2026-04-23
 
 Tier 5 installer/onboarding/seeding/model-picker/portal-mode slices and Tier 6 at-rest encryption (ENG-001 closed) tagged together. CI green on `d556904` (run 24853147133). Unsigned Windows `.exe` installer produced on tag push via Inno Setup 6.x.

Dockerfile.backend

@@ -4,7 +4,11 @@ WORKDIR /app
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     build-essential libpq-dev curl \
+    git \
     && rm -rf /var/lib/apt/lists/*
+# git is required by pip to fetch the civiccore dependency from
+# git+https://github.com/CivicSuite/civiccore.git@<sha> until civiccore
+# 0.1.0 is published to PyPI. Once published, this line can drop git.
 
 COPY backend/pyproject.toml .
 RUN pip install --no-cache-dir ".[dev]"

README.md

@@ -72,6 +72,12 @@ bash install.sh
 4. Click **Ingest Now** — documents are parsed, chunked, and indexed automatically
 5. Go to **Search** — type a natural language query and get cited results
 
+### Phase 1 migration layer
+
+CivicRecords AI backend installs `civiccore` (the shared CivicSuite schema + migration runtime) as a dependency. During the Phase 1 transition period `civiccore` is pinned by git+SHA rather than PyPI version; see `backend/pyproject.toml`. Because of this, the backend Docker build requires `git` inside the image (already wired in `Dockerfile.backend`).
+
+Migrations run in two layers: `civiccore` first (creates/updates the 16 shared tables), then this repo's Alembic chain on top. See [ADR-0003](https://github.com/CivicSuite/civicsuite/blob/main/docs/architecture/ADR-0003-civiccore-alembic-baseline-strategy.md) for the full gate contract.
+
 ## Architecture
 
 ```

backend/alembic/env.py

@@ -9,7 +9,16 @@
 
 config = context.config
 if config.config_file_name is not None:
-    fileConfig(config.config_file_name)
+    # Phase 1 Part B: programmatic alembic invocation (test_civiccore_migration_gates)
+    # runs env.py in the pytest parent process. fileConfig's default
+    # disable_existing_loggers=True sets .disabled=True on every pre-existing
+    # app.* logger, which survives beyond the migration call. Subsequent tests
+    # that rely on caplog (e.g. test_structured_log_on_fetch_failure) then see
+    # empty caplog.records because pytest's caplog.at_level does NOT reset the
+    # .disabled attribute — it only adjusts .level and logging.disable().
+    # disable_existing_loggers=False is the correct posture for alembic invoked
+    # from within a Python app that owns its own logger hierarchy.
+    fileConfig(config.config_file_name, disable_existing_loggers=False)
 
 target_metadata = Base.metadata
 
@@ -26,6 +35,28 @@ def run_migrations_offline() -> None:
 
 
 def do_run_migrations(connection):
+    # Phase-1 (civiccore extraction): bring civiccore migrations to head BEFORE
+    # records' own chain. See ADR-0003 §3.
+    #
+    # Civiccore is invoked in a subprocess rather than inline because alembic's
+    # `context._proxy` is process-global, not stacked. Calling civiccore's
+    # `command.upgrade` (even on a separate connection) would tear down records'
+    # active context proxy on civiccore's __exit__, causing
+    # `AttributeError: 'NoneType' object has no attribute 'configure'` on the
+    # next records env.py line. A subprocess gives civiccore a clean alembic
+    # process. Civiccore opens its own DB connection from DATABASE_URL.
+    import subprocess
+    import sys
+
+    subprocess.run(
+        [
+            sys.executable,
+            "-c",
+            "from civiccore.migrations.runner import upgrade_to_head; upgrade_to_head()",
+        ],
+        check=True,
+    )
+
     context.configure(connection=connection, target_metadata=target_metadata)
     with context.begin_transaction():
         context.run_migrations()

backend/alembic/versions/001_initial.py

@@ -11,6 +11,11 @@
 import sqlalchemy as sa
 from sqlalchemy.dialects import postgresql
 
+from civiccore.migrations.guards import (
+    idempotent_create_index,
+    idempotent_create_table,
+)
+
 revision: str = "001"
 down_revision: Union[str, None] = None
 branch_labels: Union[str, Sequence[str], None] = None
@@ -23,7 +28,7 @@ def upgrade() -> None:
 
     # Users table (fastapi-users compatible)
     # The Enum with create_type=True on the first table creates the user_role type
-    op.create_table(
+    idempotent_create_table(
         "users",
         sa.Column("id", fastapi_users_db_sqlalchemy.generics.GUID(), nullable=False),
         sa.Column("email", sa.String(length=320), nullable=False),
@@ -37,10 +42,10 @@ def upgrade() -> None:
         sa.Column("last_login", sa.DateTime(timezone=True), nullable=True),
         sa.PrimaryKeyConstraint("id"),
     )
-    op.create_index("ix_users_email", "users", ["email"], unique=True)
+    idempotent_create_index("ix_users_email", "users", ["email"], unique=True)
 
     # Service accounts table
-    op.create_table(
+    idempotent_create_table(
         "service_accounts",
         sa.Column("id", sa.UUID(), nullable=False),
         sa.Column("name", sa.String(255), nullable=False),
@@ -54,7 +59,7 @@ def upgrade() -> None:
     )
 
     # Audit log table (append-only, hash-chained)
-    op.create_table(
+    idempotent_create_table(
         "audit_log",
         sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
         sa.Column("prev_hash", sa.String(64), nullable=False, server_default="0" * 64),
@@ -68,12 +73,12 @@ def upgrade() -> None:
         sa.Column("ai_generated", sa.Boolean(), nullable=False, server_default=sa.text("false")),
         sa.PrimaryKeyConstraint("id"),
     )
-    op.create_index("ix_audit_log_entry_hash", "audit_log", ["entry_hash"])
-    op.create_index("ix_audit_log_timestamp", "audit_log", ["timestamp"])
-    op.create_index("ix_audit_log_action", "audit_log", ["action"])
-    op.create_index("ix_audit_log_resource_type", "audit_log", ["resource_type"])
-    op.create_index("ix_audit_log_user_id", "audit_log", ["user_id"])
-    op.create_index("ix_audit_log_user_timestamp", "audit_log", ["user_id", "timestamp"])
+    idempotent_create_index("ix_audit_log_entry_hash", "audit_log", ["entry_hash"])
+    idempotent_create_index("ix_audit_log_timestamp", "audit_log", ["timestamp"])
+    idempotent_create_index("ix_audit_log_action", "audit_log", ["action"])
+    idempotent_create_index("ix_audit_log_resource_type", "audit_log", ["resource_type"])
+    idempotent_create_index("ix_audit_log_user_id", "audit_log", ["user_id"])
+    idempotent_create_index("ix_audit_log_user_timestamp", "audit_log", ["user_id", "timestamp"])
 
 
 def downgrade() -> None:

backend/alembic/versions/002_documents.py

@@ -10,13 +10,18 @@
 from sqlalchemy.dialects import postgresql
 from pgvector.sqlalchemy import Vector
 
+from civiccore.migrations.guards import (
+    idempotent_create_index,
+    idempotent_create_table,
+)
+
 revision: str = "002"
 down_revision: Union[str, None] = "001"
 branch_labels: Union[str, Sequence[str], None] = None
 depends_on: Union[str, Sequence[str], None] = None
 
 def upgrade() -> None:
-    op.create_table("data_sources",
+    idempotent_create_table("data_sources",
         sa.Column("id", sa.UUID(), nullable=False),
         sa.Column("name", sa.String(255), nullable=False),
         sa.Column("source_type", sa.Enum("upload", "directory", name="source_type", create_type=True), nullable=False),
@@ -29,7 +34,7 @@ def upgrade() -> None:
         sa.PrimaryKeyConstraint("id"),
         sa.UniqueConstraint("name"),
     )
-    op.create_table("documents",
+    idempotent_create_table("documents",
         sa.Column("id", sa.UUID(), nullable=False),
         sa.Column("source_id", sa.UUID(), sa.ForeignKey("data_sources.id"), nullable=False),
         sa.Column("source_path", sa.Text(), nullable=False),
@@ -44,10 +49,10 @@ def upgrade() -> None:
         sa.Column("metadata", postgresql.JSONB(), nullable=True),
         sa.PrimaryKeyConstraint("id"),
     )
-    op.create_index("ix_documents_source_id", "documents", ["source_id"])
-    op.create_index("ix_documents_file_hash", "documents", ["file_hash"])
-    op.create_index("ix_documents_source_hash", "documents", ["source_id", "file_hash"])
-    op.create_table("document_chunks",
+    idempotent_create_index("ix_documents_source_id", "documents", ["source_id"])
+    idempotent_create_index("ix_documents_file_hash", "documents", ["file_hash"])
+    idempotent_create_index("ix_documents_source_hash", "documents", ["source_id", "file_hash"])
+    idempotent_create_table("document_chunks",
         sa.Column("id", sa.UUID(), nullable=False),
         sa.Column("document_id", sa.UUID(), sa.ForeignKey("documents.id", ondelete="CASCADE"), nullable=False),
         sa.Column("chunk_index", sa.Integer(), nullable=False),
@@ -57,8 +62,8 @@ def upgrade() -> None:
         sa.Column("page_number", sa.Integer(), nullable=True),
         sa.PrimaryKeyConstraint("id"),
     )
-    op.create_index("ix_document_chunks_document_id", "document_chunks", ["document_id"])
-    op.create_index("ix_chunks_doc_index", "document_chunks", ["document_id", "chunk_index"])
+    idempotent_create_index("ix_document_chunks_document_id", "document_chunks", ["document_id"])
+    idempotent_create_index("ix_chunks_doc_index", "document_chunks", ["document_id", "chunk_index"])
 
 def downgrade() -> None:
     op.drop_table("document_chunks")

backend/alembic/versions/003_model_registry.py

@@ -8,13 +8,15 @@
 from alembic import op
 import sqlalchemy as sa
 
+from civiccore.migrations.guards import idempotent_create_table
+
 revision: str = "003"
 down_revision: Union[str, None] = "002"
 branch_labels: Union[str, Sequence[str], None] = None
 depends_on: Union[str, Sequence[str], None] = None
 
 def upgrade() -> None:
-    op.create_table("model_registry",
+    idempotent_create_table("model_registry",
         sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
         sa.Column("model_name", sa.String(255), nullable=False),
         sa.Column("model_version", sa.String(100), nullable=True),

backend/alembic/versions/004_search.py

@@ -49,17 +49,21 @@ def upgrade() -> None:
     op.create_index("ix_search_results_query_id", "search_results", ["query_id"])
     op.create_index("ix_search_results_chunk_id", "search_results", ["chunk_id"])
 
-    # Add tsvector column to document_chunks for full-text search
+    # Add tsvector column to document_chunks for full-text search.
+    # IF NOT EXISTS keeps this idempotent: civiccore's baseline migration
+    # (per ADR-0003) creates document_chunks with this column already present;
+    # records' own history still applies this op for v1.2.x deployments that
+    # predate the civiccore extraction. Both paths converge on the same schema.
     op.execute("""
         ALTER TABLE document_chunks
-        ADD COLUMN content_tsvector tsvector
+        ADD COLUMN IF NOT EXISTS content_tsvector tsvector
         GENERATED ALWAYS AS (to_tsvector('english', content_text)) STORED
     """)
-    op.execute("CREATE INDEX ix_chunks_tsvector ON document_chunks USING GIN (content_tsvector)")
+    op.execute("CREATE INDEX IF NOT EXISTS ix_chunks_tsvector ON document_chunks USING GIN (content_tsvector)")
 
     # HNSW index on embedding column for fast semantic search
     op.execute("""
-        CREATE INDEX ix_chunks_embedding_hnsw ON document_chunks
+        CREATE INDEX IF NOT EXISTS ix_chunks_embedding_hnsw ON document_chunks
         USING hnsw (embedding vector_cosine_ops)
         WITH (m = 16, ef_construction = 64)
     """)

backend/alembic/versions/006_exemptions.py

@@ -9,14 +9,19 @@
 import sqlalchemy as sa
 from sqlalchemy.dialects import postgresql
 
+from civiccore.migrations.guards import (
+    idempotent_create_index,
+    idempotent_create_table,
+)
+
 revision: str = "006"
 down_revision: Union[str, None] = "005"
 branch_labels: Union[str, Sequence[str], None] = None
 depends_on: Union[str, Sequence[str], None] = None
 
 
 def upgrade() -> None:
-    op.create_table("exemption_rules",
+    idempotent_create_table("exemption_rules",
         sa.Column("id", sa.UUID(), nullable=False),
         sa.Column("state_code", sa.String(2), nullable=False),
         sa.Column("category", sa.String(100), nullable=False),
@@ -28,8 +33,8 @@ def upgrade() -> None:
         sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
         sa.PrimaryKeyConstraint("id"),
     )
-    op.create_index("ix_exemption_rules_state", "exemption_rules", ["state_code"])
-    op.create_index("ix_exemption_rules_category", "exemption_rules", ["category"])
+    idempotent_create_index("ix_exemption_rules_state", "exemption_rules", ["state_code"])
+    idempotent_create_index("ix_exemption_rules_category", "exemption_rules", ["category"])
 
     op.create_table("exemption_flags",
         sa.Column("id", sa.UUID(), nullable=False),

backend/alembic/versions/011_fix_schema_drift.py

@@ -51,6 +51,8 @@
 from alembic import op
 import sqlalchemy as sa
 
+from civiccore.migrations.guards import idempotent_add_column
+
 
 # revision identifiers
 revision: str = '011_fix_drift'
@@ -76,7 +78,8 @@ def upgrade() -> None:
     #    server_default ensures any pre-existing rows backfill cleanly. The
     #    model declares default=1 (Python-side) so new ORM inserts will also
     #    pass an explicit value.
-    op.add_column(
+    #    SHARED table (exemption_rules is CivicCore-owned) — guarded.
+    idempotent_add_column(
         "exemption_rules",
         sa.Column("version", sa.Integer(), nullable=False, server_default="1"),
     )

backend/alembic/versions/012_add_liaison_public_roles.py

@@ -32,6 +32,12 @@ def upgrade() -> None:
     # PostgreSQL 12+ allows ALTER TYPE ... ADD VALUE IF NOT EXISTS inside a transaction.
     # The project targets PostgreSQL 17, so the prior COMMIT workaround (which breaks
     # asyncpg's protocol-level transaction management) is not needed.
+    #
+    # The user_role enum is a SHARED (CivicCore-owned) type — civiccore baseline
+    # already declares the full role set including 'liaison' and 'public', so on
+    # any DB that ran the baseline these statements no-op via ADD VALUE IF NOT
+    # EXISTS. Native Postgres idempotency is sufficient here; no `has_table`
+    # gate is needed because we are mutating a TYPE, not a table.
     op.execute(sa.text("ALTER TYPE user_role ADD VALUE IF NOT EXISTS 'liaison'"))
     op.execute(sa.text("ALTER TYPE user_role ADD VALUE IF NOT EXISTS 'public'"))
 

backend/alembic/versions/013_add_connector_types.py

@@ -8,20 +8,25 @@
 import sqlalchemy as sa
 from alembic import op
 
+from civiccore.migrations.guards import idempotent_add_column
+
 revision: str = '013_connector_types'
 down_revision: Union[str, None] = '012_liaison_public_roles'
 branch_labels: Union[str, Sequence[str], None] = None
 depends_on: Union[str, Sequence[str], None] = None
 
 
 def upgrade() -> None:
-    # Add new SourceType enum values
-    # PostgreSQL requires ALTER TYPE ... ADD VALUE; cannot be done inside a transaction.
+    # Add new SourceType enum values.
+    # source_type is a SHARED (CivicCore-owned) enum, but ADD VALUE IF NOT EXISTS
+    # is natively idempotent — civiccore baseline can pre-declare these and the
+    # statements still run cleanly here.
     op.execute("ALTER TYPE source_type ADD VALUE IF NOT EXISTS 'rest_api'")
     op.execute("ALTER TYPE source_type ADD VALUE IF NOT EXISTS 'odbc'")
 
-    # Add last_sync_cursor column (last_sync_at already exists)
-    op.add_column(
+    # Add last_sync_cursor column on SHARED data_sources table — guarded.
+    # (last_sync_at already exists from 787207afc66a.)
+    idempotent_add_column(
         "data_sources",
         sa.Column("last_sync_cursor", sa.String(), nullable=True),
     )