diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5271319..27c8e91 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,6 +27,7 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install -e ".[dev]" + python -m pip install "pydantic==2.13.4" python -m pip install -e packages/sqlrules-postgresql -e packages/sqlrules-sqlite \ -e packages/sqlrules-mysql -e packages/sqlrules-mssql @@ -66,6 +67,7 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install -e ".[dev]" + python -m pip install "pydantic==2.13.4" python -m pip install -e packages/sqlrules-postgresql -e packages/sqlrules-sqlite \ -e packages/sqlrules-mysql -e packages/sqlrules-mssql @@ -113,21 +115,11 @@ jobs: python - <<'PY' from typing import Annotated - from pydantic import BaseModel, Field + from pydantic import Field from sqlalchemy import Column, MetaData, String, Table import sqlrules - from sqlrules import ( - compile, - where, - flatten, - Compiler, - PLUGIN_API_VERSION, - clear_model_cache, - pattern_text, - TranslatorRegistry, - default_registry, - ) + from sqlrules import PLUGIN_API_VERSION, RuleSchema, pattern_text from sqlrules.conformance import run_basic_conformance from sqlrules_postgresql import PostgresPlugin from sqlrules_sqlite import SQLitePlugin @@ -135,17 +127,14 @@ jobs: from sqlrules_mssql import MssqlPlugin assert sqlrules.__version__ - assert callable(compile) - assert callable(where) - assert callable(flatten) - assert callable(clear_model_cache) + assert callable(sqlrules.compile) + assert callable(sqlrules.where) + assert callable(sqlrules.flatten) + assert callable(sqlrules.notwhere) assert callable(pattern_text) - assert Compiler is not None - assert TranslatorRegistry is not None - assert callable(default_registry) - assert PLUGIN_API_VERSION == "1" + assert PLUGIN_API_VERSION == "2" - class LengthFilter(BaseModel): + class LengthFilter(RuleSchema): name: Annotated[str, Field(min_length=2)] length_table = Table("items", MetaData(), Column("name", String)) @@ -178,6 +167,72 @@ jobs: - name: Sphinx run: sphinx-build -W -b html docs docs/_build/html + database-conformance: + name: Live database conformance + runs-on: ubuntu-latest + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: sqlrules + POSTGRES_PASSWORD: sqlrules-ci + POSTGRES_DB: sqlrules + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U sqlrules -d sqlrules" + --health-interval 10s + --health-timeout 5s + --health-retries 10 + mysql: + image: mysql:8.0 + env: + MYSQL_DATABASE: sqlrules + MYSQL_USER: sqlrules + MYSQL_PASSWORD: sqlrules-ci + MYSQL_ROOT_PASSWORD: root-ci + ports: + - 3306:3306 + options: >- + --health-cmd "mysqladmin ping -h 127.0.0.1 -uroot -proot-ci" + --health-interval 10s + --health-timeout 5s + --health-retries 20 + mssql: + image: mcr.microsoft.com/mssql/server:2022-latest + env: + ACCEPT_EULA: "Y" + MSSQL_SA_PASSWORD: "SqlRules-ci!Passw0rd" + ports: + - 1433:1433 + options: >- + --health-cmd "/opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P 'SqlRules-ci!Passw0rd' -C -Q 'SELECT 1' || exit 1" + --health-interval 10s + --health-timeout 5s + --health-retries 30 + env: + SQLRULES_TEST_POSTGRESQL_URL: postgresql+psycopg://sqlrules:sqlrules-ci@127.0.0.1:5432/sqlrules + SQLRULES_TEST_MYSQL_URL: mysql+pymysql://sqlrules:sqlrules-ci@127.0.0.1:3306/sqlrules + SQLRULES_TEST_MSSQL_URL: mssql+pymssql://sa:SqlRules-ci%21Passw0rd@127.0.0.1:1433/master + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + + - name: Install + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[dev]" + python -m pip install "pydantic==2.13.4" "psycopg[binary]" pymysql pymssql + python -m pip install -e packages/sqlrules-postgresql -e packages/sqlrules-sqlite \ + -e packages/sqlrules-mysql -e packages/sqlrules-mssql + + - name: Execute backend conformance corpus + run: pytest -o addopts='' -q tests/test_live_backend_conformance.py + checks: name: CI checks if: always() @@ -186,6 +241,7 @@ jobs: - test - package - docs + - database-conformance runs-on: ubuntu-latest steps: - name: Require all jobs @@ -195,6 +251,7 @@ jobs: "${{ needs.test.result }}" "${{ needs.package.result }}" "${{ needs.docs.result }}" + "${{ needs.database-conformance.result }}" ) for result in "${results[@]}"; do if [[ "$result" != "success" ]]; then @@ -202,4 +259,4 @@ jobs: exit 1 fi done - echo "All CI jobs passed (lint, test, package, docs)." + echo "All CI jobs passed (lint, test, package, docs, database conformance)." diff --git a/CHANGELOG.md b/CHANGELOG.md index 4917868..189afba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- SQLRules 2.0 `RuleSchema`, Pydantic conversion with deterministic reports, + explicit backend providers, safe scalar preparation, `CompiledRules`, and + `notwhere()`. +- Coordinated 2.0.0 package metadata, API v2 documentation, examples, and + migration guidance. + +### Changed + +- Compilation now requires SQLRules-compatible declarations and exactly one + explicit backend provider. Every supported scalar annotation is a rule. +- Backend coercion and strictness follow the published 2.0 semantic profile; + retained rules cannot be omitted with warn/ignore settings. + ## [1.0.1] - 2026-07-11 ### Fixed diff --git a/README.md b/README.md index 2bc26a7..8ce63ae 100644 --- a/README.md +++ b/README.md @@ -5,284 +5,146 @@ [![Documentation](https://readthedocs.org/projects/sqlrules/badge/?version=latest)](https://sqlrules.readthedocs.io/en/latest/) [![License](https://img.shields.io/pypi/l/sqlrules.svg)](https://github.com/eddiethedean/sqlrules/blob/main/LICENSE) -**Compile Pydantic Field constraint metadata into SQLAlchemy WHERE expressions.** +**Compile SQLRules-owned Pydantic models into safe SQLAlchemy predicates.** -SQLRules reads constraints declared on a model (`Field(ge=18)`, `min_length`, -`Literal`, …) and turns them into SQLAlchemy boolean expressions. It depends on -Pydantic v2 and SQLAlchemy 2.x, performs **no database I/O**, and needs a -dialect plugin only for non-portable operators (regex, JSON, arrays, …). +RuleSchema is a Pydantic v2 model. It validates Python data with normal +Pydantic APIs and can be used as a FastAPI request or response model. SQLRules +accepts only declarations that have a defined SQL meaning. The compiler +requires one explicit database backend and returns a complete predicate for +matching rows plus a notwhere() complement for rows that fail any rule. -```text -SQLRules does THIS (constraint metadata → expressions): - Field(ge=18) → column >= 18 +SQLRules performs no database I/O and never changes stored values. -It does NOT do this (instance values → predicates): - UserFilter(age=25) → column == 25 # not what sqlrules.compile does -``` +## Quick start -Not an ORM, validator, or query builder: **constraint metadata → expressions**, -nothing else. +Install SQLRules and the dialect provider for your database: -**In 30 seconds:** +~~~bash +pip install "sqlrules>=2,<3" "sqlrules-postgresql>=2,<3" +~~~ -```bash -pip install "sqlrules>=1,<2" -``` - -```python +~~~python from typing import Annotated -from pydantic import BaseModel, Field +from pydantic import ConfigDict, Field as PydanticField, StringConstraints from sqlalchemy import Column, Integer, MetaData, String, Table -import sqlrules +from sqlrules import Compiler, RuleSchema, notwhere, where +from sqlrules_postgresql import PostgresPlugin users = Table( "users", MetaData(), + Column("id", Integer), Column("age", Integer), Column("name", String), ) -class UserFilter(BaseModel): - age: Annotated[int, Field(ge=18, le=65)] - name: Annotated[str, Field(min_length=2)] - -rules = sqlrules.compile(UserFilter, users) -# { -# "age": [users.c.age >= 18, users.c.age <= 65], -# "name": [length(users.c.name) >= 2], -# } - -stmt = users.select().where(*sqlrules.where(rules)) -``` - -> **`pattern` needs a plugin.** Core compiles comparisons, lengths, `Literal`, -> and `Enum`. Regex / JSON / arrays require a dialect package (see below). -> Prefer static patterns; untrusted regex can be expensive (see -> [SECURITY](https://sqlrules.readthedocs.io/en/latest/SECURITY.html)). - -Full walkthrough: [Getting started](https://sqlrules.readthedocs.io/en/latest/guides/getting-started.html) · Runnable scripts: [`examples/`](examples/). - -| | | -|---|---| -| **Latest** | [sqlrules on PyPI](https://pypi.org/project/sqlrules/) · [Changelog](https://sqlrules.readthedocs.io/en/latest/project/changelog.html) | -| **Docs** | [sqlrules.readthedocs.io](https://sqlrules.readthedocs.io/en/latest/) | -| **Python** | 3.10+ (Pydantic v2, SQLAlchemy 2.x) | - ---- - -## Choose your path - -| Path | Start here | -|---|---| -| **Not sure?** | [Start here](https://sqlrules.readthedocs.io/en/latest/guides/start-here.html) | -| **Compile constraints** | [Getting started](https://sqlrules.readthedocs.io/en/latest/guides/getting-started.html) | -| **Dialect plugins** | [Plugin system](https://sqlrules.readthedocs.io/en/latest/PLUGIN_SYSTEM.html) | -| **Evaluate fit** | [Design philosophy](https://sqlrules.readthedocs.io/en/latest/guides/design-philosophy.html) | -| **Contribute** | [Contributing](https://sqlrules.readthedocs.io/en/latest/project/contributing.html) | - ---- -## Install +class UserRules(RuleSchema): + model_config = ConfigDict(strict=False) -**Requires** Python 3.10+, Pydantic v2, and SQLAlchemy 2.x (pulled in -transitively). + id: int + age: int | None = PydanticField(ge=18) + name: Annotated[str, StringConstraints(min_length=2)] -### From PyPI -```bash -pip install "sqlrules>=1,<2" -# or: uv add "sqlrules>=1,<2" -``` +# RuleSchema remains a normal Pydantic model. +request = UserRules.model_validate({"id": "12", "age": "21", "name": "Ada"}) +assert request.id == 12 -Optional dialect plugins (same major line as core): +compiler = Compiler(plugins=[PostgresPlugin(server_version=(16, 0))]) +compiled = compiler.compile(UserRules, users) -```bash -pip install "sqlrules-postgresql>=1,<2" # ~ / ~*, JSONB, ARRAY, range -pip install "sqlrules-sqlite>=1,<2" # REGEXP helper + JSON -pip install "sqlrules-mysql>=1,<2" # REGEXP, JSON, full-text -pip install "sqlrules-mssql>=1,<2" # JSON + LEN string ops -``` +matching = users.select().where(*where(compiled)) +failing = users.select().where(*notwhere(compiled)) +~~~ -Extras are equivalent shortcuts (pull the matching plugin package): +A lone scalar annotation is a rule too: id: int requires the bound column to +contain a compatible integer. Lax mode applies the documented SQLRules +conversions; Field(strict=True), a strict Pydantic alias, or +ConfigDict(strict=True) requires the database's observable logical type to +match. Optional annotations accept SQL NULL. -```bash -pip install "sqlrules[postgresql]" -pip install "sqlrules[dialects]" # all four -``` +compiled.predicate is the complete root expression. compiled.explain() shows +bindings, coercions, backend assumptions, and diagnostics without executing a +database EXPLAIN. -SQLite `pattern` emits `REGEXP` — call `sqlrules_sqlite.register_regexp(connection)` -on each connection before executing queries. +## Convert an existing Pydantic model -### From source (contributors) - -```bash -git clone https://github.com/eddiethedean/sqlrules.git -cd sqlrules -python -m venv .venv && source .venv/bin/activate -make install # editable core + all four dialect plugins -``` - -**End users** should install from PyPI. You do not need to clone this repo or -install from `packages/` for normal application use. - -### Plugin example - -```python -from typing import Annotated, Any +Unrestricted Pydantic models remain usable in application code. Convert one +explicitly before compiling it: +~~~python from pydantic import BaseModel, Field -from sqlalchemy import Column, MetaData, String, Table -from sqlalchemy.dialects.postgresql import JSONB - -import sqlrules -from sqlrules import Compiler, JsonContains -from sqlrules_postgresql import PostgresPlugin - -table = Table( - "rows", - MetaData(), - Column("name", String), - Column("meta", JSONB), -) - -class RowFilter(BaseModel): - name: Annotated[str, Field(pattern=r"^A")] - meta: Annotated[dict[str, Any], JsonContains({"active": True})] - -# dialect= is a hint for custom translators only — it does not load plugins. -compiler = Compiler(plugins=[PostgresPlugin()], dialect="postgresql") -rules = compiler.compile(RowFilter, table) -stmt = table.select().where(*sqlrules.where(rules)) -``` - -See [`examples/postgresql_pattern.py`](examples/postgresql_pattern.py). - -### Common failures - -| Symptom | Fix | -|---|---| -| `UnsupportedConstraintError` on `pattern` | Install a dialect plugin and pass it to `Compiler(plugins=[...])` | -| `NameError: sqlrules` after `from sqlrules import Compiler` | Also `import sqlrules` (or import `where`) | -| SQLite `REGEXP` errors at execute time | Call `register_regexp(connection)` | - -More: [Troubleshooting](https://sqlrules.readthedocs.io/en/latest/guides/troubleshooting.html). - -## Supported constraints (1.0) - -| Constraint | SQLAlchemy expression | -|---|---| -| `gt` / `ge` / `lt` / `le` | `column > / >= / < / <= value` | -| `multiple_of` | `column % value == 0` | -| `min_length` / `max_length` | `func.length(column) >= / <= value` | -| `Literal[...]` | `column.in_(...)` | -| `Enum` | `column.in_(...)` | - -`pattern` is extracted into IR (`PatternSpec`) but has **no portable core -translator**. Install a dialect plugin or register a custom translator. - -With `emit_type_checks=True`, supported scalar annotations also emit -`type_check` (`TypeSpec`) IR — likewise plugin-translated only. - -Dialect markers (`JsonContains`, `ArrayContains`, `RangeContains`, -`FullTextMatch`, …) are re-exported from `sqlrules` (and live in -`sqlrules.markers`); they require a dialect plugin. - -Unsupported constraints raise `UnsupportedConstraintError` by default. -Use `on_unsupported="warn"` or `"ignore"` for unknown **operators**. -Unconstrained fields with **supported** types are omitted from the rules -dict. Fields with **unsupported** type annotations always raise — even if -unconstrained. - -## When *not* to use SQLRules - -Skip SQLRules when: - -- You need **request/instance values** as WHERE predicates - (`UserFilter(age=25)` → `age = 25`). That is a query/filter library, not - this compiler. -- You have a few static expressions and will never share constraint metadata - with a Pydantic model — write SQLAlchemy clauses directly. -- You need **portable regex** with core alone — `pattern` requires a dialect - plugin or custom translator. -- You expect `dialect="postgresql"` to **load plugins** — it is a hint only; - pass `plugins=[...]` explicitly. -- You want a general-purpose **query builder**, ORM, or SQL string generator. - -SQLRules pays off when the Pydantic model *is* the shared source of truth for -constraint metadata across many fields or dialects. - -## Public API - -```python -sqlrules.compile(model, table, *, column_map=None, on_unsupported="raise", cache=True, emit_type_checks=False) -sqlrules.where(rules) # prefer this to flatten expressions -sqlrules.flatten(rules) # alias of where() -sqlrules.clear_model_cache() # drop process-wide Phase-1 IR cache - -compiler = sqlrules.Compiler( - plugins=[...], # optional SQLRulesPlugin instances - on_conflict="raise", # raise | replace | ignore - dialect=None, # hint only — does not load plugins - on_unsupported="raise", - cache=True, - emit_type_checks=False, # opt-in type_check IR (needs plugin translator) -) -``` - -## Non-goals - -SQLRules is not an ORM, validator, query builder, SQL string generator, -migration tool, or database client. It only compiles supported Pydantic -constraints into SQLAlchemy expressions. See -[design philosophy](https://sqlrules.readthedocs.io/en/latest/guides/design-philosophy.html) -and [non-goals](https://sqlrules.readthedocs.io/en/latest/NON_GOALS.html). - -## Documentation - -Full site: **[sqlrules.readthedocs.io](https://sqlrules.readthedocs.io/en/latest/)** - -| Topic | Link | -|---|---| -| Getting started | [Guide](https://sqlrules.readthedocs.io/en/latest/guides/getting-started.html) | -| How-to guides | [ORM binding](https://sqlrules.readthedocs.io/en/latest/guides/orm-column-map.html) · [Markers](https://sqlrules.readthedocs.io/en/latest/guides/markers.html) · [Examples](https://sqlrules.readthedocs.io/en/latest/guides/examples.html) | -| Upgrade | [0.x → 1.0](https://sqlrules.readthedocs.io/en/latest/guides/upgrade-0x.html) | -| Examples | [`examples/`](examples/) | -| Spec & constraints | [SPEC](https://sqlrules.readthedocs.io/en/latest/SPEC.html) · [CONSTRAINTS](https://sqlrules.readthedocs.io/en/latest/CONSTRAINTS.html) | -| Plugins | [Plugin system](https://sqlrules.readthedocs.io/en/latest/PLUGIN_SYSTEM.html) | -| API reference | [Reference hub](https://sqlrules.readthedocs.io/en/latest/reference/index.html) | -| FAQ | [FAQ](https://sqlrules.readthedocs.io/en/latest/guides/faq.html) | -| Support | [Support & compatibility](https://sqlrules.readthedocs.io/en/latest/project/support.html) | -| Roadmap | [Roadmap](https://sqlrules.readthedocs.io/en/latest/project/roadmap.html) | -| Changelog | [Changelog](https://sqlrules.readthedocs.io/en/latest/project/changelog.html) | - -```bash -pip install -e ".[docs]" -make docs -``` - -## Repo map - -| Path | Who cares | -|---|---| -| PyPI `sqlrules` / `sqlrules-*` | **Application users** — install these | -| [`examples/`](examples/) | Copy-paste runnable scripts | -| [`docs/`](docs/) | Spec, guides, architecture (also on Read the Docs) | -| [`packages/`](packages/) | **Contributors** — official dialect plugin sources | -| [`src/sqlrules/`](src/sqlrules/) | Core library source | -| [`benchmarks/`](benchmarks/) | Optional local compile benchmarks | - -## Development (contributors) - -```bash -make install -make check # lint, types, tests, docs (matches most of CI) -make dist # packaging smoke — required before release / to match full CI -``` - -See [CONTRIBUTING.md](CONTRIBUTING.md). Maintainers: [RELEASING.md](RELEASING.md). - -## License - -MIT +from sqlrules.integrations.pydantic import from_pydantic + + +class ApiFilter(BaseModel): + age: int = Field(ge=18) + name: str + + +conversion = from_pydantic(ApiFilter, on_incompatible="warn") +RulesModel = conversion.model +report = conversion.report +~~~ + +The converter returns a usable RuleSchema class and a deterministic report. +It preserves supported types, constraints, defaults, aliases, and descriptive +metadata. It reports Python validators, serializers, computed fields, +unsupported metadata, and other behavior it removes or changes. Conversion +policies are warn (default), drop (return the report without warnings), and +raise (reject any semantic loss). Default factories and Python callbacks are +never executed to infer SQL rules. + +## Supported declarations + +The scalar vocabulary is bool, int, float, Decimal, str, date, datetime, time, +and UUID, plus homogeneous Literal and Enum domains, and nullable forms. +Supported constraints include numeric bounds, multiple_of, string lengths, +patterns, literal/enum membership, and the existing dialect markers. Use +Pydantic Field, ConfigDict, strict aliases, constrained aliases, and supported +Annotated metadata directly. + +Unsupported declarations fail when a RuleSchema class is created. Custom +validators, serializers, computed fields, arbitrary annotation metadata, +string normalization, general unions, nested item validation, and arbitrary +Python transforms do not have a 2.0 SQL representation. + +The [type and backend support matrix](docs/TYPE_SUPPORT.md) lists supported +source types, conversions, database versions, collation assumptions, and +compile-time capability errors. Unsupported backend behavior is never skipped +by a warning or ignore policy. + +## Dialect providers + +The four official packages are versioned with the core package: + +~~~bash +pip install "sqlrules-postgresql>=2,<3" +pip install "sqlrules-sqlite>=2,<3" +pip install "sqlrules-mysql>=2,<3" +pip install "sqlrules-mssql>=2,<3" +pip install "sqlrules[dialects]" # all four +~~~ + +Select exactly one backend provider when compiling. PostgreSQL text-to-scalar +coercion requires PostgresPlugin(server_version=(16, 0)); MySQL text-to-int +coercion requires MySQL 8.0+; SQL Server safe text conversions require SQL +Server 2012+. SQLite text checks use runtime typeof() and need the bundled +register_sqlite_functions() helper on each connection when text coercion, +patterns, or string length constraints are used. The older register_regexp() +helper remains as an alias. Unsupported combinations raise CapabilityError +during compilation. + +## Project links + +- [Getting started](https://sqlrules.readthedocs.io/en/latest/guides/getting-started.html) +- [API reference](https://sqlrules.readthedocs.io/en/latest/API.html) +- [Plugin API v2](https://sqlrules.readthedocs.io/en/latest/PLUGIN_SYSTEM.html) +- [Migration from 1.x](https://sqlrules.readthedocs.io/en/latest/guides/upgrade-1x.html) +- [Security](https://sqlrules.readthedocs.io/en/latest/SECURITY.html) +- [Runnable examples](examples/) + +Requires Python 3.10+, Pydantic v2, and SQLAlchemy 2.x. diff --git a/ROADMAP.md b/ROADMAP.md index 21270b0..6229a95 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,7 +1,8 @@ # Roadmap The 2.x series introduces SQLRules-owned rule schemas and an explicit Pydantic -conversion bridge. This is a planned design; the current shipped line is 1.x. +conversion bridge. Phase 2.0 is in implementation; its release gates remain +open until database conformance and package checks pass. API sketches and semantic decisions live in [the 2.0 design](docs/V2_DESIGN.md), with implementation steps and release gates in [Milestones](docs/MILESTONES.md). @@ -16,7 +17,7 @@ Core and all four official dialect packages remain versioned together. | 2.2 | 2.2.0 | Nested JSON schemas, tagged unions, and collection rules | | 2.3 | 2.3.0 | Named transforms, custom type mappings, schema export, and integration tools | -## 2.0.0 — Rule schemas and semantic foundations (planned) +## 2.0.0 — Rule schemas and semantic foundations (implementation in progress) - Own `RuleSchema`, `Field`, and `RuleConfig` on Pydantic v2. Models retain normal construction, validation, serialization, and FastAPI use. Restrict @@ -42,8 +43,8 @@ Core and all four official dialect packages remain versioned together. semantic changes. The original remains a normal Pydantic model, and the converted result is also a usable Pydantic model. Compilation always enforces every retained rule. -- Introduce plugin API v2, explicit backend capabilities, bounded schema caches, - and independent compilation state. Provide migration from direct Pydantic +- Introduce plugin API v2, explicit backend capabilities, immutable normalized + schema metadata, and independent compilation state. Provide migration from direct Pydantic input, bare rule dictionaries, and opt-in type checks. Release gate: freeze and implement a useful capability matrix for all four @@ -91,8 +92,8 @@ needed to preserve existing result sets. ## Quality and compatibility across 2.x -Database execution conformance, compiler benchmarks, diagnostics, concurrency, -cache bounds, documentation, and package checks begin in 2.0 and continue through +Database execution conformance, compiler benchmarks, diagnostics, concurrent +compilation behavior, documentation, and package checks begin in 2.0 and continue through every phase. Later releases extend the supported vocabulary without silently changing the acceptance rules of existing schemas. Pydantic compatibility is a versioned, documented subset with explicit SQL storage differences. diff --git a/benchmarks/bench_compile.py b/benchmarks/bench_compile.py index 7e0a2f9..caf2968 100644 --- a/benchmarks/bench_compile.py +++ b/benchmarks/bench_compile.py @@ -1,9 +1,4 @@ -"""Compile benchmarks for small / medium / large models (cold vs warm cache). - -Illustrative targets from docs/PERFORMANCE.md (not enforced): - Small (< 5 fields) < 0.5 ms - Medium (~25 fields) < 2 ms - Large (100+ fields) < 10 ms +"""Compile benchmarks for small, medium, and large RuleSchema classes. Run:: @@ -15,21 +10,22 @@ import timeit from typing import Annotated, Any -from pydantic import BaseModel, Field, create_model +from pydantic import Field, create_model from sqlalchemy import Column, Integer, MetaData, String, Table +from sqlrules_sqlite import SQLitePlugin import sqlrules -from sqlrules.cache import ModelIRCache +from sqlrules.models import RuleSchema -def _build_model(n_fields: int) -> type[BaseModel]: +def _build_model(n_fields: int) -> type[RuleSchema]: fields: dict[str, Any] = {} for i in range(n_fields): if i % 2 == 0: fields[f"n{i}"] = (Annotated[int, Field(ge=0, le=100)], ...) else: fields[f"s{i}"] = (Annotated[str, Field(min_length=1, max_length=50)], ...) - return create_model(f"Filter{n_fields}", **fields) # type: ignore[call-overload] + return create_model(f"Filter{n_fields}", __base__=RuleSchema, **fields) def _build_table(n_fields: int) -> Table: @@ -43,22 +39,22 @@ def _build_table(n_fields: int) -> Table: return Table(f"t{n_fields}", metadata, *columns) -def _bench(label: str, model: type[BaseModel], table: Table, *, repeats: int = 50) -> None: - cache = ModelIRCache() - compiler = sqlrules.Compiler(cache=True, model_cache=cache) - - # Cold: empty cache - cache.clear() - cold = timeit.timeit(lambda: compiler.compile(model, table), number=1) - - # Warm: IR already cached - warm = min(timeit.repeat(lambda: compiler.compile(model, table), number=1, repeat=repeats)) - - print(f"{label:12} cold={cold * 1000:8.3f} ms warm={warm * 1000:8.3f} ms") +def _bench(label: str, model: type[RuleSchema], table: Table, *, repeats: int = 50) -> None: + compiler = sqlrules.Compiler(plugins=[SQLitePlugin()]) + normalized = compiler.compile_model(model) + normalization = min( + timeit.repeat(lambda: compiler.compile_model(model), number=1, repeat=repeats) + ) + binding = min(timeit.repeat(lambda: compiler.bind(normalized, table), number=1, repeat=repeats)) + full = min(timeit.repeat(lambda: compiler.compile(model, table), number=1, repeat=repeats)) + print( + f"{label:12} normalize={normalization * 1000:8.3f} ms " + f"bind={binding * 1000:8.3f} ms full={full * 1000:8.3f} ms" + ) def main() -> None: - print("SQLRules compile benchmarks (cache cold vs warm)") + print("SQLRules compile benchmarks (best of repeated local runs)") print("-" * 56) for size, n in (("small", 5), ("medium", 25), ("large", 100)): model = _build_model(n) diff --git a/docs/API.md b/docs/API.md index 5cc987b..48f9f23 100644 --- a/docs/API.md +++ b/docs/API.md @@ -1,209 +1,169 @@ # Public API -SQLRules exposes three stability tiers. Semver applies to the -**Application** and **Plugin** tiers. The **Internal** tier may change -in minor releases without notice. +SQLRules 2.0 has an application API, a versioned plugin API, and internal +implementation modules. Application and plugin APIs follow semantic +versioning. Internal modules may change without notice. ------------------------------------------------------------------------- - -## Application API (stable) - -Primary surface for application code: +## Application API | Symbol | Role | |---|---| -| `compile` | One-shot compile (no plugins) | -| `where` / `flatten` | Flatten a rules dict (identical aliases; both supported) | -| `Compiler` | Reusable compiler with plugins / registry / cache | -| `clear_model_cache` | Clear the process-wide default Phase-1 IR cache | -| Exception hierarchy under `SQLRulesError` | Fail-fast errors | -| `__version__` | Package version | -| Markers (`JsonContains`, …) | `Annotated` metadata for dialect operators | - -```python -sqlrules.compile( - model, - table, - *, - column_map=None, - on_unsupported="raise", - cache=True, - emit_type_checks=False, -) -> dict[str, list[ColumnElement[bool]]] -``` - -| Parameter | Description | -|---|---| -| `model` | Pydantic `BaseModel` subclass | -| `table` | SQLAlchemy `Table`, alias, ORM class, or object with `.c` | -| `column_map` | Optional explicit field/alias → column mapping | -| `on_unsupported` | `"raise"` (default), `"warn"`, or `"ignore"` for unknown **operators** | -| `cache` | Cache Phase-1 model IR (default `True`) | -| `emit_type_checks` | When `True`, emit `type_check` IR for supported scalars (needs a plugin translator) | +| RuleSchema | Pydantic v2 BaseModel restricted to SQL-compilable declarations | +| Field | Pydantic Field wrapper with SQLRules-only column binding metadata | +| RuleConfig | SQLRules model options, including explicit allow_empty | +| from_pydantic | Convert an unrestricted Pydantic class and return a loss report | +| Compiler / compile | Normalize a RuleSchema and bind it through one explicit backend | +| CompiledRules | Complete total predicate, field results, diagnostics, and explain plan | +| where / flatten | One-item list containing the complete root predicate | +| notwhere | One-item list containing the root predicate complement | +| CapabilityError | Backend cannot prove or implement a retained rule | +| SQLRulesError hierarchy | Stable public compilation errors | -Rule dictionary keys are always the Python field names. String field aliases -are used only for column binding. Unconstrained fields are omitted from the -rules dict (unless `emit_type_checks=True`) but **must still use a supported -type annotation**. +Example: -Unsupported **types** always raise, regardless of `on_unsupported`. +~~~python +from sqlrules import Compiler, RuleSchema, notwhere, where +from sqlrules_postgresql import PostgresPlugin -Module-level `compile` does not accept plugins; use `Compiler`. -**Raises** (typical): `InvalidModelError`, `MissingColumnError`, -`UnsupportedConstraintError`, `TranslatorError`, `ConfigurationError`. -See [ERRORS](ERRORS.md). +class UserRules(RuleSchema): + id: int + age: int | None -### `where` / `flatten` -```python -sqlrules.where(rules) -> list[ColumnElement[bool]] -sqlrules.flatten(rules) -> list[ColumnElement[bool]] -``` +compiler = Compiler(plugins=[PostgresPlugin(server_version=(16, 0))]) +compiled = compiler.compile(UserRules, users) +matches = users.select().where(*where(compiled)) +failures = users.select().where(*notwhere(compiled)) +~~~ -Identical aliases. Both are part of the stable Application API. +The module-level compile() has the same explicit provider requirement: -### `clear_model_cache` +~~~python +sqlrules.compile(model, table, *, plugins, column_map=None) -> CompiledRules +~~~ -```python -sqlrules.clear_model_cache() -> None -``` +model must be a RuleSchema class. Direct compilation of an unrestricted +Pydantic BaseModel raises InvalidModelError; call from_pydantic() first. +column_map maps Python field names to SQLAlchemy columns. Field(column=...) +selects a column name without changing Pydantic aliases. -Clears the process-wide default Phase-1 `ModelIR` cache. Call this when -creating many ephemeral models (for example `pydantic.create_model`) so -cached IR does not grow without bound. Compilers constructed with a custom -`model_cache=` are unaffected. +## RuleSchema behavior -### `Compiler` +RuleSchema is a full Pydantic v2 model. Instances support normal construction, +model_validate(), model_dump(), Pydantic validation errors, and FastAPI route +integration. It also validates declarations during class construction. -```python -compiler = sqlrules.Compiler( - on_unsupported="raise", - registry=None, - plugins=None, - on_conflict="raise", - dialect=None, - cache=True, - model_cache=None, - emit_type_checks=False, -) -rules = compiler.compile(model, table, column_map=None) +Supported declarations include scalar annotations, homogeneous Literal and +Enum domains, nullable fields, supported Pydantic constraints, strict aliases, +and supported metadata in Annotated. Type-only scalar annotations compile a +type predicate. Pydantic defaults and default factories keep their runtime +behavior but never become SQL defaults or bypass column checks. -# Two-phase (advanced Application API): -model_ir = compiler.compile_model(model) -rules = compiler.bind(model_ir, table, column_map=None) -compiler.diagnostics # from the last bind/compile (translate phase) -``` +SQL-only column mapping is written with SQLRules Field: -| Parameter | Description | -|---|---| -| `plugins` | Optional `SQLRulesPlugin` instances registered at init | -| `on_conflict` | Default for plugin `register()` / `register_constraint()`: `"raise"`, `"replace"`, `"ignore"` | -| `dialect` | **Hint only** for custom translators on `CompilationContext`. Does **not** load plugins or change built-ins. Pass `plugins=[...]` explicitly. | -| `registry` | Optional base `TranslatorRegistry`; always **copied** into the compiler | -| `emit_type_checks` | Opt-in `type_check` IR from scalar annotations (`TypeSpec`) | +~~~python +from sqlrules import Field, RuleSchema -**Mutation:** do not call `compiler.registry.register(...)` (or -`register_constraint`) after construction. Register translators via -`plugins=` at init. -**Concurrency:** do not call `compile` / `bind` / `compile_model` concurrently -on the same `Compiler` instance. The shared Phase-1 IR cache is thread-safe -across instances; call `clear_model_cache()` if you create many ephemeral -models. +class UserRules(RuleSchema): + display_name: str = Field(column="name") +~~~ ------------------------------------------------------------------------- +An empty RuleSchema is rejected unless it declares +__rule_config__ = RuleConfig(allow_empty=True). An allowed empty schema +compiles to TRUE. -## Plugin API (stable) +## Pydantic conversion -For dialect packages and custom translators. Import from `sqlrules`: +~~~python +from sqlrules.integrations.pydantic import from_pydantic -| Symbol | Role | -|---|---| -| `PLUGIN_API_VERSION` | Contract version string (`"1"`) — exact match required | -| `SQLRulesPlugin` | Protocol: `name`, `api_version`, `register(registry)` | -| `TranslatorRegistry` | Register / lookup / copy translators | -| `default_registry` | Copy of built-in portable translators | -| `pattern_text` | Unpack `PatternSpec` or `str` → `(pattern, ignore_case)` | -| `type_spec` / `TypeSpec` | Helpers for `type_check` IR (opt-in `emit_type_checks`) | -| `Constraint`, `PatternSpec`, `CompilationContext` | IR types used by translators | -| `ModelIR` | Two-phase / caching IR root | -| `ConstraintMarker` + marker dataclasses | Dialect operator metadata | -| `SQLRulesWarning` | Warning class used by `on_unsupported="warn"` | -| `sqlrules.conformance` | Test helpers for plugin authors (supported) | - -Prefer `registry.register_constraint(..., on_conflict=...)`. -`register(..., replace=)` remains as a thin compatibility alias. - -```python -from sqlrules import ( - PLUGIN_API_VERSION, - Compiler, - TranslatorRegistry, - pattern_text, +conversion = from_pydantic( + ApiModel, + on_incompatible="warn", # warn, drop, or raise ) +RulesModel = conversion.model +report = conversion.report +~~~ -class MyPlugin: - name = "my-plugin" - api_version = PLUGIN_API_VERSION +The conversion report records retained rules, descriptive metadata, +unsupported callbacks, dropped fields or constraints, and changed or unknown +semantics. Conversion never runs validators, serializers, or default +factories. - def register(self, registry: TranslatorRegistry) -> None: - registry.register_constraint( - "pattern", - lambda c, col, ctx: col.op("~")(pattern_text(c.value)[0]), - on_conflict="replace", - ) +## CompiledRules -compiler = Compiler(plugins=[MyPlugin()], dialect="postgresql") -``` +CompiledRules is an immutable result containing: -### `PLUGIN_API_VERSION` policy +- predicate: the complete SQLAlchemy predicate with total TRUE/FALSE behavior. +- fields: ordered FieldResult entries containing field predicates, bindings, + logical types, strictness, nullability, coercions, and capabilities. +- diagnostics: compile-scoped structured diagnostics. +- backend, server_version, and assumptions. +- explain(): a structured plan; it does not execute database EXPLAIN. -Version `"1"` requires an **exact** string match (`api_version == "1"`). -It includes `PatternSpec` for `pattern` values. Always use -`pattern_text(constraint.value)` — do not assume a bare `str`. +where(compiled) returns [compiled.predicate]. notwhere(compiled) returns +[~compiled.predicate]. flatten(compiled) remains an alias for where(). Each +helper accepts only a CompiledRules result. -Bump `PLUGIN_API_VERSION` to a new string (for example `"2"`) only when -changing translator signatures, registry methods, or IR value types for -built-in operators. See [PLUGIN_SYSTEM.md](PLUGIN_SYSTEM.md) and -[IR_CONTRACT.md](IR_CONTRACT.md). +## Compiler -Frozen marker operator names: `json_contains`, `json_has_key`, -`array_contains`, `array_overlap`, `range_contains`, `range_overlap`, -`fulltext_match`. - -`register_type`, `register_dialect`, and `register_compiler_pass` are -**not** present on `TranslatorRegistry` in API v1 — do not probe with -`hasattr`. +~~~python +compiler = Compiler( + plugins=[PostgresPlugin(server_version=(16, 0))], + registry=None, + on_conflict="raise", + dialect=None, +) +schema_ir = compiler.compile_model(UserRules) +compiled = compiler.bind(schema_ir, users, column_map=None) +~~~ ------------------------------------------------------------------------- +Exactly one backend provider must be present when bind() or compile() runs. +Constraint-only plugins may be included with it. on_conflict controls plugin +registration. Unsupported retained operators always raise; warn and ignore +policies are not available because they could omit a declared rule. -## Internal API (unstable) +compile_model() normalizes the schema without a database backend. bind() +resolves columns and uses a backend provider. The compiler keeps an immutable +registry snapshot and per-call diagnostics, so independent compiles do not +share mutable compilation state. -Not covered by semver. Prefer Application/Plugin imports. +## Plugin API -- `sqlrules.inspectors`, `sqlrules.columns`, `sqlrules.cache` helpers -- `DiagnosticsCollector`, private translator factories -- Module layout details +| Symbol | Role | +|---|---| +| PLUGIN_API_VERSION | Exact API contract string, currently 2 | +| SQLRulesPlugin | name, api_version, and register(registry) protocol | +| BackendProvider | Adds prepare_value() and capabilities() | +| TranslatorRegistry | Copyable registry for normalized-expression translators | +| PreparedValue | Source, normalized value, validity, null state, and provenance | +| Constraint, PatternSpec, CompilationContext | Translator-facing IR types | +| ConstraintMarker and marker dataclasses | Dialect operator metadata | -See [INTERNAL_API.md](INTERNAL_API.md). +The translator's expression argument is the backend-prepared value, not the +raw source column. Backend providers are responsible for safe conversion and +source type evidence. API v1 plugins must be adapted to these requirements. ------------------------------------------------------------------------- +See [PLUGIN_SYSTEM](PLUGIN_SYSTEM.md) and [IR_CONTRACT](IR_CONTRACT.md). -## Exceptions +## Internal API -All public exceptions inherit from `SQLRulesError`: +Implementation modules such as sqlrules.models, sqlrules.backend, +sqlrules.columns, sqlrules.constraints, and private translator factories are +not compatibility surfaces. Import public types from sqlrules instead. -- `InvalidModelError` -- `MissingColumnError` -- `UnsupportedConstraintError` -- `TranslatorError` -- `InvalidTranslatorError` -- `RegistryError` -- `ConfigurationError` -- `PluginError` -- `InternalCompilerError` (reserved) +## Exceptions -`SQLRulesWarning` is used when `on_unsupported="warn"`. +- InvalidModelError +- MissingColumnError +- UnsupportedConstraintError +- CapabilityError +- TranslatorError / InvalidTranslatorError +- RegistryError +- ConfigurationError +- PluginError +- InternalCompilerError -See [ERRORS.md](ERRORS.md). +See [ERRORS](ERRORS.md). diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 484281b..921b49c 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -1,38 +1,68 @@ # Architecture -This page describes the shipped 1.x compiler. The proposed 2.0 schema, -coercion, and result architecture is in [the 2.0 design](V2_DESIGN.md). +SQLRules compiles an immutable RuleSchema description into one total SQL +predicate. The compiler does not connect to a database or execute statements. -```text -Pydantic Model +~~~text +RuleSchema class ↓ -inspectors.inspect_model → FieldDescriptor[] +Pydantic declaration validation → immutable SchemaSpec ↓ -constraints.extract_constraints → Constraint IR (PatternSpec / markers) +backend provider: source type evidence and safe PreparedValue ↓ -cache.ModelIRCache (optional) → ModelIR +constraint translators → grouped field predicates ↓ -columns.resolve_column - ↓ -translators.TranslatorRegistry.translate - ↓ -Rule dictionary (+ diagnostics) -``` +CompiledRules.predicate → where() / notwhere() +~~~ + +## Declaration normalization + +RuleSchema subclasses Pydantic BaseModel. Class creation checks supported +scalar types, nullability, strictness, Pydantic Field metadata, Annotated +markers, model configuration, and SQLRules column bindings. The resulting +SchemaSpec preserves source order, runtime metadata, constraints, and +conversion provenance. Unsupported Python-only callbacks and incompatible +declarations fail before a compiler is constructed. + +Unrestricted Pydantic classes enter through from_pydantic(). The converter +returns a generated RuleSchema and a report of retained rules, metadata, and +removed or changed behavior. It never executes validators, serializers, or +default factories. + +## Backend preparation + +The caller selects exactly one BackendProvider. It receives a bound source +column and normalized RuleField, reports its versioned capabilities, and +returns PreparedValue: + +- source: original bound column expression. +- value: safe normalized SQL expression for constraints. +- valid: non-null boolean for supported type/coercion behavior. +- is_null: whether the original source is SQL NULL. +- logical_type, coercion, capability: explain-plan evidence. + +Known row mismatches become false. Missing type evidence or unsafe conversions +raise CapabilityError during compilation. The provider must not rely on a +separate boolean expression to guard a potentially failing CAST. -SQLRules is a pure compiler pipeline with two phases: +## Predicate assembly -**Phase 1 — static model IR** +Each field combines its null branch, type validity, and all constraints into +one group. Field groups combine under one root AND. Every predicate is +normalized from SQL UNKNOWN to false, so notwhere() is a complete complement +of where(). -1. **Inspect** the Pydantic model and preserve field declaration order. -2. **Extract** supported constraints into a dialect-neutral IR. -3. **Cache** immutable `ModelIR` keyed by model class (optional). +CompiledRules retains the root predicate, ordered field results, compile-local +diagnostics, backend version, assumptions, and explain() output. It contains +table-bound expressions and is never stored in a process-wide cache. -**Phase 2 — bind and translate** +## Concurrency and caches -4. **Resolve** each constrained field to a SQLAlchemy column. -5. **Translate** each IR constraint into one SQLAlchemy expression. -6. **Assemble** expressions into `dict[str, list[ColumnElement[bool]]]`. +Each compile has a local diagnostics collector and context. Compiler instances +hold private immutable registry snapshots; callers can inspect a copy of the +registry. SQLRules does not maintain a global mutable schema cache. RuleSchema +classes retain Pydantic's own compiled validators, while SQLRules normalizes +the field declarations per compile. -The compiler never connects to a database, executes SQL, or renders SQL -strings. See [COMPILER.md](COMPILER.md) and [INTERNAL_API.md](INTERNAL_API.md) -for module-level detail. +See [COMPILER](COMPILER.md), [IR_CONTRACT](IR_CONTRACT.md), and +[TYPE_SUPPORT](TYPE_SUPPORT.md). diff --git a/docs/COMPILER.md b/docs/COMPILER.md index 0016de0..2ae19a3 100644 --- a/docs/COMPILER.md +++ b/docs/COMPILER.md @@ -1,180 +1,101 @@ # SQLRules Compiler Architecture -## Purpose - -The SQLRules compiler transforms a constrained Pydantic model into a -dictionary of SQLAlchemy WHERE expressions. - -The compiler never connects to a database, executes SQL, or generates -SQL strings. It only produces SQLAlchemy expression objects. +The compiler normalizes SQLRules-owned Pydantic schemas, asks one explicitly +selected backend to prepare each bound column, translates retained constraints, +and combines the results into one total predicate. It does not connect to a +database or execute SQL. ## Pipeline -``` text -Pydantic Model - │ - ▼ -Model Introspection - │ - ▼ -Field Extraction - │ - ▼ -Constraint Extraction - │ - ▼ -Intermediate Representation (IR) - │ - ▼ -Constraint Translators - │ - ▼ -SQLAlchemy Expressions - │ - ▼ -Rule Dictionary -``` - -## Stage 1 -- Model Introspection - -Inputs: - Pydantic BaseModel subclass - -Responsibilities: - Validate the input model. - Enumerate model -fields. - Preserve declaration order. - -Output: - Iterable of field definitions (later cached as `ModelIR`). - -Table / column binding happens in Phase 2 (`bind`), not Stage 1. - -## Stage 2 -- Field Extraction - -For each field: - -- Python type -- Optionality -- Metadata -- Field name -- Alias (optional) - -Unsupported field definitions fail fast. - -## Stage 3 -- Constraint Extraction - -Read only constraints that have deterministic SQL equivalents. - -Examples: - gt - ge - lt - le - multiple_of - min_length - max_length - -Literal - Enum - -`pattern` is normalized into IR (`operator="pattern"`) but has no -portable core translator. - -Constraints without SQL equivalents are delegated to the -unsupported-constraint policy. - -## Stage 4 -- Intermediate Representation - -Each constraint becomes a normalized object. - -Example: - -``` text -Constraint( - field="age", - operator="ge", - value=18, -) +```text +RuleSchema declaration + │ class-time validation and metadata normalization + ▼ +Immutable SchemaSpec / RuleField values + │ bind field names to SQLAlchemy expressions + ▼ +Backend PreparedValue(source, value, valid, is_null) + │ translate constraints over the prepared value + ▼ +Field predicates + │ AND + SQL NULL normalization + ▼ +CompiledRules.predicate ``` -The IR isolates compiler logic from both Pydantic and SQLAlchemy APIs. - -## Stage 5 -- Translators +`RuleSchema` subclasses retain Pydantic construction and validation behavior. +Unsupported declarations fail when the class is created. Full Pydantic models +are accepted through `from_pydantic()`, which returns a generated rule model +and a conversion report. -Each constraint is translated independently. +## Schema representation -Examples: +`SchemaSpec` and `RuleField` contain table-independent information: field +order, annotation, logical type, nullability, strictness, constraints, +defaults, descriptive metadata, SQL column name, and conversion provenance. +They contain no table-bound SQL expressions. Every declared field must bind; +an annotation alone is a rule. -- ge → column \>= value -- gt → column \> value -- min_length → func.length(column) \>= value -- Literal → column.in\_(...) +The normalizer reads public Pydantic field information and an explicit metadata +allowlist. It does not execute validators, serializers, or default factories. +The compiler normalizes at each compile call so rule compilation does not rely +on mutable shared diagnostics or cached SQL expressions. -Every translator returns one SQLAlchemy expression. +## Source preparation and predicate grouping -## Stage 6 -- Rule Assembly +Each backend provider returns a `PreparedValue` containing the original +column, a safe normalized expression, a type/coercion validity predicate, a +SQL NULL test, and capability details. Bounds, lengths, patterns, and domains +consume the normalized expression. -Expressions are grouped by field. +For a required field: -Example: - -``` python -{ - "age": [ - users.c.age >= 18, - users.c.age <= 65, - ] -} +```text +column IS NOT NULL AND valid AND every constraint(value) ``` -Ordering is deterministic and follows the source model. - -## Error Handling - -Modes: - -- raise (default) -- warn -- ignore +For a nullable field, SQL NULL is an allowed branch; a failed conversion that +produces NULL remains distinguishable from an original SQL NULL. The compiler +normalizes the result to SQL TRUE/FALSE, then ANDs all fields into +`CompiledRules.predicate`. `notwhere()` complements that total root. -Errors include: +The backend must make conversions safe regardless of SQL predicate evaluation +order. Unsupported representations raise `CapabilityError` during compile; +invalid row values become false predicates. -- UnsupportedConstraintError -- MissingColumnError -- InvalidModelError -- TranslatorError +## Translators and providers -## Caching +Portable translators handle scalar operators such as `ge`, `multiple_of`, and +`min_length`. Plugins add backend operators such as JSON containment, arrays, +ranges, regex, or full text. Backend providers implement source preparation +and capability reporting in addition to registering translators. -Phase-1 model IR is cached in-process by model class (default on). -Cached values are immutable field descriptors and constraints — never -SQLAlchemy column objects. Disable with `cache=False`. +`Compiler(plugins=[...])` requires exactly one backend provider. Its translator +registry is a private snapshot; compile calls keep their diagnostics local. +The `dialect=` option is only a compatibility consistency check and never +selects or discovers a backend. -Translation remains deterministic. +## Result API -## Extension Points +`CompiledRules` contains the full predicate, per-field results, diagnostics, +backend version, assumptions, and conversion provenance. Its `explain()` method +returns a structured compile plan without running database `EXPLAIN`. -Available today: +`where()` and `flatten()` return `[compiled.predicate]` to preserve the +spread-style SQLAlchemy call. `notwhere()` returns its total complement. -- Inject a custom `TranslatorRegistry` via `Compiler(registry=...)` -- Register versioned plugins via `Compiler(plugins=[...])` -- Override operators with `register_constraint(..., on_conflict=...)` -- Optional `dialect=` hint on `CompilationContext` +## Errors -Reserved for later: +- `InvalidModelError`: an unrestricted Pydantic model was passed directly. +- `UnsupportedConstraintError`: a declaration has no accepted SQL meaning. +- `MissingColumnError`: a declared rule field cannot be bound. +- `CapabilityError`: the selected provider cannot safely implement a retained + type conversion or operator. +- `PluginError` / `RegistryError`: a provider or translator contract is invalid. -- Compiler pass plugins -- Entry-point / auto-discovery - -## Design Principles - -- Pure function compiler -- No database dependency -- Deterministic output -- SQLAlchemy-first -- Easy to test -- Easy to extend -- Fail fast on unsupported semantics - -## Public Contract - -``` python -rules = sqlrules.compile(UserFilter, users) -``` - -Returns: - -``` python -dict[str, list] -``` +## Build extension -where each value is a list of SQLAlchemy boolean expressions suitable -for passing directly to `.where(*expressions)`. +Custom operators register a translator accepting `(constraint, prepared_value, +context)` and returning a SQLAlchemy boolean expression. Backend providers may +implement `prepare_value()` and `capabilities()` to establish source type +evidence and safe conversions. See [PLUGIN_SYSTEM](PLUGIN_SYSTEM.md). diff --git a/docs/CONSTRAINTS.md b/docs/CONSTRAINTS.md index 1eceb44..5b6fff0 100644 --- a/docs/CONSTRAINTS.md +++ b/docs/CONSTRAINTS.md @@ -1,60 +1,71 @@ -# Supported Constraint Mapping +# Constraint Reference -| Constraint | SQLAlchemy expression | +Every supported scalar annotation creates a type rule. Supported Pydantic +constraints add value rules that run against the backend-prepared expression. +The full source-type matrix is in [TYPE_SUPPORT](TYPE_SUPPORT.md). + +## Portable operators + +| Declaration | Rule | |---|---| -| `gt` | `column > value` | -| `ge` | `column >= value` | -| `lt` | `column < value` | -| `le` | `column <= value` | -| `min_length` | `func.length(column) >= value` | -| `max_length` | `func.length(column) <= value` | -| `multiple_of` | `column % value == 0` | -| `Literal[...]` | `column.in_(...)` | -| `Enum` | `column.in_(member values)` | -| `pattern` | IR (`PatternSpec`); translators via plugins / custom registry | -| `type_check` | IR (`TypeSpec`) when `emit_type_checks=True`; translators via plugins | - -## Dialect markers (`sqlrules.markers`) - -| Marker | IR operator | Typical plugin translation | -|---|---|---| -| `JsonContains` | `json_contains` | JSONB / JSON containment | -| `JsonHasKey` | `json_has_key` | JSON key existence | -| `ArrayContains` | `array_contains` | PostgreSQL array containment | -| `ArrayOverlap` | `array_overlap` | PostgreSQL array overlap | -| `RangeContains` | `range_contains` | PostgreSQL range `@>` | -| `RangeOverlap` | `range_overlap` | PostgreSQL range `&&` | -| `FullTextMatch` | `fulltext_match` | MySQL `MATCH ... AGAINST` | - -Accepted input forms for portable constraints include `Field(...)`, -`annotated_types` primitives, `Interval`, `Len`, `conint`/`constr`, and -`StringConstraints` (length attributes; `pattern` becomes `PatternSpec` IR). - -Constraints without a deterministic SQL equivalent are rejected by default. -See [TYPE_SUPPORT.md](TYPE_SUPPORT.md) and [ERRORS.md](ERRORS.md). - -To translate `pattern`, `type_check`, or markers, use a dialect plugin: - -```python -from sqlrules import Compiler, JsonContains -from sqlrules_postgresql import PostgresPlugin - -compiler = Compiler( - plugins=[PostgresPlugin()], - dialect="postgresql", - emit_type_checks=True, -) -``` - -```python -from sqlrules.constraints import pattern_text -from sqlrules.translators import default_registry - -registry = default_registry() -registry.register_constraint( - "pattern", - lambda c, col, ctx: col.op("~")(pattern_text(c.value)[0]), - on_conflict="replace", -) -compiler = sqlrules.Compiler(registry=registry) -``` +| gt / ge / lt / le | Comparison after type preparation | +| multiple_of | Modulo equality where the backend has safe numeric modulo semantics | +| min_length / max_length | Backend length comparison | +| Literal[...] | Membership in the homogeneous literal domain | +| Enum | Membership in the enum value domain | +| pattern | Pattern match when the selected plugin implements it | + +Portable translators are registered by core for comparisons, multiple_of, +length, and domain membership. Pattern requires a backend translator. The +built-in `multiple_of` translator raises `CapabilityError` for floating-point +fields, float divisors, and SQLite non-integral `Decimal` divisors. A plugin +can replace that translator with an implementation suited to its backend. + +~~~python +from typing import Annotated +from pydantic import Field, StringConstraints +from sqlrules import RuleSchema + + +class UserRules(RuleSchema): + age: int = Field(ge=18, le=120) + name: Annotated[str, StringConstraints(min_length=2, max_length=80)] +~~~ + +Constraints are checked during class creation. Duplicate operators, +contradictory bounds, invalid divisors, negative lengths, and unsupported +field/constraint combinations fail before compilation. + +## Strictness and nullability + +Use Field(strict=True), a Pydantic strict alias, or model_config strict=True to +require an exact observable database logical type. The default is lax mode; +its supported conversions are listed per backend in TYPE_SUPPORT. + +An annotation such as int | None accepts SQL NULL. The nullable branch wraps +both type validity and constraints. Non-nullable fields reject SQL NULL even +when they have a default in Python. + +## Dialect markers + +Import SQL-only markers from sqlrules.markers or sqlrules: + +- JsonContains / JsonHasKey +- ArrayContains / ArrayOverlap +- RangeContains / RangeOverlap +- FullTextMatch + +Markers need one backend plugin that supports the source SQL type and +operator. A marker does not add nested item validation or validate a Pydantic +collection value against every database element. + +## Result helpers + +~~~python +compiled = compiler.compile(UserRules, users) +matches = users.select().where(*sqlrules.where(compiled)) +fails = users.select().where(*sqlrules.notwhere(compiled)) +~~~ + +where() and flatten() return one complete predicate. notwhere() returns its +total complement. See [IR_CONTRACT](IR_CONTRACT.md) for the result shape. diff --git a/docs/DESIGN_DECISIONS.md b/docs/DESIGN_DECISIONS.md index 8200177..2915466 100644 --- a/docs/DESIGN_DECISIONS.md +++ b/docs/DESIGN_DECISIONS.md @@ -1,474 +1,83 @@ # SQLRules Design Decisions -## Purpose +SQLRules 2.0 compiles an intentionally limited subset of Pydantic model +declarations into SQL predicates. These decisions define the semantic contract +for the 2.x series. -This document records the major design decisions behind SQLRules so -future development stays consistent, focused, and easy to reason about. +## 1. Rule schemas remain Pydantic models -SQLRules exists to do one thing well: +`RuleSchema` is a `pydantic.BaseModel` subclass. Instances keep normal Pydantic +validation, serialization, JSON Schema, and framework integration behavior. +SQLRules limits the declarations inside a RuleSchema to the types, constraints, +and compatible metadata that it can compile; it does not restrict how callers +instantiate or use the model. -> Compile constrained Pydantic models into SQLAlchemy WHERE-rule -> dictionaries. +Compatible Pydantic imports work directly in class declarations. SQLRules adds +`Field(column=...)` and `RuleConfig` for database binding and SQL-specific +options. Unrestricted Pydantic models can be passed through `from_pydantic()`; +conversion returns another Pydantic model and a report for removed semantics. ------------------------------------------------------------------------- +## 2. Type annotations always create row rules -## Decision 1: SQLRules is a Compiler, Not a Query Builder +Every scalar annotation contributes a predicate, even without an explicit +constraint. Lax mode is the default and follows the documented SQLRules +coercion profile. `strict=True`, `ConfigDict(strict=True)`, and compatible +strict metadata require the observed database value to have the declared +logical type. Strictness and supported constraints are inspectable on the +compiled plan. -## Decision +The database cannot recover the Python object originally supplied to an ORM. +SQLRules therefore defines types in terms of database storage evidence and +documents backend differences, including representations it cannot prove. -SQLRules compiles model metadata into SQLAlchemy boolean expressions. +## 3. Prepare values safely before applying constraints -It does not build complete queries. +An explicit backend provider prepares each source column into a logical value, +validity predicate, NULL state, and capability description. Constraints run +against the prepared value. Invalid coercions are non-matches. Conversion +safety cannot depend on SQL evaluation order or short-circuiting. -## Rationale +If a provider cannot prove that a mapping or conversion is safe, compilation +raises `CapabilityError`. A known row-level type mismatch is a false predicate +instead. The capability matrix is part of the public semantic contract. -SQLAlchemy already provides excellent query composition. SQLRules should -produce reusable pieces that fit naturally into SQLAlchemy Core and ORM -workflows. +## 4. Every retained declaration must be enforced -## Consequences +Unsupported declarations on a RuleSchema raise during class construction or +compilation. Unsupported retained constraints never produce warnings or get +silently dropped. `from_pydantic()` is the explicit conversion boundary for +dropping incompatible behavior, and its report records what changed. -SQLRules returns expressions such as: +## 5. Compile predicates, not queries -``` python -users.c.age >= 18 -``` +SQLRules creates SQLAlchemy boolean expressions and never opens a connection, +inspects live database metadata, renders SQL strings, or executes statements. +Applications compose the result into Core or ORM queries. -not full statements such as: +`CompiledRules` contains a total root predicate, per-field results, +diagnostics, backend assumptions, and conversion provenance. `where()` returns +the root predicate in a list for spread-style SQLAlchemy calls. `notwhere()` +returns its complement. SQL NULL, invalid conversions, and failed constraints +are partitioned by those two helpers. -``` python -select(users).where(users.c.age >= 18) -``` +## 6. Backend selection is explicit ------------------------------------------------------------------------- +Exactly one backend provider prepares source values for a bind. Constraint +plugins may add translators, but cannot silently substitute source coercion +semantics. Server versions and collation or storage assumptions are explicit +and appear in capabilities or the compiled explain plan. -## Decision 2: Return a Dictionary Grouped by Field +## 7. Reuse normalized declarations without global model caches -## Decision +`Compiler.compile_model()` and `bind()` expose normalization and table binding +as separate operations. Normalized schema metadata is immutable. SQLRules does +not keep process-wide model-class cache entries; table-bound predicates are +created for the supplied columns. Legacy cache arguments remain inert +compatibility shims in 2.0. -The primary output format is: +## 8. Version semantics, not just syntax -``` python -dict[str, list[ColumnElement[bool]]] -``` - -Example: - -``` python -{ - "age": [ - users.c.age >= 18, - users.c.age <= 65, - ], - "name": [ - func.length(users.c.name) >= 2, - ], -} -``` - -## Rationale - -A dictionary preserves field-level meaning and lets users compose rules -selectively. - -## Consequences - -Users can apply all rules: - -``` python -stmt = select(users).where(*sqlrules.where(rules)) -``` - -or only some rules: - -``` python -stmt = select(users).where(*rules["age"]) -``` - ------------------------------------------------------------------------- - -## Decision 3: SQLAlchemy Expressions, Not SQL Strings - -## Decision - -SQLRules returns SQLAlchemy expression objects. - -## Rationale - -SQL strings are dialect-sensitive, harder to compose, and easier to -misuse. SQLAlchemy expressions are safer, composable, and backend-aware. - -## Consequences - -SQLRules never renders SQL strings directly. - ------------------------------------------------------------------------- - -## Decision 4: Pydantic v2 First - -## Decision - -SQLRules targets Pydantic v2 first. - -## Rationale - -Pydantic v2 has a modern metadata model based heavily on `Annotated` and -constraint metadata. Supporting v1 would add complexity before the -package proves its core value. - -## Consequences - -Initial compatibility target: - -- Python 3.10+ -- Pydantic v2 -- SQLAlchemy 2.x - -Pydantic v1 support may be considered later as a compatibility plugin or -separate adapter. - ------------------------------------------------------------------------- - -## Decision 5: Fail Fast by Default - -## Decision - -Unsupported constraints raise errors by default. - -## Rationale - -Silently ignoring constraints could produce overly broad SQL filters and -create correctness or security issues. - -## Consequences - -Default mode: - -``` python -on_unsupported="raise" -``` - -Optional modes: - -- warn -- ignore - ------------------------------------------------------------------------- - -## Decision 6: Only Deterministic Constraints Are Supported - -## Decision - -SQLRules only supports constraints that have deterministic SQL -equivalents. - -## Supported Examples - -- gt -- ge -- lt -- le -- min_length -- max_length -- multiple_of -- Literal -- Enum - -## Unsupported Examples - -- custom validators -- model validators -- computed fields -- arbitrary Python predicates - -## Rationale - -SQLRules should never pretend that arbitrary Python validation logic can -be safely converted to SQL. - ------------------------------------------------------------------------- - -## Decision 7: Optionality Does Not Generate Rules by Default - -## Decision - -`Optional[T]` does not automatically produce `IS NULL` or `IS NOT NULL` -expressions. - -## Rationale - -Optionality describes whether a value may be absent or null during -validation. It does not always imply a SQL filtering rule. - -## Consequences - -Future configuration may allow explicit nullability policies, but the -default behavior remains no rule. - ------------------------------------------------------------------------- - -## Decision 8: Keep the Intermediate Representation Dialect-Neutral - -## Decision - -The IR describes semantic intent, not database-specific syntax. - -Example: - -``` python -Constraint(field="name", operator="min_length", value=2) -``` - -## Rationale - -Dialect-specific behavior belongs in translators, not in the compiler -core. - -## Consequences - -The same IR can be translated differently by SQLite, PostgreSQL, MySQL, -SQL Server, or Oracle plugins. - ------------------------------------------------------------------------- - -## Decision 9: Plugins Are Explicit - -## Decision - -SQLRules 0.3 ships an explicit plugin API: - -```python -Compiler(plugins=[PostgresPlugin()], on_conflict="raise", dialect="postgresql") -``` - -There is no automatic discovery. Plugins declare `api_version` matching -`PLUGIN_API_VERSION`. - -## Rationale - -Explicit plugin registration avoids hidden behavior, improves -reproducibility, and keeps the compiler deterministic. - -## Consequences - -See [PLUGIN_SYSTEM.md](PLUGIN_SYSTEM.md). Official starter packages: -`sqlrules-postgresql` and `sqlrules-sqlite`. - ------------------------------------------------------------------------- - -## Decision 10: No Database Connections - -## Decision - -SQLRules never opens database connections or reflects database metadata. - -## Rationale - -The package should be safe to use in any environment, including -application startup, tests, CI, and code generation tools. - -## Consequences - -Users provide SQLAlchemy tables, ORM classes, aliases, or explicit -column maps. - ------------------------------------------------------------------------- - -## Decision 11: Small Public API - -## Decision - -The public API should remain minimal. - -Initial API: - -``` python -sqlrules.compile(...) -sqlrules.where(...) -sqlrules.flatten(...) -``` - -## Rationale - -A small API is easier to document, test, stabilize, and maintain. - -## Consequences - -Advanced behavior should live behind compiler options, internal -components, or plugins rather than expanding the top-level API too -early. - ------------------------------------------------------------------------- - -## Decision 12: Two-Phase Compilation - -## Decision - -SQLRules separates static model compilation from table binding. - -## Phase 1: Static Compilation - -- inspect Pydantic model -- extract constraints -- build `ModelIR` (cached by default) - -## Phase 2: Binding - -- resolve SQLAlchemy columns -- translate IR to expressions - -## Rationale - -Static model work can be cached and reused across multiple tables, -aliases, ORM models, or column maps. - -## Consequences - -`Compiler.compile_model` / `Compiler.bind` expose the phases. -`sqlrules.compile(...)` remains the one-shot public API. - ------------------------------------------------------------------------- - -## Decision 13: Prefer Simple Translators - -## Decision - -Basic constraints should use simple function-based translators where -possible. - -Example: - -``` python -"gt": operator.gt -"ge": operator.ge -``` - -## Rationale - -Many translations are direct operator mappings and do not need complex -class hierarchies. - -## Consequences - -Dedicated translator classes should be reserved for more complex -constraints such as: - -- min_length -- max_length -- pattern -- Literal -- Enum -- dialect-specific behavior - ------------------------------------------------------------------------- - -## Decision 14: Stable Errors Matter - -## Decision - -SQLRules exceptions should be structured and stable. - -## Rationale - -Clear errors make the package easier to use, test, and integrate into -developer tools. - -## Consequences - -All public exceptions inherit from: - -``` python -SQLRulesError -``` - -Exceptions should include structured context such as: - -- field -- constraint -- value -- translator -- suggested fix - ------------------------------------------------------------------------- - -## Decision 15: Documentation Drives Implementation - -## Decision - -SQLRules planning documents are treated as implementation guidance. - -## Rationale - -The package is intentionally small, so high-quality design docs can -prevent scope creep and architectural drift. - -## Consequences - -New features should update relevant docs before implementation. - ------------------------------------------------------------------------- - -## Open Questions - -## Should SQLRules Support Pydantic v1? - -Likely not in core initially. - -Possible future path: - -- `sqlrules-pydantic-v1` -- adapter layer -- compatibility mode - ------------------------------------------------------------------------- - -## Should SQLRules Include a CompiledRules Object? - -Potential API: - -``` python -compiled = sqlrules.compile(UserFilter, users) - -compiled.rules -compiled.where() -compiled.field("age") -``` - -This may improve ergonomics while preserving dictionary output. - ------------------------------------------------------------------------- - -## Should Regex Be Core or Dialect Plugin? - -**Decided for 0.2 / 0.3:** - -- IR support in core (`Constraint(operator="pattern", ...)`) -- No portable core translator -- Translator support via `TranslatorRegistry` and dialect plugins - (`sqlrules-postgresql`, `sqlrules-sqlite`) - ------------------------------------------------------------------------- - -## Should Nullability Rules Be Optional? - -Potential future configuration: - -``` python -null_policy="ignore" # default -null_policy="not_null" -null_policy="is_null" -``` - -This should not be part of the initial MVP unless a strong use case -emerges. - ------------------------------------------------------------------------- - -## Summary - -SQLRules should stay small, explicit, deterministic, and composable. - -The most important design boundary is this: - -> SQLRules converts supported Pydantic constraints into SQLAlchemy WHERE -> expressions. Nothing more. +The supported Pydantic declaration subset, coercion rules, backend evidence, +NULL behavior, and unsupported cases are versioned. New 2.x phases can add +features, but must preserve the acceptance behavior of existing schemas unless +a deliberate compatibility change is documented. diff --git a/docs/DIALECT_SUPPORT.md b/docs/DIALECT_SUPPORT.md index e9ce030..0338c74 100644 --- a/docs/DIALECT_SUPPORT.md +++ b/docs/DIALECT_SUPPORT.md @@ -1,208 +1,68 @@ -# SQLRules Dialect Support +# SQLRules 2.0 Dialect Support -## Purpose +SQLRules requires the caller to select one backend provider. A dialect name +alone never changes compilation behavior, and the compiler does not inspect a +live connection. -SQLRules compiles Pydantic constraints into SQLAlchemy expressions. -SQLAlchemy already abstracts many database differences, but some -operators and functions have dialect-specific behavior. This document -defines how SQLRules handles those differences while keeping the -compiler deterministic. +## Official providers ------------------------------------------------------------------------- +| Package | Provider | Backend-specific rules | +|---|---|---| +| sqlrules-postgresql | PostgresPlugin | Regex, JSONB, ARRAY, range, and safe text parsing on PostgreSQL 16+ | +| sqlrules-sqlite | SQLitePlugin | Runtime storage-class checks, REGEXP, Python-compatible string length, JSON helpers | +| sqlrules-mysql | MysqlPlugin | REGEXP, JSON, full-text, and safe integer text conversion on MySQL 8.0+ | +| sqlrules-mssql | MssqlPlugin | JSON at SQL Server 2016+ / compatibility level 130+, SQL Server LEN behavior, and TRY_CAST conversions on SQL Server 2012+ | -## Philosophy +The MySQL provider targets MySQL 8.0+. MariaDB is outside the supported server +matrix because its regular-expression function API differs from MySQL's. -1. Prefer portable SQLAlchemy constructs. -2. Keep the core compiler dialect-neutral. -3. Isolate database-specific behavior behind dialect extensions. -4. Never silently change semantics based on the active database. +Select exactly one backend provider, optionally with constraint plugins: ------------------------------------------------------------------------- +~~~python +from sqlrules import Compiler +from sqlrules_postgresql import PostgresPlugin -## Support Levels +compiler = Compiler(plugins=[PostgresPlugin(server_version=(16, 0))]) +compiled = compiler.compile(UserRules, users) +~~~ - Level Meaning - -------------- ------------------------------------------------- - Core Uses only portable SQLAlchemy expressions. - Enhanced Uses optional dialect-specific translators. - Experimental Feature exists but semantics differ by backend. +Each provider separates source preparation from constraint translation. It +reports capabilities and produces a PreparedValue containing the original +source, normalized SQL expression, null state, validity predicate, and +coercion description. Constraint translators operate on the prepared +expression. The compiler combines the complete field rules under one total +root predicate. ------------------------------------------------------------------------- +## Capability rules -## Dialects +- A known row mismatch becomes a non-match. +- A source representation the provider cannot safely inspect or convert raises + CapabilityError during compilation. +- Retained rules are never skipped by warn or ignore policies. +- The provider does not emit casts that can throw for malformed row data. +- Server versions, database compatibility levels, and storage assumptions are + visible in capabilities() and CompiledRules.explain(). -Core emits portable SQLAlchemy Core expressions. The table below describes -**intended** dialect posture, not a certified multi-backend test matrix. +See [TYPE_SUPPORT](TYPE_SUPPORT.md) for the type, conversion, collation, null, +and version matrix. - Dialect Status Notes - ------------ -------------------- -------------------------------- - SQLite Core + plugin `sqlrules-sqlite` — REGEXP + JSON - PostgreSQL Core + plugin `sqlrules-postgresql` — regex, JSONB, ARRAY, range - MySQL Core + plugin `sqlrules-mysql` — REGEXP, JSON, full-text - MariaDB Core + plugin Follows MySQL plugin where practical - SQL Server Core + plugin `sqlrules-mssql` — JSON + `LEN` - Oracle Planned Core support after 1.0 +## Markers ------------------------------------------------------------------------- +Dialect markers are declared with sqlrules.markers. The official plugins keep +the stable operators json_contains, json_has_key, array_contains, +array_overlap, range_contains, range_overlap, and fulltext_match where the +backend has an applicable type. -## Core Portable Constraints +Container item validation and nested JSON schemas are not part of 2.0. +RuleSchema accepts list/dict annotations only for supported marker-driven +fields. A backend still requires an actual JSON, array, or range source type. -These should compile identically across supported databases: +## Plugin API v2 -- gt -- ge -- lt -- le -- multiple_of -- min_length -- max_length -- Literal -- Enum +Custom constraint plugins implement SQLRulesPlugin with name, api_version, +and register(registry). Backend providers also implement prepare_value() and +capabilities(). The exact version string is exported as PLUGIN_API_VERSION. +Plugins written for API v1 must be adapted because they do not expose source +preparation or backend capabilities. ------------------------------------------------------------------------- - -## Shared dialect markers - -Dialect-oriented operators are expressed with `sqlrules.markers` and -extracted into IR. Core does not translate them; plugins register -translators for the stable operator names: - -- `JsonContains` → `json_contains` -- `JsonHasKey` → `json_has_key` -- `ArrayContains` → `array_contains` -- `ArrayOverlap` → `array_overlap` -- `RangeContains` → `range_contains` -- `RangeOverlap` → `range_overlap` -- `FullTextMatch` → `fulltext_match` - -`list` / `dict` field annotations are allowed for marker-driven fields. -Portable constraints on containers still raise. - ------------------------------------------------------------------------- - -## PostgreSQL - -Package: `sqlrules-postgresql` - -- `pattern` → `~` or `~*` (`PatternSpec.ignore_case`) -- JSONB containment / key checks -- ARRAY contains / overlap -- range `@>` / `&&` - ------------------------------------------------------------------------- - -## SQLite - -Package: `sqlrules-sqlite` - -- `pattern` via `REGEXP` — call `register_regexp(connection)` -- Case-insensitive patterns use a `(?i)` prefix understood by the helper -- JSON helpers via JSON1 `json_extract` / `json_type` - ------------------------------------------------------------------------- - -## MySQL / MariaDB - -Package: `sqlrules-mysql` - -- `pattern` → `REGEXP` -- JSON operators (`JSON_CONTAINS`, `JSON_CONTAINS_PATH`) -- `fulltext_match` → `MATCH ... AGAINST` (requires FULLTEXT index) - ------------------------------------------------------------------------- - -## SQL Server - -Package: `sqlrules-mssql` - -- `min_length` / `max_length` → `LEN` -- JSON helpers via `JSON_VALUE` / `JSON_QUERY` -- `pattern` is not registered (no deterministic portable regex) - ------------------------------------------------------------------------- - -## Oracle - -Long-term goals: - -- Oracle-specific translators -- Optimized string semantics -- Date/time enhancements - ------------------------------------------------------------------------- - -## Translator Selection - -The compiler dispatches through a registry. - -``` text -Constraint IR - │ - ▼ -Core Translator - │ - ├── Portable implementation - │ - └── Dialect override (optional) -``` - -If no override exists, the portable translator is used. - ------------------------------------------------------------------------- - -## Plugin Packages - -Official plugins: - -- sqlrules-postgresql -- sqlrules-sqlite -- sqlrules-mysql -- sqlrules-mssql - -Each package registers only the translators relevant to its backend. - ------------------------------------------------------------------------- - -## Testing Strategy - -Every supported dialect should include: - -- Unit tests -- SQL compilation tests -- Regression tests - -The same Pydantic model should produce equivalent logical behavior -across supported databases whenever a portable translation exists. - ------------------------------------------------------------------------- - -## Compatibility Policy - -Minor SQLRules releases may add dialect features. - -Major releases may change plugin APIs but should preserve the behavior -of portable translators whenever possible. - ------------------------------------------------------------------------- - -## Non-Goals - -SQLRules will not: - -- Open database connections -- Detect the active database automatically -- Rewrite SQL generated by SQLAlchemy -- Emulate unsupported database features - -Applications remain responsible for selecting the correct dialect -plugin. - ------------------------------------------------------------------------- - -## Design Principles - -- Portable by default -- Explicit dialect extensions -- Stable compiler behavior -- No hidden database assumptions -- Maximum SQLAlchemy compatibility +See [PLUGIN_SYSTEM](PLUGIN_SYSTEM.md) for the registry and translator contract. diff --git a/docs/ERRORS.md b/docs/ERRORS.md index c69d147..214d2e4 100644 --- a/docs/ERRORS.md +++ b/docs/ERRORS.md @@ -1,221 +1,79 @@ -# SQLRules Error Handling +# Errors and Diagnostics -## Philosophy +SQLRules separates invalid model declarations, missing columns, unsupported +rules, and backend capability gaps. Compilation never skips a retained rule. -SQLRules follows a fail-fast approach. If a Pydantic constraint cannot -be translated into a deterministic SQLAlchemy expression, the compiler -should report the problem immediately rather than silently producing -incomplete rules. +## Public error hierarchy -Goals: - -- Clear exception hierarchy -- Helpful error messages -- Deterministic behavior -- Configurable handling for unsupported **constraint operators** -- Structured diagnostics for skipped constraints - ------------------------------------------------------------------------- - -## Exception Hierarchy - -``` text +~~~text SQLRulesError ├── InvalidModelError ├── MissingColumnError ├── UnsupportedConstraintError +├── CapabilityError ├── TranslatorError ├── InvalidTranslatorError ├── RegistryError ├── ConfigurationError -├── PluginError -└── InternalCompilerError (reserved) -``` +└── PluginError +~~~ -All public exceptions inherit from `SQLRulesError`. +## RuleSchema declaration errors ------------------------------------------------------------------------- +RuleSchema validates supported declarations when the class is created. +Unsupported scalar/container types, custom validators, serializers, computed +fields, string normalization, arbitrary metadata, duplicate constraints, and +incompatible constraint/type combinations raise TypeError with the original +SQLRules error chained as the cause. ## InvalidModelError -Raised when the input is not a supported Pydantic model. - -Examples: - -- Not a BaseModel subclass - ------------------------------------------------------------------------- +Compiler.compile accepts a RuleSchema class, not an unrestricted Pydantic +BaseModel or an instance. Use from_pydantic() to convert a full model and +inspect its report. ## MissingColumnError -Raised when a constrained model field cannot be matched to a SQLAlchemy -column. Unconstrained fields are skipped and do not trigger this error. - -Non-column table attributes (for example `Table.name`) are never treated -as columns. - -Example message: - - No SQLAlchemy column found for field 'age'. Provide a matching table - column, ORM attribute, or column_map entry. - ------------------------------------------------------------------------- +Every RuleSchema field must bind to a database column, including fields with +type-only annotations. Binding order is an explicit column_map entry, then +Field(column=...), then the Python field name. Pydantic aliases do not select +columns implicitly. ## UnsupportedConstraintError -Raised when SQLRules encounters a constraint operator with no translator, -an unsupported type, or an invalid operator/type combination. - -Examples: - -- `pattern` (no core translator), `max_digits`, `decimal_places` -- custom validators / `Strict` -- unsupported types (containers, `timedelta`) -- `multiple_of <= 0` -- length constraints on non-`str` fields - -Example: - - Field 'name': constraint 'pattern' is not supported by SQLRules. - Remove the constraint, or set on_unsupported='warn'/'ignore'. - ------------------------------------------------------------------------- - -## TranslatorError - -Raised when a registered translator fails while generating a SQLAlchemy -expression (unexpected SQLAlchemy errors wrapped by the registry). - ------------------------------------------------------------------------- - -## InvalidTranslatorError - -Raised when registering a translator that is not callable or does not -accept at least three positional parameters (`constraint`, `column`, -`context`). Variadic `*args` translators are accepted. - ------------------------------------------------------------------------- - -## RegistryError - -Raised for translator registry failures. - -Examples: - -- Duplicate registrations when `on_conflict="raise"` (default) -- Legacy `replace=True` on `register()` is equivalent to `on_conflict="replace"` - on `register_constraint()` — prefer `on_conflict` in new code - -Missing operators raise `UnsupportedConstraintError`, not `RegistryError`. - ------------------------------------------------------------------------- - -## ConfigurationError - -Raised when compiler configuration is inconsistent. - -Raised for an invalid `on_unsupported` or `on_conflict` mode. - ------------------------------------------------------------------------- - -## PluginError - -Raised when a plugin fails validation (missing `name` / `register`, or -`api_version` mismatch with `PLUGIN_API_VERSION`). - ------------------------------------------------------------------------- - -## InternalCompilerError - -Reserved for unexpected internal failures. Not raised on the normal -1.0 Application compile path. - ------------------------------------------------------------------------- - -## Compiler Policies - -The compiler supports three behaviors for unsupported **constraint -operators**: - -## raise (default) - -Immediately raises `UnsupportedConstraintError`. - -## warn - -Emits a `SQLRulesWarning`, records a `Diagnostic`, and skips the -constraint. - -## ignore - -Records a `Diagnostic` and silently skips unsupported constraints. - -Unsupported **types** always raise, regardless of `on_unsupported`. - ------------------------------------------------------------------------- - -## Structured Diagnostics - -After `compile` / `bind` / `compile_model`, inspect skipped constraints: - -```python -compiler = sqlrules.Compiler(on_unsupported="warn") -rules = compiler.compile(Model, table) -for diag in compiler.diagnostics: - print(diag.code, diag.severity, diag.field, diag.operator, diag.message) -``` - -`Diagnostic` fields: `severity` (`"warning"` | `"info"`), `field`, -`operator`, `value`, `message`, `code`. - -Stable diagnostic codes (1.0): - -| Code | When | -|---|---| -| `unsupported_constraint` | Operator skipped under `warn` / `ignore` | - -`compile_model` clears diagnostics. Diagnostics are separate from -exceptions and do not change the rules dict return type. - -Do not call `compile` / `bind` concurrently on the same `Compiler` -instance; diagnostics collection is not locked. - ------------------------------------------------------------------------- - -## Error Message Guidelines - -Every public exception should include: - -- field name (when applicable) -- constraint/operator -- offending value (when applicable) -- suggested resolution - ------------------------------------------------------------------------- +The core or selected plugin has no translator for a retained constraint. +Unlike SQLRules 1.x, warn and ignore policies cannot remove a rule from the +compiled predicate. -## Logging +## CapabilityError -SQLRules does not log by default. +The backend cannot prove the requested source type, storage representation, +server feature, or safe conversion. This is different from a known row +mismatch: -Applications decide how to handle exceptions, warnings, and diagnostics. +- Known mismatched value/type: the SQL predicate evaluates to FALSE. +- Missing implementation capability: compilation raises CapabilityError. +- SQL NULL: matches only when the RuleSchema field is nullable. ------------------------------------------------------------------------- +The error includes backend, field, target type, source type, and reason. Use +the [type support matrix](TYPE_SUPPORT.md) to select supported mappings. -## Testing +## TranslatorError and InvalidTranslatorError -Each actively raised exception should have tests covering: +InvalidTranslatorError is raised when a registry translator is not callable +or cannot accept the translator signature. TranslatorError wraps an exception +raised while building a SQLAlchemy expression or a non-expression return +value. -- expected trigger -- message contents -- inheritance from SQLRulesError -- policy interactions (where applicable) +## ConfigurationError and PluginError ------------------------------------------------------------------------- +ConfigurationError covers invalid conflict policies, unsupported skipped-rule +policies, and dialect assertions that disagree with the selected provider. +PluginError covers missing or incompatible API v2 declarations, multiple +backend providers, or missing backend hooks. -## Design Principles +## Diagnostics -- Fail fast -- Never silently change semantics -- Preserve deterministic compilation -- Prefer explicit errors over implicit behavior -- Keep exceptions stable across minor releases +CompiledRules.diagnostics contains compile-scoped structured information. +CompiledRules.explain() includes bindings, conversions, assumptions, and field +predicates. SQLRules does not retain diagnostics across calls on a Compiler. diff --git a/docs/INTERNAL_API.md b/docs/INTERNAL_API.md index 3cee6f4..10eef39 100644 --- a/docs/INTERNAL_API.md +++ b/docs/INTERNAL_API.md @@ -1,91 +1,68 @@ # SQLRules Internal API -## Purpose - -This document describes **implementation modules** used inside SQLRules. -They are **not** part of the Application or Plugin semver contracts -unless re-exported in [API.md](API.md). - -Plugin authors should import from `sqlrules` (`TranslatorRegistry`, -`pattern_text`, `Constraint`, `PatternSpec`, …), not from private -helpers. - ------------------------------------------------------------------------- - -## Pipeline (actual modules) - -``` text -compile() / Compiler - │ - ├── inspectors.inspect_model → FieldDescriptor[] - ├── constraints.extract_constraints / ensure_supported_type - ├── cache.ModelIRCache → ModelIR (optional) - ├── columns.resolve_column - ├── translators.TranslatorRegistry.translate - └── assemble dict[str, list[ColumnElement[bool]]] +This document describes implementation modules. They are not covered by the +Application or Plugin API compatibility contracts unless explicitly re-exported +from `sqlrules`. + +## Compilation pipeline + +```text +RuleSchema class + └── models.normalize_schema → immutable SchemaSpec + └── Compiler.bind → resolve columns and select provider + ├── backend provider → PreparedValue with safe type evidence + ├── TranslatorRegistry → constraint predicates + └── CompiledRules → total root predicate + plan metadata ``` -There are no separate `IRBuilder` / `RuleAssembler` / `ModelInspector` -classes — those names are historical. Behavior lives in the modules above -plus `sqlrules.compiler`. - ------------------------------------------------------------------------- - -## Compiler - -Orchestrates the two phases: `compile_model` (IR) and `bind` (columns + -translate). See the public `Compiler` in [API.md](API.md). - ------------------------------------------------------------------------- - -## inspectors - -`inspect_model(model) -> list[FieldDescriptor]` - -Enumerates fields in declaration order, including string aliases. - ------------------------------------------------------------------------- - -## constraints - -`extract_constraints` / `ensure_supported_type` / `pattern_text` - -`pattern_text` is part of the **Plugin API** (also exported from -`sqlrules`). Other helpers here are internal. - ------------------------------------------------------------------------- - -## columns - -`resolve_column(...)` maps field names / aliases / `column_map` to -SQLAlchemy columns. Never treats non-column `Table` attributes as columns. - ------------------------------------------------------------------------- - -## translators - -`TranslatorRegistry` and `default_registry` are **Plugin API**. - ------------------------------------------------------------------------- - -## cache - -`ModelIRCache` — process-lifetime strong-key cache. Call `clear()` when -using many ephemeral model classes. Not a public Application export. - ------------------------------------------------------------------------- - -## plugins / conformance - -`SQLRulesPlugin`, `PLUGIN_API_VERSION`, and `sqlrules.conformance` are -Plugin API. See [PLUGIN_SYSTEM.md](PLUGIN_SYSTEM.md). - ------------------------------------------------------------------------- - -## Design Principles - -- Small Application API -- Explicit Plugin API -- Internals may change between minors -- Deterministic compilation -- No database I/O +`compile_model()` normalizes one schema without binding it to a table. +`bind()` resolves the declared columns, asks the selected backend provider to +prepare each source value, translates every retained constraint, and returns a +`CompiledRules` object. The compiler does not connect to a database or execute +SQL. + +## Modules + +- `sqlrules.models` defines `RuleSchema`, the SQLRules `Field()` helper, + `RuleConfig`, and schema normalization. It accepts compatible Pydantic field + declarations and rejects declarations that cannot be represented as SQL + predicates. +- `sqlrules.integrations.pydantic` implements `from_pydantic()` and conversion + reports. It is public through the `sqlrules` package re-exports. +- `sqlrules.backend` prepares safe scalar expressions for providers. It is an + implementation module; provider authors implement the public `BackendProvider` + protocol instead of importing its private helpers. +- `sqlrules.columns` binds `RuleSchema` field names, aliases, explicit + `Field(column=...)` metadata, and `column_map` entries to SQLAlchemy columns. +- `sqlrules.constraints` extracts compatible normalized constraints. +- `sqlrules.ir` contains schema, prepared-value, compiled-result, and translator + data types. The public subset is re-exported from `sqlrules`. +- `sqlrules.translators` provides the public translator registry and built-in + operator translators. +- `sqlrules.conformance` contains helpers used to check plugin contracts. + +## State and caching + +SQLRules does not maintain a process-wide strong-reference cache of model +classes. `RuleSchema` declarations are normalized when compiled. The legacy +`Compiler(cache=...)` argument and `clear_model_cache()` helper remain as +compatibility shims; neither changes caching behavior in 2.0. SQLAlchemy +columns and compiled predicates are always specific to a bind operation. + +## Plugin boundary + +Plugin authors should import the supported contracts from `sqlrules`: +`SQLRulesPlugin`, `BackendProvider`, `PLUGIN_API_VERSION`, `TranslatorRegistry`, +`Constraint`, `PatternSpec`, `PreparedValue`, `CompilationContext`, and marker +types. See [PLUGIN_SYSTEM.md](PLUGIN_SYSTEM.md) and +[IR_CONTRACT.md](IR_CONTRACT.md). + +## Design constraints + +- Every retained rule must translate or raise an actionable capability error. +- Type conversion and validation must not rely on database predicate + evaluation order. +- Model validation remains ordinary Pydantic behavior; SQL compilation only + consumes the supported SQLRules subset. +- Internal module names and helpers may change without a Plugin API version + bump. diff --git a/docs/IR_CONTRACT.md b/docs/IR_CONTRACT.md index 7c3ece9..ff19cd2 100644 --- a/docs/IR_CONTRACT.md +++ b/docs/IR_CONTRACT.md @@ -1,61 +1,70 @@ -# IR contract (Plugin API v1) +# IR contract (Plugin API v2) -This document freezes the intermediate representation that Plugin API -version `"1"` may rely on. Semver for plugins follows -`PLUGIN_API_VERSION`, not the core package minor. +This page describes the stable intermediate representation available to +Plugin API version 2. The exact compatibility value is +sqlrules.PLUGIN_API_VERSION. -## Roots +## Schema IR | Type | Role | |---|---| -| `ModelIR` | Frozen model + ordered `FieldIR` tuple | -| `FieldIR` | `FieldDescriptor` + ordered `Constraint` tuple | -| `FieldDescriptor` | Field name, annotation, metadata, aliases | -| `Constraint` | `(field, operator, value)` | -| `PatternSpec` | `(pattern: str, ignore_case: bool)` for `pattern` | -| `TypeSpec` | `(python_type, strict, allow_none)` for `type_check` | -| `CompilationContext` | `on_unsupported`, diagnostics collector, optional `dialect` hint | -| `Diagnostic` | Structured skip/warn records (Plugin-visible; prefer not to depend on fields from Application code) | - -All IR dataclasses are frozen (`slots=True`). Phase-1 IR never stores -SQLAlchemy columns. +| SchemaSpec | RuleSchema class, ordered fields, empty-schema policy, provenance | +| RuleField | Name, annotation, logical type, nullability, strictness, constraints, metadata | +| Constraint | Field name, stable operator name, normalized value | -## Operators +Schema IR is independent of table bindings. It contains no SQLAlchemy +expressions. RuleField values preserve Python field order and source model +metadata. The compiler normalizes each class per compile, so mutable extension +payloads cannot leave stale process-wide schema IR. -Portable operators extracted by core: `gt`, `ge`, `lt`, `le`, -`multiple_of`, `min_length`, `max_length`, `pattern`, `literal`, `enum`. +## Backend preparation -Opt-in operator (when `emit_type_checks=True`): `type_check` with -`TypeSpec` values. Core extracts IR only — no portable translator -(same pattern as `pattern`). +| Type | Role | +|---|---| +| PreparedValue | Source column, normalized SQL value, total validity, null identity, logical type, coercion, capability | +| CompilationContext | Backend name, configured server version, assumptions, diagnostics | +| Diagnostic | Compile-scoped structured result message | -Frozen marker operator names: `json_contains`, `json_has_key`, -`array_contains`, `array_overlap`, `range_contains`, `range_overlap`, -`fulltext_match`. +BackendProvider.prepare_value(column, field, context) returns PreparedValue. +Every conversion must be safe for malformed and out-of-range row values even +if a database changes predicate evaluation order. Validity is total and does +not rely on a CAST being guarded by a separate AND expression. -`pattern` values are `PatternSpec` (or legacy `str` via `pattern_text()`). -Always call `pattern_text(constraint.value)` in translators. +Constraint translators receive the normalized PreparedValue.value expression +as their SQLAlchemy expression argument. They do not reimplement source +coercion or infer stored Python types. -`type_check` values are `TypeSpec`. Always call `type_spec(constraint.value)` -in translators. +## Result IR -## Application vs Plugin use +| Type | Role | +|---|---| +| FieldResult | Ordered bound field predicate and source/capability summary | +| CompiledRules | Complete root predicate, fields, diagnostics, backend, and assumptions | -- **Application:** prefer `compile` / `Compiler` / markers / exceptions. - Treat IR types as advanced (two-phase `compile_model` / `bind`). -- **Plugin:** may depend on `Constraint`, `PatternSpec`, `TypeSpec`, - `CompilationContext`, `ModelIR`, `TranslatorRegistry`, markers, - `pattern_text`, and `type_spec`. +CompiledRules.predicate is the authoritative complete predicate. It evaluates +to SQL TRUE or FALSE. where(compiled) returns a one-item list with the root; +notwhere(compiled) returns a one-item list with its complement. flatten() +aliases where(). Empty schemas are accepted only when explicitly configured; +their root is TRUE. -## Whole-model type matrix +## Operators -Every field annotation on a compiled model must be inside the type support -matrix — including unconstrained fields. Unsupported types (for example -`timedelta`) raise even when the field has no constraints. Split filter -models from DTOs that carry unsupported types. +Portable core operators are gt, ge, lt, le, multiple_of, min_length, +max_length, literal, and enum. Pattern and dialect markers remain plugin +operators. -## Not in IR (1.0) +Frozen marker operator names are json_contains, json_has_key, array_contains, +array_overlap, range_contains, range_overlap, and fulltext_match. -`max_digits` and `decimal_places` are rejected at extract time. Unknown -Field metadata keys are rejected rather than turned into invented -operators. +The opt-in type_check / TypeSpec IR from API v1 is removed from the 2.0 +compiler path. Scalar annotations now produce a backend-prepared type rule +automatically. + +## Plugin compatibility + +API v2 plugins must expose the exact api_version string. Backend providers +must expose prepare_value() and capabilities(). The compiler requires exactly +one provider when binding a schema. Constraint plugins can be included with +that provider and register translators through TranslatorRegistry. + +See [PLUGIN_SYSTEM](PLUGIN_SYSTEM.md) for translator examples. diff --git a/docs/MILESTONES.md b/docs/MILESTONES.md index b2728c1..51be7bf 100644 --- a/docs/MILESTONES.md +++ b/docs/MILESTONES.md @@ -195,8 +195,8 @@ No new major features. ## 2.x Release Policy -Status: planned. The [2.0 design](V2_DESIGN.md) defines the proposed API and -semantic contract. Phase 2.0 ends with **2.0.0**, phase 2.1 with **2.1.0**, and +Status: implementation in progress. The [2.0 design](V2_DESIGN.md) defines the +API and semantic contract. Phase 2.0 ends with **2.0.0**, phase 2.1 with **2.1.0**, and so on. Internal work packages below are dependency steps within a release. Dates are assigned after each phase's capability and conformance scope is fixed. @@ -206,7 +206,7 @@ depends on Pydantic v2; the conversion helper ships in the core distribution. while preserving the 2.0 semantic profile for existing schemas. Additional coercions that broaden accepted data require an explicit opt-in or profile. -## 2.0.0 — Rule Schemas and Semantic Foundations (planned) +## 2.0.0 — Rule Schemas and Semantic Foundations (in progress) ### Outcome @@ -315,8 +315,8 @@ Depends on A; integrates the schema representation from B. - Define plugin API v2 with distinct source preparation, capability reporting, and constraint translation. Require an explicit backend provider and stable capability errors. Do not allow `warn`/`ignore` to omit a retained rule. -- Freeze registry snapshots, isolate per-call state, and bound schema caches. - Keep diagnostics and table-bound expressions out of shared cached schema IR. +- Freeze registry snapshots and isolate per-call state. Do not keep + diagnostics, mutable payloads, or table-bound expressions in shared schema IR. Exit: public result behavior is fixed, nullable/grouping semantics are representable without later API changes, and IR-level truth tables show every @@ -360,9 +360,10 @@ Depends on B, C, and D passing their exits. release tooling. Remove stale references to plugin API v1 and package pins `>=1,<2`; align all five distribution versions at **2.0.0**. - Run database conformance, converter reports, Pydantic model validation, - FastAPI request/response checks, concurrency/cache checks, docs builds, and - wheel installation checks from clean environments. Record warm/cold compile - performance baselines for later comparison. + FastAPI request/response checks, concurrent compilation and cache-compatibility + checks, docs builds, and wheel installation checks from clean environments. + Record normalization, bind, and full compile performance baselines for later + comparison. - Confirm each example in the migration guide runs against its documented backend and adapter setup. Publish the frozen type/coercion capability matrix. @@ -433,7 +434,7 @@ capability/conformance checks. Existing schemas keep their previous semantics. - Type/coercion behavior is versioned; a later Pydantic release cannot silently change the meaning of an existing SQLRules schema. -- Database execution checks, performance tracking, bounded caches, concurrent +- Database execution checks, performance tracking, cache behavior, concurrent compilation, diagnostics, docs, and packaging start in 2.0 and continue in every phase. - Compilation enforces all retained rules. Permissive stripping is confined to diff --git a/docs/PERFORMANCE.md b/docs/PERFORMANCE.md index 452880d..2d17b8d 100644 --- a/docs/PERFORMANCE.md +++ b/docs/PERFORMANCE.md @@ -1,248 +1,48 @@ # SQLRules Performance -## Purpose +Compilation builds SQLAlchemy expression objects only. It does not connect to +a database, inspect live schema metadata, render SQL strings, or execute a +query. Work should scale approximately with the number of declared fields and +constraints. -This document defines the performance goals, optimization strategy, and -benchmarking methodology for SQLRules. +## Pipeline costs -The compiler should be fast enough that users rarely need to think about -its cost. Compilation should be deterministic, lightweight, and suitable -for use during application startup or on demand. +`Compiler.compile_model()` validates and normalizes a `RuleSchema` into an +immutable `SchemaSpec`. `Compiler.bind()` resolves its columns, prepares source +expressions through one backend provider, translates constraints, and creates +the final predicate and explain plan. Schema normalization and table binding +are measured separately by the benchmark. ------------------------------------------------------------------------- +SQLRules does not keep a process-wide model cache. The `cache` constructor +argument and `clear_model_cache()` function are compatibility shims and do not +enable or clear a cache. Compiled predicates remain bound to the SQLAlchemy +columns used to create them; applications should reuse a predicate only with +the table or alias it was compiled against. -## Performance Goals +## Benchmark -## Primary Goals - -- Low latency compilation -- Zero database I/O -- Minimal memory allocations -- Deterministic execution -- Linear scalability with respect to model size - -Target complexity: - -- **Time:** O(fields + constraints) -- **Memory:** O(fields + constraints) - ------------------------------------------------------------------------- - -## Compiler Pipeline - -``` text -Model - │ - ▼ -Inspect - │ - ▼ -Extract Constraints - │ - ▼ -Build IR - │ - ▼ -Translate - │ - ▼ -Assemble Rules -``` - -Each stage should process data in a single pass where practical. - ------------------------------------------------------------------------- - -## Expected Workload - -Typical models: - - Fields Constraints - -------- ------------- - 5 10 - 20 40 - 100 200 - -Compilation should scale approximately linearly. - ------------------------------------------------------------------------- - -## Caching - -## Metadata Cache - -Cache expensive model introspection (Phase 1). - -Key: - -``` python -model_class -``` - -Cached values: - -- field descriptors -- extracted constraints -- normalized `ModelIR` - -Avoid caching SQLAlchemy column objects, which are table-specific. - -Enabled by default (`cache=True`). Thread-safe via a lock-guarded -``dict`` keyed by model class. Entries are **strong** references for the -process lifetime (``ModelIR`` retains the model type). Call -``sqlrules.clear_model_cache()`` when creating many ephemeral models. - ------------------------------------------------------------------------- - -## Translator Cache - -Translator lookup should be O(1). - -Use dictionary-based dispatch keyed by constraint operator. - -``` python -{ - "ge": GreaterEqualTranslator(), - "min_length": MinLengthTranslator(), -} -``` - ------------------------------------------------------------------------- - -## Memory Strategy - -Prefer immutable lightweight objects. - -Recommendations: - -- dataclasses with `slots=True` -- tuples instead of mutable lists where appropriate -- avoid deep copies -- reuse immutable compiler state - ------------------------------------------------------------------------- - -## Allocation Strategy - -Avoid: - -- repeated metadata parsing -- duplicate constraint objects -- unnecessary intermediate collections - -Prefer generators internally when they improve readability without -sacrificing determinism. - ------------------------------------------------------------------------- - -## SQLAlchemy Interaction - -SQLRules should only construct SQLAlchemy expression objects. - -It must never: - -- connect to a database -- compile SQL strings -- inspect database metadata -- execute queries - -This keeps runtime predictable. - ------------------------------------------------------------------------- - -## Benchmark Suite - -Representative benchmarks live in `benchmarks/bench_compile.py`: +Run the representative local benchmark with: ```bash python -m benchmarks.bench_compile ``` -Sizes: - -- Small model (5 fields) -- Medium model (25 fields) -- Large model (100+ fields) - -Measure: - -- wall-clock compilation time (cold vs warm cache) -- cache hit benefit - -Illustrative latency targets (not CI-gated): - -- Small model cold compile on the order of sub-millisecond to a few ms -- Medium model under a few milliseconds when warm - -CI does **not** currently enforce performance regression gates. Formal -gates remain a post-1.0 / performance-milestone item. - ------------------------------------------------------------------------- - -## Profiling - -Use profiling to identify: - -- repeated introspection -- unnecessary allocations -- slow translator implementations - -Optimize only after profiling demonstrates a measurable benefit. - ------------------------------------------------------------------------- - -## Concurrency - -The compiler should be safe for concurrent reads after initialization. - -Recommendations: - -- immutable compiler configuration -- read-only translator registry -- thread-safe caches where applicable - ------------------------------------------------------------------------- - -## Future Optimizations - -Potential improvements: - -- incremental compilation -- persistent metadata caches -- plugin cache integration -- ahead-of-time IR generation -- optional C/Rust acceleration for metadata extraction - ------------------------------------------------------------------------- - -## Performance Targets - -Illustrative goals on modern hardware: - - Model Size Target Compile Time - ------------ --------------------- - Small \< 0.5 ms - Medium \< 2 ms - Large \< 10 ms - -Exact values should be validated through continuous benchmarking. - ------------------------------------------------------------------------- - -## Testing - -Local benchmarks: `python -m benchmarks.bench_compile`. - -CI performance regression gates are **not** enabled yet; treat the -latency table above as illustrative until a post-1.0 performance -milestone adds them. - ------------------------------------------------------------------------- - -## Design Principles - -- Measure before optimizing -- Optimize the common case -- Keep algorithms simple -- Favor deterministic performance -- Preserve readability over micro-optimizations +It measures normalization, bind/compile, and full compile for schemas with 5, +25, and 100 fields. The script reports best-of-repeat timings; its numbers are +diagnostic baselines, not CI thresholds. Run it on the same Python, SQLAlchemy, +hardware, and plugin versions when comparing changes. + +The benchmark intentionally does not imply a cache benefit or promise fixed +latency targets. SQLRules 2.0 establishes the measurement method; later 2.x +performance work can add stable CI thresholds after representative baselines +are collected. + +## Design guidance + +- Measure before optimizing and profile the stage that accounts for the cost. +- Keep schema and plan metadata immutable where practical. +- Avoid database I/O and dialect guessing in compilation. +- Keep provider conversion expressions safe if the database evaluates + predicates in a different order. +- Prefer straightforward linear passes over speculative caches or persistent + state. diff --git a/docs/PLUGIN_SYSTEM.md b/docs/PLUGIN_SYSTEM.md index 4785256..01a6a21 100644 --- a/docs/PLUGIN_SYSTEM.md +++ b/docs/PLUGIN_SYSTEM.md @@ -1,229 +1,114 @@ # SQLRules Plugin System -## Purpose +Plugin API v2 separates backend source preparation from constraint +translation. The application chooses one backend explicitly; SQLRules does +not inspect connections or discover packages automatically. -The SQLRules plugin system allows applications and libraries to extend -the compiler without modifying SQLRules itself. +## Plugin types -Plugins may register: +A constraint plugin implements name, api_version, and register(registry). A +backend provider implements those methods plus prepare_value() and +capabilities(). One provider is required when a schema is bound to a table. +Additional constraint plugins may be registered with that provider. -- New constraint translators -- Dialect-specific translators (overrides for the same IR operators) +The exact API version is available as PLUGIN_API_VERSION: -The core package remains small while advanced functionality lives in -plugins. Compiler passes, type registration, and entry-point discovery -are reserved for later releases. - ------------------------------------------------------------------------- - -## Design Goals - -- Zero-cost when unused -- Deterministic compilation -- Stable public extension API (`PLUGIN_API_VERSION`) -- No monkey-patching -- Explicit registration - ------------------------------------------------------------------------- - -## Architecture - -``` text - SQLRules Compiler - │ - ┌─────────┴─────────┐ - ▼ ▼ - Built-in Registry Plugin register() - │ │ - └─────────┬─────────┘ - ▼ - Translator Dispatch - ▼ - SQLAlchemy Expressions -``` - ------------------------------------------------------------------------- - -## Plugin Interface - -A plugin is a Python object that exposes `name`, `api_version`, and -`register(registry)`. - -``` python +~~~python from sqlrules import PLUGIN_API_VERSION, TranslatorRegistry, pattern_text -class CompanyPlugin: - name = "company" + +class PatternPlugin: + name = "my-patterns" api_version = PLUGIN_API_VERSION def register(self, registry: TranslatorRegistry) -> None: registry.register_constraint( "pattern", - lambda c, col, ctx: col.op("~")(pattern_text(c.value)[0]), + lambda constraint, value, context: value.op("~")(pattern_text(constraint.value)[0]), on_conflict="replace", ) -``` - -`api_version` must equal `sqlrules.PLUGIN_API_VERSION` (`"1"`). +~~~ -## Version policy +The translator's second argument is the backend-prepared SQL expression, not +the raw bound column. It must return one SQLAlchemy boolean expression. +Unsupported retained rules are errors; translators must not silently omit +them. -API v1 includes ``PatternSpec`` for ``pattern`` and ``TypeSpec`` for -``type_check`` constraint values. Always use ``pattern_text`` / -``type_spec``; do not assume bare ``str`` / ``type`` values. +## Backend provider contract -Bump ``PLUGIN_API_VERSION`` when changing translator signatures, registry -methods, or IR value types for built-in operators. Core package minor -bumps alone do not change the plugin API version. +A backend provider exposes: -``register_type``, ``register_dialect``, and ``register_compiler_pass`` -are **not** implemented on ``TranslatorRegistry`` in API v1. Do not probe -with ``hasattr``. +- name: stable backend identifier. +- api_version: exact match with sqlrules.PLUGIN_API_VERSION. +- server_version: configured server version, or None when the feature does not + require one. +- capabilities(): immutable-looking metadata describing supported source + types, versions, and assumptions. +- prepare_value(column, field, context): return a PreparedValue with source, + normalized value, total validity, null state, logical type, coercion, and + capability information. +- register(registry): install backend-specific constraint translators. ------------------------------------------------------------------------- +Preparation must make conversions safe for every stored value. A database row +that fails a known type or constraint rule evaluates to false. If the provider +cannot prove the requested conversion or source type, it raises +CapabilityError before returning SQL. -## Markers +## Register a backend and compile -Dialect-oriented constraints use `sqlrules.markers` (for example -`JsonContains`, `ArrayContains`). Markers implement the -`ConstraintMarker` protocol (`operator`, `value`) and are extracted into -IR. Plugins register translators for those operator names. +~~~python +from sqlrules import Compiler, where +from sqlrules_postgresql import PostgresPlugin -`ConstraintMarker` is intentionally duck-typed (`@runtime_checkable`). -Prefer the official marker dataclasses so operator names stay stable. +compiler = Compiler(plugins=[PostgresPlugin(server_version=(16, 0))]) +compiled = compiler.compile(UserRules, users) +statement = users.select().where(*where(compiled)) +~~~ -Frozen operator names: `json_contains`, `json_has_key`, `array_contains`, -`array_overlap`, `range_contains`, `range_overlap`, `fulltext_match`. +Pass exactly one backend provider. Passing multiple providers is an error. +The optional dialect argument is only an assertion against the selected +provider name; it never selects a backend. ------------------------------------------------------------------------- +## Registry -## Registry API +The registry starts with SQLRules portable comparison, length, and domain +translators. Plugins may add or replace translators with: -``` python +~~~python registry.register_constraint( operator="pattern", translator=translate_pattern, - on_conflict="raise", # or "replace" / "ignore" -) -``` - -Legacy alias: - -``` python -registry.register("pattern", translate_pattern, replace=False) -``` - -Introspection: `registry.operators()`, `operator in registry`, `registry.copy()`. - -Invalid translators raise `InvalidTranslatorError`. Duplicate operators -raise `RegistryError` unless `on_conflict` / `replace` allows otherwise. - -`register_type`, `register_dialect`, and `register_compiler_pass` are -not present on `TranslatorRegistry` in API v1 — do not probe with -`hasattr`. - ------------------------------------------------------------------------- - -## Using Plugins - -SQLRules does not auto-discover plugins. Register them explicitly: - -``` python -from sqlrules import Compiler -from sqlrules_postgresql import PostgresPlugin - -compiler = Compiler( - plugins=[PostgresPlugin()], - dialect="postgresql", # optional hint for translators - on_conflict="raise", # default when plugins call register() + on_conflict="replace", ) -rules = compiler.compile(MyModel, table) -``` - -Module-level `sqlrules.compile(...)` does not accept plugins; use -`Compiler` when you need extensions. - -When `plugins=` is set, the compiler copies the base registry first so -caller-owned registries are never mutated. - ------------------------------------------------------------------------- - -## Dialect Plugins - -Official packages (monorepo under `packages/`): - -- `sqlrules-postgresql` — regex, JSONB, ARRAY, range -- `sqlrules-sqlite` — REGEXP helper + JSON -- `sqlrules-mysql` — REGEXP, JSON, full-text -- `sqlrules-mssql` — JSON + `LEN` length overrides - ------------------------------------------------------------------------- - -## Conflict Resolution - -`Compiler(on_conflict=...)` sets the default for plugin `register()` calls: - -| Mode | Behavior | -|---|---| -| `raise` | `RegistryError` on duplicate (default) | -| `replace` | Overwrite existing translator | -| `ignore` | Keep the existing translator | - -Plugins may still pass an explicit `on_conflict=` to -`register_constraint`. +~~~ ------------------------------------------------------------------------- +Conflict policies are raise, replace, and ignore. Compiler(on_conflict=...) +provides the default policy for plugins that do not pass an explicit +on_conflict value. The compiler freezes a private registry snapshot; reading +compiler.registry returns a copy. -## Version Compatibility +## Official packages -``` python -from sqlrules import PLUGIN_API_VERSION # "1" -``` +- sqlrules-postgresql: regex, JSONB, arrays, ranges, PostgreSQL 16+ safe text + parsing. +- sqlrules-sqlite: runtime type checks, REGEXP, Python-compatible string + length, and JSON helpers. +- sqlrules-mysql: regex, JSON, full-text, and MySQL 8.0+ integer text parsing. +- sqlrules-mssql: JSON on SQL Server 2016+ with database compatibility level + 130+, LEN string behavior, and SQL Server 2012+ TRY_CAST. -Major plugin API changes increment this value. Mismatched plugins raise -`PluginError` at compiler construction. +All official packages must share a version line with the core package. The +[dialect support matrix](DIALECT_SUPPORT.md) lists capabilities and limits. ------------------------------------------------------------------------- +## Version compatibility -## Conformance Testing - -``` python -from sqlrules.conformance import run_basic_conformance -from sqlrules_postgresql import PostgresPlugin - -run_basic_conformance(PostgresPlugin(), operator="pattern") -``` - -Helpers also cover API version checks, builtin preservation, and -deterministic translation. Pass `model=` / `table=` / `field=` when -conformance-checking non-`pattern` operators. - ------------------------------------------------------------------------- +Plugin api_version must exactly match PLUGIN_API_VERSION, currently 2. API v1 +plugins must be adapted: the API v2 backend provider prepares values and +reports capabilities before translators run. A core package minor release +does not change the plugin API. ## Security -Plugins execute Python code. SQLRules does not sandbox plugins. -Install only trusted plugins. - ------------------------------------------------------------------------- - -## Non-Goals - -The plugin system does not support: - -- Runtime SQL execution -- Database connections -- Automatic package downloads / entry-point discovery -- Dynamic code generation -- Compiler pass plugins (future) - ------------------------------------------------------------------------- - -## Design Principles - -- Small core -- Extensible architecture -- Explicit registration -- Stable plugin API -- Pure compiler extensions -- Deterministic behavior +Plugins execute Python code and SQLAlchemy expression constructors. SQLRules +does not sandbox plugins. Install only trusted packages and use static regular +expressions where possible. diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 5a035a4..9e9d515 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -25,19 +25,20 @@ SQL. Security boundaries are: |---|---| | PostgreSQL `~` / `~*` | Engine-side regex cost | | MySQL `REGEXP` | Engine-side regex cost | -| SQLite `REGEXP` via `sqlrules_sqlite.register_regexp` | **Python `re.search` per row** in-process — catastrophic backtracking can stall the application worker | +| SQLite `REGEXP` via `sqlrules_sqlite.register_sqlite_functions` | **Python `re.search` per row** in-process — catastrophic backtracking can stall the application worker | **Recommendations** - Prefer **static / allowlisted** patterns authored with the model, not patterns taken from end-user input. - Do not expose free-form regex from untrusted clients on hot query paths. -- For SQLite, understand that `register_regexp` is not a sandbox; it runs +- For SQLite, understand that `register_sqlite_functions` is not a sandbox; it runs Python's `re` in your process. - Official dialect packages are released in lockstep with core — pin the - same major line (`sqlrules>=1,<2` and matching dialect extras). + same major line (`sqlrules>=2,<3` and the matching dialect package). ## Non-goals -SQLRules will not provide a plugin capability system, bytecode sandbox, -or automatic dialect detection. See [NON_GOALS.md](NON_GOALS.md). +SQLRules does not sandbox plugins or automatically detect a backend. Providers +report capabilities, but installing and selecting a trusted package remains +the application's responsibility. See [NON_GOALS.md](NON_GOALS.md). diff --git a/docs/SPEC.md b/docs/SPEC.md index 691cac6..c8c04ef 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -1,118 +1,96 @@ -# Specification +# SQLRules 2.0 Specification -## Primary API +## Public contract -```python -rules = sqlrules.compile(MyModel, table) -``` +`RuleSchema` is a Pydantic v2 `BaseModel` subclass whose accepted fields have +SQL meaning. It remains a normal Pydantic model: callers can instantiate it, +call `model_validate()` and `model_dump()`, and use it as a FastAPI model. +Unrestricted application models remain supported through the explicit +`from_pydantic()` converter. -## Returns +Every supported scalar annotation creates a type rule, whether or not the +field has another constraint. The caller selects exactly one backend provider: -```python -dict[str, list[ColumnElement[bool]]] -``` +~~~python +from sqlrules import Compiler, RuleSchema, where +from sqlrules_postgresql import PostgresPlugin -Ordering is deterministic and follows model field declaration order. -Within a field, expressions follow constraint extraction order. -Fields that produce no expressions are omitted from the dictionary. -## Supported constraints (v1.0) +class UserRules(RuleSchema): + id: int + age: int | None -- `gt` -- `ge` -- `lt` -- `le` -- `min_length` -- `max_length` -- `multiple_of` -- `Literal` -- `Enum` -`pattern` is extracted into IR as `PatternSpec` but has no portable core -translator. It raises by default; use `on_unsupported="warn"` / `"ignore"`, -register a custom translator, or install a dialect plugin. Treat untrusted -patterns as a CPU/ReDoS cost risk (see [SECURITY.md](SECURITY.md)). +compiled = Compiler(plugins=[PostgresPlugin()]).compile(UserRules, users) +statement = users.select().where(*where(compiled)) +~~~ -Dialect markers (`JsonContains`, `ArrayContains`, `RangeContains`, -`FullTextMatch`, …) are extracted into IR and require a dialect plugin. +Compilation performs no database I/O. It returns a `CompiledRules` instance +containing one total root predicate, per-field results, capability assumptions, +diagnostics, and `explain()` output. -`max_digits` and `decimal_places` are rejected at extract time (no portable -SQL mapping in 1.0). +## Predicate helpers -## Supported types (v1.0) +- `where(compiled)` returns a one-item list containing the complete predicate. +- `flatten(compiled)` is an alias for `where(compiled)`. +- `notwhere(compiled)` returns a one-item list containing the exact complement. -- `bool`, `int`, `float`, `Decimal`, `str` -- `date`, `datetime`, `time` -- `UUID` (Literal / Enum only) -- `Literal[...]`, `Enum` -- `list` / `dict` (allowed annotations; portable constraints raise; - unconstrained containers are skipped like unconstrained scalars; - dialect markers require a plugin) +The predicate is always SQL TRUE or FALSE. SQL NULL matches a field only when +its annotation is nullable. Invalid values and failed constraints do not raise +from the SQL expression; they produce a non-match. `notwhere()` therefore +selects each row that fails one or more rules, including NULL and invalid +values. -**Whole-model rule:** every field annotation must be in this matrix, including -fields with no constraints. Unsupported types (for example `timedelta`) always -raise. Split filter models from DTOs that carry unsupported types. +## Supported declarations -## `where` / `flatten` +Scalar annotations: `bool`, `int`, `float`, `Decimal`, `str`, `date`, +`datetime`, `time`, and `UUID`; homogeneous `Literal` and `Enum` domains; and +nullable forms. Lists and dictionaries are accepted only with a supported +dialect marker. Collection item validation, general unions, nested JSON +schemas, and custom Python validators are outside the 2.0 subset. -Both names are supported aliases of the same function. +Supported constraints include `gt`, `ge`, `lt`, `le`, `multiple_of`, string +`min_length` / `max_length`, `pattern` where the provider registers a +translator, and compatible Literal/Enum membership. Existing JSON, array, +range, and full-text markers remain provider-specific. -## Unsupported constraints +Pydantic `Field`, `ConfigDict`, strict aliases, supported constrained aliases, +`annotated_types` constraints, and compatible `Annotated` metadata can be +imported directly. SQLRules `Field(column=...)` adds an optional database +column binding. Class creation rejects unsupported metadata, invalid bounds, +duplicate constraints, custom validators/serializers, computed fields, and +custom schema hooks. -Unsupported **constraint operators** raise `UnsupportedConstraintError` by -default. +## Type and conversion behavior -Policies (`on_unsupported`): +Lax mode uses the versioned SQLRules conversion profile in +[TYPE_SUPPORT](TYPE_SUPPORT.md). Strict mode checks the logical type observable +in the database. An explicit field strict setting overrides a reusable type +setting, which overrides model `ConfigDict(strict=...)`; lax is the default. +SQLite relies on runtime `typeof()` because column affinity cannot prove the +stored type of each row. -| Mode | Behavior | -|---|---| -| `raise` | Raise immediately (default) | -| `warn` | Emit a `SQLRulesWarning`, record a diagnostic, and skip | -| `ignore` | Record a diagnostic and silently skip | - -`on_unsupported` applies only to unknown constraint operators. Unsupported -**types** (for example containers and `timedelta`) always raise. +A backend mismatch that can be established safely is a non-match. If a backend +cannot safely inspect or convert a source representation, compilation raises +`CapabilityError`. Retained rules are never skipped by a warning or ignore +policy. ## Column binding -Each constrained field is bound to a SQLAlchemy column via, in order: - -1. `column_map` (keys may be the field name or a string alias) -2. `table.c` -3. ORM / attribute lookup, only when the attribute is a `ColumnElement` or - exposes `__clause_element__()` - -String `Field(alias=...)`, `validation_alias`, and `serialization_alias` are -tried before the Python field name. Non-column table attributes (for example -`Table.name` or `Table.is_selectable`) are never treated as columns. - -Unconstrained fields are skipped and do not require a matching column. +For each rule field, resolution order is: -## Two-phase compilation +1. `column_map[field_name]` +2. `sqlrules.Field(column="database_name")` +3. the Python field name -```python -compiler = sqlrules.Compiler() -model_ir = compiler.compile_model(MyModel) # cached by default -rules = compiler.bind(model_ir, table) -``` - -Phase 1 caches immutable model IR. Phase 2 binds columns and translates. +Pydantic validation and serialization aliases keep their normal model +behavior; they do not select database columns automatically. Every schema +field must bind to a column, including fields with only a type annotation. ## Plugins -```python -compiler = sqlrules.Compiler( - plugins=[PostgresPlugin()], - dialect="postgresql", - on_conflict="raise", -) -``` - -Plugins must declare `api_version == PLUGIN_API_VERSION`. See -[PLUGIN_SYSTEM.md](PLUGIN_SYSTEM.md). - -## Compatibility - -- Python 3.10+ -- Pydantic v2 -- SQLAlchemy 2.x +Constraint plugins register translators through `TranslatorRegistry`. +Backend providers additionally prepare source values and expose a capability +profile. The plugin API version is exported as +`sqlrules.PLUGIN_API_VERSION` and currently equals `"2"`. See +[PLUGIN_SYSTEM](PLUGIN_SYSTEM.md) and [DIALECT_SUPPORT](DIALECT_SUPPORT.md). diff --git a/docs/TESTING.md b/docs/TESTING.md index a59492f..95cefa2 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -1,16 +1,19 @@ # Testing Strategy -## Unit tests +## Test layers -- One test module per constraint family (numeric, string, literal/enum) -- Two-phase compile, cache, diagnostics, UUID/time, pattern IR -- Plugins, registry conflicts, diagnostic codes, conformance -- Markers / dialect packages under `packages/*/tests` -- Opt-in `emit_type_checks` / `TypeSpec` -- Error-path coverage for missing columns, invalid models, and policies -- Helper coverage for `where` / `flatten` +- RuleSchema declaration and normal Pydantic validation behavior +- Pydantic conversion policies, reports, aliases, defaults, and callback loss +- Compiler result grouping, explicit backend selection, column binding, and + capability errors +- SQLite execution with malformed values, strict/lax types, SQL NULL, and the + `where()` / `notwhere()` partition +- Dialect translation and provider conformance for PostgreSQL, MySQL, and SQL + Server +- Live database execution conformance for all four official backends in CI +- Version synchronization, docs, package builds, wheel imports, lint, and types -## Plugin packages +Run the fast suite with: ```bash pytest tests \ @@ -20,52 +23,33 @@ pytest tests \ packages/sqlrules-mssql/tests ``` -Or simply `make test` (also runs lint, mypy, and version sync). +Live conformance tests use the `SQLRULES_TEST_*_URL` environment variables and +are skipped locally when no database URL is configured. CI starts PostgreSQL, +MySQL, and SQL Server services and exercises their text-to-integer and +nullable-complement behavior. SQLite tests execute in memory. -Use `sqlrules.conformance.run_basic_conformance(plugin)` in package tests. +## Pydantic reference -**Coverage:** core package (`sqlrules`) is gated at ≥95% via pytest -`--cov-fail-under=95`. Dialect plugin packages are tested but are **not** -included in that fail-under gate. +The semantic profile is compared against Pydantic **2.13.4**. The runtime +dependency permits the Pydantic v2 line; CI pins the reference job to this +version so coercion changes are deliberate and reviewable. -## Benchmarks - -Local only (not a CI gate): - -```bash -python -m benchmarks.bench_compile -``` - -## Compatibility - -- CI matrix covers Python 3.10–3.13 with current Pydantic v2 / SQLAlchemy 2.x - from package dependencies (no separate Pydantic version matrix) -- SQLAlchemy Core expression construction in unit tests -- Occasional SQLite / PostgreSQL dialect `.compile(...)` smoke assertions - -## Quality gates +## CI gates CI (`.github/workflows/ci.yml`) runs: -- **Lint** (Python 3.12): `ruff check`, `ruff format --check`, `mypy`, - version sync between `pyproject.toml` and `sqlrules.__version__` -- **Test** matrix (Python 3.10–3.13): core + dialect plugin tests, - coverage fail-under 95% (core only) -- **Docs**: `sphinx-build -W` -- **Package**: build/twine-check core and dialect plugins; import wheel - -Locally, match CI with: - -```bash -make check # lint + types + tests + docs -make dist # packaging smoke -``` - -Benchmarks are local-only and are not a CI gate. +- Ruff lint and formatting, mypy, and version synchronization +- Core and dialect package tests on Python 3.10–3.13 with an 80% combined + statement and branch coverage floor for the rewritten 2.0 core +- Live backend execution conformance on PostgreSQL, MySQL, and SQL Server +- Sphinx with warnings treated as errors +- Core and dialect wheel builds, Twine checks, and installed-wheel imports ## Principles -- Prefer expression-shape assertions over rendered SQL when possible -- Keep tests free of database connections -- Every new translator needs success and failure cases -- Plugins must preserve built-in operators unless explicitly replacing +- Test behavior through model validation and executed SQL where possible. +- Compile-time SQL assertions supplement database execution tests. +- Every retained rule must either translate or raise a stable capability + error; warning and ignore policies must not silently omit it. +- Every live fixture proves that `where(compiled)` and `notwhere(compiled)` + select disjoint sets whose union is the full fixture set. diff --git a/docs/TRANSLATORS.md b/docs/TRANSLATORS.md index 853d4d3..6231ba6 100644 --- a/docs/TRANSLATORS.md +++ b/docs/TRANSLATORS.md @@ -65,7 +65,7 @@ Returns: ge `column >= value` lt `column < value` le `column <= value` - multiple_of `column % value == 0` + multiple_of `column % value == 0` for backend-supported numeric types ------------------------------------------------------------------------ @@ -73,15 +73,15 @@ Returns: Constraint Translation ------------ -------------------------------- - min_length `func.length(column) >= value` - max_length `func.length(column) <= value` + min_length backend-specific character-length comparison + max_length backend-specific character-length comparison -**`pattern` (1.0):** extracted as `PatternSpec` in core IR; **no portable +**`pattern`:** extracted as `PatternSpec` in core IR; **no portable core translator**. Install `sqlrules-postgresql` / `sqlrules-sqlite` / `sqlrules-mysql`, or register a custom translator. SQL Server (`sqlrules-mssql`) does **not** register `pattern`. -**Not in 1.0:** `starts_with`, `ends_with`, `contains`. +`starts_with`, `ends_with`, and portable `contains` do not have 2.0 translators. ------------------------------------------------------------------------ @@ -162,11 +162,8 @@ Each translator should: ## Unsupported Constraints -If no translator exists, behavior depends on the compiler policy. - -- raise -- warn -- ignore +If no translator exists, compilation raises `CapabilityError`. The 2.0 +compiler does not allow warning or ignore policies to remove a retained rule. ------------------------------------------------------------------------ @@ -184,9 +181,7 @@ from sqlrules.translators import default_registry registry = default_registry() registry.register_constraint( "pattern", - lambda constraint, column, context: column.op("~")( - pattern_text(constraint.value)[0] - ), + lambda constraint, column, context: column.op("~")(pattern_text(constraint.value)[0]), on_conflict="replace", ) compiler = Compiler(registry=registry) @@ -204,6 +199,7 @@ class PatternPlugin: on_conflict="replace", ) + compiler = Compiler(plugins=[PatternPlugin()]) ``` @@ -214,7 +210,7 @@ compiler = Compiler(plugins=[PatternPlugin()]) Some SQL constructs differ by backend. Official packages: - `sqlrules-postgresql` — `~` / `~*`, JSONB, ARRAY, range -- `sqlrules-sqlite` — REGEXP helper + JSON +- `sqlrules-sqlite` — REGEXP and Unicode-aware length helpers + JSON - `sqlrules-mysql` — REGEXP, JSON, full-text - `sqlrules-mssql` — JSON + `LEN` length overrides diff --git a/docs/TYPE_SUPPORT.md b/docs/TYPE_SUPPORT.md index 8693554..e89a21c 100644 --- a/docs/TYPE_SUPPORT.md +++ b/docs/TYPE_SUPPORT.md @@ -1,294 +1,120 @@ -# SQLRules Type Support Matrix - -## Purpose - -This document defines which Python and Pydantic field types SQLRules -supports and how each type is translated into SQLAlchemy WHERE -expressions. - -SQLRules only supports types whose constraints can be translated -deterministically into SQLAlchemy boolean expressions. - ------------------------------------------------------------------------- - -## Guiding Principles - -- Prefer deterministic translations. -- Never guess SQL semantics. -- Unsupported types always fail fast (they are not softened by - `on_unsupported`). -- Type support is independent of database dialect whenever possible. - ------------------------------------------------------------------------- - -## Supported types (1.0) - - Python Type Supported Notes - ------------- ----------- ------------------------------------- - bool ✅ Equality / literal constraints only - int ✅ Numeric comparisons - float ✅ Numeric comparisons - Decimal ✅ Numeric comparisons (`max_digits` / - `decimal_places` still unsupported) - str ✅ Length constraints, Literal, Enum; - `pattern` IR only - date ✅ Range comparisons - datetime ✅ Range comparisons (no TZ conversion) - time ✅ Range comparisons - Enum ✅ Translated to `IN (...)` - Literal ✅ Translated to `IN (...)` - UUID ✅ Literal / Enum only - timedelta ❌ No deterministic SQL mapping yet - ------------------------------------------------------------------------- - -## Numeric Types - -Supported constraints: - -- gt -- ge -- lt -- le -- multiple_of - -Example: - -``` python -age: Annotated[int, Field(ge=18, le=65)] -``` - -↓ - -``` python -[ - users.c.age >= 18, - users.c.age <= 65, -] -``` - ------------------------------------------------------------------------- - -## Strings - -Supported constraints: - -- min_length -- max_length - -IR only (no portable translator in core): - -- pattern - -Planned (later milestones): - -- starts_with -- ends_with -- contains - -Example: - -``` python -name: Annotated[str, Field(min_length=2)] -``` - -↓ - -``` python -func.length(users.c.name) >= 2 -``` - ------------------------------------------------------------------------- - -## Boolean - -Boolean fields generally do not produce rules unless constrained through -`Literal` or similar constructs. - -Examples: - -``` python -Literal[True] -Literal[False] -``` - -↓ - -``` python -column.in_([True]) -``` - ------------------------------------------------------------------------- - -## Date, DateTime, and Time - -Supported operators: - -- gt -- ge -- lt -- le - -`multiple_of` is rejected for temporal types. - -No timezone conversion is performed by SQLRules. Aware and naive -`datetime` values are passed through unchanged. - ------------------------------------------------------------------------- - -## UUID - -Supported: - -- Unconstrained fields (skipped; no column required) -- `Literal[...]` / `Enum` - -Rejected: - -- length constraints -- numeric comparisons - ------------------------------------------------------------------------- - -## Enum - -Example: - -``` python -class Status(Enum): - ACTIVE = "ACTIVE" - DISABLED = "DISABLED" -``` - -↓ - -``` python -column.in_(["ACTIVE", "DISABLED"]) -``` - ------------------------------------------------------------------------- - -## Literal - -Example: - -``` python -Literal["A", "B", "C"] -``` - -↓ - -``` python -column.in_(["A", "B", "C"]) -``` - ------------------------------------------------------------------------- - -## Optional Types - -`Optional[T]` / `T | None` is supported. - -By itself, optionality does not generate SQL rules unless -`emit_type_checks=True`, in which case `TypeSpec.allow_none` is set and -dialect translators emit `(column IS NULL) OR `. - ------------------------------------------------------------------------- - -## Type checks (`emit_type_checks`) - -Opt-in: `Compiler(emit_type_checks=True)` or -`sqlrules.compile(..., emit_type_checks=True)`. - -When enabled, supported **scalar** annotations emit a `type_check` -constraint with `TypeSpec(python_type, strict, allow_none)`: - -- Strictness from `Field(strict=…)`, `Strict()`, then - `model_config = ConfigDict(strict=…)` -- `Literal` / `Enum` / `list` / `dict` do **not** get `type_check` - (domain already covered or marker-only) - -Core extracts IR only — there is **no portable translator**. Install a -dialect plugin (or register a custom `type_check` translator). - -Approximations are dialect-specific and intentionally incomplete versus -full Pydantic coercion. Unsupported `(type, strict)` pairs raise -`UnsupportedConstraintError` rather than guessing. See dialect plugin -READMEs for the matrix. - -Example: - -```python -from sqlrules import Compiler -from sqlrules_postgresql import PostgresPlugin - -class Filter(BaseModel): - age: int - -compiler = Compiler(plugins=[PostgresPlugin()], emit_type_checks=True) -rules = compiler.compile(Filter, users) -``` - ------------------------------------------------------------------------- - -## Containers - - Type Status - ------- -------- - list ✅ Marker-driven fields only (e.g. `ArrayContains`) - dict ✅ Marker-driven fields only (e.g. `JsonContains`) - tuple ❌ - set ❌ - -Portable constraints (`gt`, `min_length`, `pattern`, …) on `list` / `dict` -still raise. Use `sqlrules.markers` with a dialect plugin. - ------------------------------------------------------------------------- - -## Unsupported Pydantic Features - -The following are intentionally unsupported: - -- validators -- field_validator -- model_validator -- computed_field -- serializer hooks -- arbitrary custom metadata (except `ConstraintMarker` instances) -- custom validation functions -- `max_digits` / `decimal_places` (deferred; no deterministic WHERE map) - -These features require runtime execution or lack portable SQL semantics. - ------------------------------------------------------------------------- - -## Post-1.0 expansion - -Historical 0.2–0.4 delivery notes live in [MILESTONES](MILESTONES.md), not -here. This matrix is the **1.0** contract. - -Planned *after* 1.0 (not available yet) only if SQL semantics are -deterministic: `starts_with` / `ends_with` / `contains`, decimal precision -constraints. See the project roadmap. - -Support should expand only when the resulting SQL semantics are -well-defined across supported SQLAlchemy backends. - ------------------------------------------------------------------------- - -## Compatibility - -Target versions: - -- Python 3.10+ -- Pydantic v2 -- SQLAlchemy 2.x - ------------------------------------------------------------------------- - -## Design Principles - -- Explicit support matrix -- Stable translations -- Backend-neutral semantics -- Fail fast for unsupported types -- Predictable compiler output +# SQLRules 2.0 Type and Coercion Support + +This page defines the logical values accepted by RuleSchema and the source +representations that each official backend can safely prepare for SQL +constraints. Compilation never probes a database connection. A caller selects +one backend provider and supplies server-version information when a capability +depends on it. + +## Logical types + +RuleSchema supports bool, int, float, Decimal, str, date, datetime, time, UUID, +homogeneous Literal domains, and Enum domains with one supported scalar value +type. Optional annotations add a SQL NULL branch. List and dict fields are +marker-driven and do not validate nested values or collection items. + +Every scalar annotation produces a type rule, even when it has no other +constraints. The supported Pydantic declarations include Field constraints, +ConfigDict strictness, strict aliases, supported constrained aliases, and +compatible Annotated markers. + +## Coercion profile + +Lax mode follows this SQLRules profile rather than relying on implicit database +casts: + +| Source representation | Target | 2.0 behavior | +|---|---|---| +| Matching native logical type | Same type | Match, then apply constraints | +| Integer | float or Decimal | Lax conversion where the backend can represent the full signed 64-bit source range; see the backend matrix | +| Float or Decimal | int | Lax conversion only when the value is an exact integer in signed 64-bit range | +| Numeric text | int | Signed decimal digits with optional surrounding whitespace; accepted magnitude is limited by the backend profile | +| Numeric text | float | Decimal/exponent grammar where the backend has a safe parser | +| Numeric text | Decimal | PostgreSQL 16+ only | +| Any known mismatched type in strict mode | Any other type | Non-match | +| Unknown SQLAlchemy type or unavailable evidence | Any target | CapabilityError | + +SQLRules integer coercion uses a signed 64-bit target range because the +supported SQL backends do not share Python's unbounded integer arithmetic. +SQLite and MySQL text-to-int conversion conservatively accepts at most 18 +digits. The SQLRules profile rejects fractional values for integer fields and +does not coerce numbers to strings. Textual date, datetime, time, and UUID +parsing is not part of 2.0. + +Pydantic and SQLRules validation are intentionally close for supported +annotations, but database storage only exposes database logical types. SQLRules +does not claim to recover the Python object originally inserted into a row. + +## Backend matrix + +The following matrix describes compile-time support in the implementation. +Exact execution evidence is tracked separately from SQL rendering and must be +recorded for each supported server release before publishing 2.0.0. The CI +`Live database conformance` job runs against PostgreSQL 16, MySQL 8.0, and SQL +Server 2022. Its shared corpus checks malformed text, integer overflow, +fractional/strict mismatches, SQL NULL, native scalar constraints, UUID where +native storage exists, and the `where()`/`notwhere()` partition. SQLite runs +the corresponding in-memory storage and malformed JSON cases in the regular +test suite. + +| Backend | Native scalar source types | Lax conversions | Important limitations | +|---|---|---|---| +| PostgreSQL 16+ | bool, int, float, Decimal, str, date, datetime, time, UUID | text to int/float/Decimal; int or Decimal to float; integral float/Decimal to int | Pass server_version for text conversions. String Literal/Enum requires collation C or POSIX. | +| SQLite 3.x | runtime integer, real, and text classes | text to int/float; integral real to int; integer to float; lax bool from integer 0/1 | Strict bool, Decimal, date/time, and UUID require storage mappings and raise CapabilityError. Text checks and string length constraints require `register_sqlite_functions()`. Text-to-int is limited to 18 digits. | +| MySQL 8.0+ | Boolean (0/1 checked), integer, float, Decimal, text, date/time | text to int; integer/Decimal to float; integral float/Decimal to int | UUID storage is not treated as native evidence. Text-to-float and text-to-Decimal are rejected. Integer-to-Decimal is rejected because the default cast precision is too small for every signed 64-bit integer. String Literal/Enum requires an explicit `_bin` collation; membership checks character length to account for PAD SPACE collations. `CHAR` columns are rejected because trailing-space retrieval depends on a session mode. | +| SQL Server 2012+ | bit, integer, float, Decimal, text, date/time, uniqueidentifier | text to int/float; integer/Decimal to float; integral float/Decimal to int | Text-to-Decimal and integer-to-Decimal are rejected because a default cast precision cannot represent the full source domain. JSON markers additionally require SQL Server 2016+ and explicit database `compatibility_level >= 130` because they use `OPENJSON`; `JsonContains` supports object payloads and structurally compared arrays, rejects top-level scalar payloads, and rejects numeric JSON comparisons because `OPENJSON` exposes numbers as text and no lossless general conversion is available. String Literal/Enum requires an explicit `_BIN2` collation; membership also checks byte length to account for padded string equality. Regex pattern translation is unavailable. | + +The provider capability report lists the configured version and assumptions. +If the source SQLAlchemy type does not establish the required logical type, +compilation raises CapabilityError instead of emitting a guessed predicate. + +SQLite uses typeof() at runtime because column affinity does not prove the +storage class of an individual row. Its integer storage class cannot +distinguish a Python bool from a Python int, so strict bool is unavailable +without an explicit mapping. SQLite NUMERIC affinity cannot prove exact +Decimal precision. SQLAlchemy Date, DateTime, Time, and UUID representations +are emulated on SQLite and are not treated as native logical types. + +## Constraints and domains + +Portable scalar constraints are gt, ge, lt, le, multiple_of, min_length, +max_length, Literal membership, and Enum membership. Pattern is available only +when the selected backend registers a translator. The SQL Server plugin has no +pattern translator. + +String Literal and Enum membership requires comparisons with case-sensitive +semantics. PostgreSQL requires C or POSIX collation, SQLite uses BINARY, and +MySQL requires an explicit `_bin` collation and SQL Server requires `_BIN2`. +Their string-domain predicates additionally compare candidate and column lengths +so trailing spaces cannot make distinct Python strings match. +This prevents server-default case folding from silently broadening a domain. + +The type and constraint combination is checked when RuleSchema is created. +Duplicate constraints, contradictory bounds, unsupported metadata, arbitrary +validators, serializers, computed fields, and normalization transforms are +rejected. A full Pydantic model can still use those features outside SQLRules; +from_pydantic() reports and removes unsupported behavior before compilation. + +## Nulls and complements + +For a field, SQLRules computes: + +~~~text +SQL NULL -> match only when the annotation is nullable +non-NULL -> type/coercion validity AND every field constraint +~~~ + +Each field result and the complete root predicate are normalized to SQL TRUE +or FALSE. where(compiled) returns a one-element list containing the complete +predicate. notwhere(compiled) returns its complement, so every row is selected +by exactly one side. An explicitly allowed empty schema compiles to TRUE and +its notwhere() complement is FALSE. + +## Deferred capabilities + +2.0 does not translate general unions, recursive types, nested JSON schemas, +collection item validation, cross-field rules, arbitrary Python callbacks, +named transforms, or custom storage mappings. Later 2.x phases build on the +prepared-value and capability contracts defined here. + +Requires Python 3.10+, Pydantic v2, and SQLAlchemy 2.x. diff --git a/docs/V2_DESIGN.md b/docs/V2_DESIGN.md index 5d679af..7653b5a 100644 --- a/docs/V2_DESIGN.md +++ b/docs/V2_DESIGN.md @@ -1,7 +1,8 @@ # SQLRules 2.0 Design -Status: proposed implementation contract. The APIs below are design sketches; -the shipped package remains 1.x until the 2.0.0 release gates are met. +Status: SQLRules 2.0 implementation contract. The core and four official +packages target the 2.0.0 line; the milestone gates track remaining release +conformance evidence. See [Milestones](MILESTONES.md) for implementation order and release scope. ## Product contract @@ -22,7 +23,7 @@ predicate selects matching rows; it neither rewrites stored values nor changes the values projected by a SELECT. Checking that an entire table passes requires a caller-executed query for rows failing the predicate. -## Proposed authoring and compilation API +## Authoring and compilation API ```python from typing import Annotated, Literal @@ -31,6 +32,7 @@ import sqlrules from pydantic import ConfigDict, Field, PositiveInt, StrictBool, StringConstraints from sqlrules_postgresql import PostgresPlugin + class UserRules(sqlrules.RuleSchema): model_config = ConfigDict(strict=False) @@ -40,13 +42,16 @@ class UserRules(sqlrules.RuleSchema): status: Literal["active", "disabled"] verified: StrictBool -user = UserRules.model_validate({ - "id": "12", - "age": "21", - "name": "Ada", - "status": "active", - "verified": True, -}) + +user = UserRules.model_validate( + { + "id": "12", + "age": "21", + "name": "Ada", + "status": "active", + "verified": True, + } +) data = user.model_dump() compiler = sqlrules.Compiler( @@ -287,11 +292,11 @@ exposes its complement. A `notwhere()` helper over the 1.x rule dictionary cannot provide this guarantee by simply negating its existing expressions. Use immutable registry snapshots and per-call compilation state. Bound -SQLAlchemy expressions are never stored in the process-wide schema cache. -Bounded schema caching keys include schema and semantic-profile identity; -configuration or marker payload mutation cannot leave stale cache entries. -Unsupported mutable extension payloads must disable caching or supply a stable -snapshot protocol. Diagnostics must not leak between concurrent compilations. +SQLAlchemy expressions are never stored in shared schema IR. SQLRules does not +maintain a process-wide schema cache; it normalizes declarations per compile +so mutable marker payload changes cannot leave stale entries. Pydantic keeps +its own compiled model validators. Diagnostics must not leak between +concurrent compilations. ## Explicit Pydantic conversion @@ -395,7 +400,7 @@ unavailable types explicitly. No all-types/all-dialects parity claim is made. - Include reference semantic tests for `model_validate()` and SQL predicates, backend execution tests that partition rows into matches and failures under `notwhere(compiled)` (including SQL NULL, invalid conversions, and allowed empty - schemas), adapter reports, concurrent compilation, bounded caches, + schemas), adapter reports, concurrent compilation, cache-compatibility checks, documentation builds, and wheel installation in release validation. Establish performance baselines in 2.0 and measure subsequent changes. diff --git a/docs/VISION.md b/docs/VISION.md index 6df94d9..80e226f 100644 --- a/docs/VISION.md +++ b/docs/VISION.md @@ -1,8 +1,8 @@ # Vision -SQLRules 1.x is a tiny compiler that bridges Pydantic model constraints and -SQLAlchemy WHERE expressions. The 2.x roadmap makes SQLRules-owned rule -schemas the primary input, with explicit Pydantic conversion. +SQLRules compiles SQL-compatible rule declarations into SQLAlchemy predicates. +The 2.x contract makes SQLRules-owned rule schemas the primary input, with +explicit conversion from unrestricted Pydantic models. Native rule schemas remain Pydantic v2 models: users can instantiate them, validate and serialize data, and use them in FastAPI. SQLRules restricts the diff --git a/docs/guides/design-philosophy.md b/docs/guides/design-philosophy.md index f6c6878..996f932 100644 --- a/docs/guides/design-philosophy.md +++ b/docs/guides/design-philosophy.md @@ -1,53 +1,49 @@ # Design philosophy -SQLRules does **one** job: compile a safe subset of Pydantic **Field -constraint metadata** into SQLAlchemy WHERE expressions. +SQLRules compiles a supported subset of Pydantic model declarations into +SQLAlchemy predicates over stored rows. ```text -Field(ge=18) → column >= 18 -UserFilter(age=25) → column == 25 # NOT what SQLRules does +RuleSchema field declaration → type/coercion check + supported constraints +request field value → not used as a database filter value ``` ## Why it exists -Applications often encode filter intent twice—once in Pydantic models and -again as hand-written SQLAlchemy clauses. SQLRules keeps a single source of -truth: the model’s constraints become deterministic column expressions. +Applications can use one Pydantic model to describe compatible input and row +rules. A `RuleSchema` is still a normal Pydantic model; SQLRules only limits +which declarations it accepts inside the class. An unrestricted Pydantic model +can be converted with `from_pydantic()`, which reports the semantics it drops. ## Principles -- **Do one thing well.** Not an ORM, validator, query builder, SQL string - generator, migration tool, or database client. -- **Deterministic output.** Same model + table → same rule dictionary order - and expressions. -- **Refuse to guess.** If a Pydantic feature cannot be translated safely, the - compiler raises (or warns/ignores for unknown *operators* when configured). -- **No database I/O.** Compilation never connects to a database (it still - depends on the SQLAlchemy library for expression objects). -- **Plugins for dialects.** Portable core; dialect-specific operators live in - versioned plugins. `dialect=` is a hint only — it does not load plugins. +- **Compile predicates, not queries.** SQLRules returns a `CompiledRules` + result; applications compose its root predicate into SQLAlchemy statements. +- **Type annotations matter.** A scalar annotation adds a row type rule. Lax + coercion follows the SQLRules profile, and strict mode checks the observable + database type. +- **Fail rather than guess.** Known row mismatches are false. Missing backend + capabilities and unsupported retained rules raise errors. +- **Make complements total.** `where()` selects matching rows and `notwhere()` + selects rows that fail the complete rule set, including NULLs and failed + conversions. +- **No database I/O.** Compilation builds SQLAlchemy expressions and does not + connect to a database or execute statements. +- **Select providers explicitly.** Dialect packages implement source + preparation and constraints. `dialect=` does not load a backend. ## What SQLRules is not -See [NON_GOALS](../NON_GOALS.md). Applications remain responsible for composing -queries, choosing dialects, and executing SQL through SQLAlchemy. +It is not an ORM, request-value filter, general query builder, SQL string +generator, migration tool, or database client. Applications remain responsible +for composing queries and executing them through SQLAlchemy. -## When *not* to use it +## When not to use it -Skip SQLRules when: +- You need request values to become SQL equality predicates. +- You have a small number of static filters and do not need shared model rules. +- You need a dialect feature that is not in the support matrix. +- You expect the compiler to discover a provider from an engine or dialect. -- You need **request/instance values** as WHERE predicates. -- You have a couple of static filters and will never share constraint - metadata with a Pydantic model — write SQLAlchemy expressions directly. -- You need **portable regex** with core alone (`pattern` needs a plugin). -- You expect `dialect=` to load plugins automatically. -- You want a general-purpose query builder or SQL string generator. - -SQLRules pays off when the model is the shared source of truth across many -fields or dialects. - -## Deeper reading - -- [VISION](../VISION.md) · [PHILOSOPHY](../PHILOSOPHY.md) · [NON_GOALS](../NON_GOALS.md) -- [DESIGN_DECISIONS](../DESIGN_DECISIONS.md) (internals) -- [ARCHITECTURE](../ARCHITECTURE.md) +See [VISION](../VISION.md), [DESIGN_DECISIONS](../DESIGN_DECISIONS.md), and +[ARCHITECTURE](../ARCHITECTURE.md). diff --git a/docs/guides/examples.md b/docs/guides/examples.md index 19bce28..ae38cb3 100644 --- a/docs/guides/examples.md +++ b/docs/guides/examples.md @@ -7,11 +7,11 @@ directory. ## Run them ```bash -pip install "sqlrules>=1,<2" +pip install "sqlrules>=2,<3" "sqlrules-sqlite>=2,<3" python examples/basic_compile.py python examples/select_usage.py -pip install "sqlrules[postgresql]" +pip install "sqlrules>=2,<3" "sqlrules-postgresql>=2,<3" python examples/postgresql_pattern.py ``` diff --git a/docs/guides/faq.md b/docs/guides/faq.md index 0a569ee..cc09893 100644 --- a/docs/guides/faq.md +++ b/docs/guides/faq.md @@ -1,75 +1,60 @@ # FAQ -## What does SQLRules actually return? +## What does SQLRules return? -A dictionary mapping **Python field names** to lists of SQLAlchemy boolean -`ColumnElement`s. Use `sqlrules.where(rules)` (alias `flatten`) to pass them -to `.where(...)`. +Compiler.compile() returns CompiledRules, including the total root predicate, +ordered field results, compile diagnostics, and a structured explain plan. +where(compiled) and notwhere(compiled) return one-item lists for spread-style +SQLAlchemy WHERE calls. -## Does it validate request data or apply instance values? +## Does SQLRules validate request data? -No. Pydantic still validates inputs. SQLRules only reads **constraint -metadata** (`Field(ge=18)`, `min_length`, markers, …) and compiles -expressions. Runtime values such as `UserFilter(age=25)` are **not** turned -into `column == 25`. +RuleSchema is a normal Pydantic model and supports model_validate(), +model_dump(), and FastAPI integration. SQLRules compilation uses its field +declarations to query database rows; it does not turn an instance value such +as age=25 into a database equality predicate. -## Why doesn’t `pattern` work out of the box? +## How do I use a full Pydantic model? -`pattern` is extracted into IR as `PatternSpec`, but there is no portable -regex operator across databases. Install a dialect plugin or register a -custom translator. See [CONSTRAINTS](../CONSTRAINTS.md) and -[PLUGIN_SYSTEM](../PLUGIN_SYSTEM.md). +Call sqlrules.integrations.pydantic.from_pydantic(). It returns a generated +RuleSchema and a report describing unsupported Python-only behavior that was +removed or changed. Direct compile of an unrestricted BaseModel is rejected. -SQL Server (`sqlrules-mssql`) does **not** register `pattern`. +## Why does a plain type annotation compile? -## How do I use JSON / array / full-text operators? +Every scalar annotation is a rule. Lax mode checks supported backend +conversions; strict mode checks the observable database logical type. +Optional fields allow SQL NULL. -Use markers from `sqlrules.markers` (for example `JsonContains`) in -`Annotated[...]`, then compile with a dialect plugin that registers those -operators. See [Use dialect markers](markers.md). +## Why do I need a backend plugin? -## What is `dialect=` on `Compiler`? +Backend providers establish source type evidence, safe coercions, dialect +capabilities, and the versioned support profile. Select exactly one provider +when compiling. The compiler does not inspect a connection or infer a dialect +from the SQLAlchemy expression. -A **hint** for translators—not automatic dialect detection and **not** plugin -loading. Pass `plugins=[...]` explicitly. See [API](../API.md). +## What is dialect= on Compiler? -## `compile` vs `Compiler`? +It is an optional assertion against the provider name. It does not select or +load a provider. -Module-level `sqlrules.compile` is the one-shot Application API (no plugins). -Use `Compiler(plugins=[...])` when you need plugins, a custom registry, or -two-phase `compile_model` / `bind`. +## How do I bind renamed columns? -Do not call `compile` / `bind` concurrently on the **same** `Compiler` -instance. +Use column_map keyed by the Python field name or SQLRules Field(column=...). +Pydantic aliases keep their usual runtime behavior but do not bind columns. -## How do I bind ORM models or renamed columns? +## Can notwhere() return every failing row? -Pass a Declarative class as `table`, or use `column_map={...}`. See -[ORM / column_map](orm-column-map.md). +Yes. SQLRules normalizes field and root predicates so they do not produce SQL +UNKNOWN. notwhere(compiled) returns the total complement of the complete +root predicate. -## Why did an unconstrained field disappear from the rules dict? +## Where are type and dialect limitations documented? -Fields with no supported constraints are omitted from the rules dict. Their -**types** must still be in the support matrix (whole-model rule). - -Pass `emit_type_checks=True` to emit `type_check` IR for supported scalar -annotations (requires a dialect plugin or custom translator to bind — same -footgun as `pattern`). - -## When should I clear the model IR cache? - -Call `sqlrules.clear_model_cache()` when creating many ephemeral models -(`pydantic.create_model`) so the process-wide Phase-1 cache does not grow -without bound. Use `cache=False` on `compile` / `Compiler` to skip caching -for a one-off compile. - -## Where is the stable API boundary? - -[API](../API.md) documents Application, Plugin, and Internal tiers. Semver -applies to Application and Plugin surfaces. See also -[Support & compatibility](../project/support.md). +See [TYPE_SUPPORT](../TYPE_SUPPORT.md), [DIALECT_SUPPORT](../DIALECT_SUPPORT.md), +and [PLUGIN_SYSTEM](../PLUGIN_SYSTEM.md). ## More help [Troubleshooting](troubleshooting.md) · [Errors](../ERRORS.md) · -[Start here](start-here.md) +[Getting started](getting-started.md) diff --git a/docs/guides/getting-started.md b/docs/guides/getting-started.md index 6ac1d38..2955f95 100644 --- a/docs/guides/getting-started.md +++ b/docs/guides/getting-started.md @@ -1,147 +1,112 @@ # Getting started -Compile constrained Pydantic **Field metadata** into SQLAlchemy WHERE -expressions in a few minutes. - -**Requires** Python {{ python_requires }}, Pydantic v2, and SQLAlchemy 2.x. - -```text -SQLRules does THIS: Field(ge=18) → column >= 18 -It does NOT do this: UserFilter(age=25) → column == 25 -``` - -Runtime / request values are never applied by the compiler. Pydantic still -validates inputs separately. +SQLRules compiles the supported fields in a RuleSchema into one total +SQLAlchemy predicate. RuleSchema is also a normal Pydantic v2 model. ## 1. Install -```bash -pip install "sqlrules>=1,<2" -# or: uv add "sqlrules>=1,<2" -``` - -Optional dialect plugins (regex, JSON, arrays, and related operators): +~~~bash +pip install "sqlrules>=2,<3" "sqlrules-postgresql>=2,<3" +~~~ -```bash -pip install "sqlrules-postgresql>=1,<2" # or sqlite / mysql / mssql -# or: pip install "sqlrules[postgresql]" / "sqlrules[dialects]" -``` +Choose the dialect package for the database you query. Core never discovers +or selects a backend from a connection. -## 2. Define a filter model and a table +## 2. Define a schema and table -```python +~~~python from typing import Annotated -from pydantic import BaseModel, Field +from pydantic import ConfigDict, Field, StringConstraints from sqlalchemy import Column, Integer, MetaData, String, Table -import sqlrules +from sqlrules import RuleSchema users = Table( "users", MetaData(), + Column("id", Integer), Column("age", Integer), Column("name", String), ) -class UserFilter(BaseModel): - age: Annotated[int, Field(ge=18, le=65)] - name: Annotated[str, Field(min_length=2)] -``` -## 3. Compile and apply +class UserRules(RuleSchema): + model_config = ConfigDict(strict=False) -```python -rules = sqlrules.compile(UserFilter, users) -# { -# "age": [users.c.age >= 18, users.c.age <= 65], -# "name": [length(users.c.name) >= 2], -# } + id: int + age: int | None = Field(ge=18, le=120) + name: Annotated[str, StringConstraints(min_length=2)] +~~~ -stmt = users.select().where(*sqlrules.where(rules)) -``` +The Pydantic constraints are imported directly. The scalar annotation itself +creates a database type rule, so id must match the selected backend's int +profile even without a Field constraint. -Prefer `sqlrules.where(rules)` (identical alias: `flatten`). +## 3. Validate and compile -**Success check:** `rules` is a `dict[str, list[...]]` keyed by Python field -names. Unconstrained fields are omitted. +~~~python +from sqlrules import Compiler, notwhere, where +from sqlrules_postgresql import PostgresPlugin -## 4. The `pattern` footgun (and fix) +request = UserRules.model_validate({"id": "42", "age": "21", "name": "Ada"}) +assert request.id == 42 -`Field(pattern=...)` is extracted into IR, but **core has no portable regex -translator**. This fails on purpose: +compiler = Compiler(plugins=[PostgresPlugin(server_version=(16, 0))]) +compiled = compiler.compile(UserRules, users) -```python -class NameFilter(BaseModel): - name: Annotated[str, Field(pattern=r"^A")] +matches = users.select().where(*where(compiled)) +fails = users.select().where(*notwhere(compiled)) +~~~ -sqlrules.compile(NameFilter, users) # UnsupportedConstraintError -``` +where() returns a one-item list containing the complete root predicate. +notwhere() returns the complement and selects every row that fails one or more +rules. SQLRules performs no database I/O. -Install a dialect plugin and pass it to `Compiler`: +## 4. Convert an existing Pydantic model -```python -from sqlrules import Compiler -from sqlrules_postgresql import PostgresPlugin +Use from_pydantic() for a Pydantic model that includes features outside the +SQLRules declaration subset: -compiler = Compiler(plugins=[PostgresPlugin()], dialect="postgresql") -rules = compiler.compile(NameFilter, users) -``` +~~~python +from pydantic import BaseModel, Field +from sqlrules.integrations.pydantic import from_pydantic -`dialect=` is a **hint only** — it does not load plugins. See -[`examples/postgresql_pattern.py`](https://github.com/eddiethedean/sqlrules/blob/main/examples/postgresql_pattern.py). -### Type checks are opt-in IR, not portable SQL +class ApiFilter(BaseModel): + age: int = Field(ge=18) -`emit_type_checks=True` adds `type_check` constraints for supported scalars. -Core still has **no** portable translator — without a dialect plugin (or a -custom `type_check` translator), compilation raises -`UnsupportedConstraintError`, just like `pattern`. Unconstrained fields still -must use a supported annotation even when they produce no rules. -## 5. What compiled (and what did not) +conversion = from_pydantic(ApiFilter, on_incompatible="warn") +RulesModel = conversion.model +print(conversion.report) +~~~ -| Constraint | Core behavior | -|---|---| -| `gt` / `ge` / `lt` / `le` | Comparison on the bound column | -| `multiple_of` | Modulo equality | -| `min_length` / `max_length` | `func.length(column)` | -| `Literal[...]` / `Enum` | `column.in_(...)` | -| `pattern` | Extracted to IR only — needs a dialect plugin or custom translator | +The converter strips unsupported behavior according to the selected policy +and reports changed, unknown, or dropped semantics. It never runs validators, +serializers, or default factories to infer SQL. -Unsupported **operators** raise by default. Use -`on_unsupported="warn"` or `"ignore"` to skip unknown operators. Unsupported -**types** always raise — including on unconstrained fields (whole-model type -matrix). Unconstrained fields with **supported** types are simply omitted. +## 5. Patterns and markers -Untrusted `Field(pattern=...)` values are a CPU/ReDoS cost risk once a dialect -plugin translates them; prefer static patterns. See -[SECURITY](../SECURITY.md). +Patterns and JSON/array/range operators need a plugin for the selected +database. Select one backend provider: -## When *not* to use SQLRules +~~~python +compiler = Compiler(plugins=[PostgresPlugin(server_version=(16, 0))]) +compiled = compiler.compile(NameRules, users) +~~~ -Skip SQLRules when you need request/instance values as WHERE predicates, a -general query builder, portable regex without a plugin, or when -`dialect=` alone should load plugins (it does not). Two static filters you -will never share with a Pydantic model? Write the SQLAlchemy expressions by -hand. See [design philosophy](design-philosophy.md). +SQLite text coercion and pattern matching use REGEXP, and string length rules +use a Python-compatible Unicode code-point counter. Register +`sqlrules_sqlite.register_sqlite_functions()` on each SQLite connection when +those rules are used. The +[support matrix](../TYPE_SUPPORT.md) describes source types, conversions, +server versions, and compile-time capability errors. ## Next steps -- [Examples](examples.md) — runnable scripts -- [Upgrade from 0.x](upgrade-0x.md) — pin and plugin notes -- [ORM / `column_map`](orm-column-map.md) — DeclarativeBase and aliases -- [Markers](markers.md) — JSON / array / full-text with plugins -- [Examples on GitHub](https://github.com/eddiethedean/sqlrules/tree/main/examples) -- [Constraint map](../CONSTRAINTS.md) — full operator → expression table -- [Plugin system](../PLUGIN_SYSTEM.md) — dialect packages and custom translators -- [Public API](../API.md) — `Compiler`, two-phase compile, API tiers -- [FAQ](faq.md) · [Troubleshooting](troubleshooting.md) - -:::{admonition} Not a query builder -:class: note - -SQLRules does not connect to a database, generate SQL strings, or validate -request payloads at runtime. It only compiles supported constraints into -expressions. -::: +- [Migration from 1.x](upgrade-1x.md) +- [Dialect markers](markers.md) +- [ORM and column mapping](orm-column-map.md) +- [Plugin API v2](../PLUGIN_SYSTEM.md) +- [Troubleshooting](troubleshooting.md) diff --git a/docs/guides/markers.md b/docs/guides/markers.md index 4003f77..41bf4fd 100644 --- a/docs/guides/markers.md +++ b/docs/guides/markers.md @@ -1,66 +1,68 @@ -# Use dialect markers (JSON, arrays, full-text) +# Use dialect markers -Markers in `sqlrules.markers` (also re-exported from `sqlrules`) attach -dialect-specific operators to a field. Core extracts them into IR; a dialect -**plugin** must register translators. +Markers in sqlrules.markers attach a database-specific operator to a +RuleSchema field. The selected backend must support both the marker and the +bound SQL column type. -## Install and register a plugin +## Select a plugin -```bash -pip install "sqlrules-postgresql>=1,<2" -# or from this repo: make install -``` +~~~bash +pip install "sqlrules>=2,<3" "sqlrules-postgresql>=2,<3" +~~~ -```python +~~~python from typing import Annotated, Any -from pydantic import BaseModel from sqlalchemy import Column, MetaData, Table from sqlalchemy.dialects.postgresql import JSONB -import sqlrules -from sqlrules import Compiler, JsonContains +from sqlrules import Compiler, JsonContains, RuleSchema, where from sqlrules_postgresql import PostgresPlugin rows = Table("rows", MetaData(), Column("meta", JSONB)) -class RowFilter(BaseModel): + +class RowRules(RuleSchema): meta: Annotated[dict[str, Any], JsonContains({"active": True})] -compiler = Compiler(plugins=[PostgresPlugin()], dialect="postgresql") -rules = compiler.compile(RowFilter, rows) -stmt = rows.select().where(*sqlrules.where(rules)) -``` -`dialect=` is a **hint** for translators — it does not load plugins. +compiler = Compiler(plugins=[PostgresPlugin(server_version=(16, 0))]) +compiled = compiler.compile(RowRules, rows) +statement = rows.select().where(*where(compiled)) +~~~ ## Common markers | Marker | Typical plugin support | |---|---| -| `JsonContains` / `JsonHasKey` | PostgreSQL, SQLite, MySQL, MSSQL (JSON helpers) | -| `ArrayContains` / `ArrayOverlap` | PostgreSQL | -| `RangeContains` / `RangeOverlap` | PostgreSQL | -| `FullTextMatch` | MySQL | +| JsonContains / JsonHasKey | PostgreSQL, SQLite, MySQL, SQL Server JSON helpers | +| ArrayContains / ArrayOverlap | PostgreSQL | +| RangeContains / RangeOverlap | PostgreSQL | +| FullTextMatch | MySQL | -See [DIALECT_SUPPORT](../DIALECT_SUPPORT.md) for the full matrix. +List and dict fields require a compatible marker. Markers express database +operators; they do not validate nested Python items or JSON schemas. -## Pattern + markers together +## Patterns and markers -```python +~~~python +from typing import Annotated, Any from pydantic import Field -from sqlrules import JsonContains +from sqlrules import JsonContains, RuleSchema + -class RowFilter(BaseModel): - name: Annotated[str, Field(pattern=r"^A")] +class RowRules(RuleSchema): + name: Annotated[str, Field(pattern="^A")] meta: Annotated[dict[str, Any], JsonContains({"active": True})] -``` +~~~ -Both need a plugin that registers `pattern` and the marker operators -(PostgreSQL does). SQL Server (`sqlrules-mssql`) does **not** register -`pattern`. +The selected provider must register both pattern and JSON operations. +PostgreSQL does; SQL Server does not register pattern. -## Security note +SQLite emits REGEXP for pattern and selected text coercions. String length +constraints use a registered Python-compatible code-point counter. Register +`sqlrules_sqlite.register_sqlite_functions()` on each SQLite connection when +those expressions are used. The older `register_regexp()` helper remains an +alias. -Prefer static marker payloads and static `Field(pattern=...)` values. -Untrusted regex can be expensive once translated — see [SECURITY](../SECURITY.md). +Prefer static marker payloads and pattern values. See [SECURITY](../SECURITY.md). diff --git a/docs/guides/orm-column-map.md b/docs/guides/orm-column-map.md index 8e31485..b8750fc 100644 --- a/docs/guides/orm-column-map.md +++ b/docs/guides/orm-column-map.md @@ -1,78 +1,58 @@ -# Bind ORM models and `column_map` +# ORM and column mapping -SQLRules binds each constrained field to a SQLAlchemy column. By default it -looks up columns on the object you pass as `table` (a `Table`, selectable, or -ORM class/entity). Use `column_map` when names differ or binding is ambiguous. +Every RuleSchema field binds to a database column. The binding precedence is: -## Table (default) +1. Explicit column_map entry keyed by the Python field name. +2. SQLRules Field(column=...) metadata. +3. A column whose name matches the Python field. -```python -from typing import Annotated +Pydantic aliases affect validation and serialization, but never choose a +database column automatically. -from pydantic import BaseModel, Field -from sqlalchemy import Column, Integer, MetaData, String, Table +## Explicit column_map -import sqlrules +~~~python +from sqlrules import Compiler, RuleSchema +from sqlrules_postgresql import PostgresPlugin -users = Table( - "users", - MetaData(), - Column("age", Integer), - Column("full_name", String), -) - -class UserFilter(BaseModel): - age: Annotated[int, Field(ge=18)] - name: Annotated[str, Field(min_length=1, alias="full_name")] - -# String Field aliases are tried before the Python field name. -rules = sqlrules.compile(UserFilter, users) -``` -## Explicit `column_map` +class UserRules(RuleSchema): + display_name: str + minimum_age: int -Pass a map from **Python field name or alias string** to a column element when -automatic lookup fails (joined tables, renamed columns, hybrid attributes): -```python -rules = sqlrules.compile( - UserFilter, +compiled = Compiler(plugins=[PostgresPlugin(server_version=(16, 0))]).compile( + UserRules, users, column_map={ - "age": users.c.age, - "full_name": users.c.full_name, # alias used on the Field + "display_name": User.display_name, + "minimum_age": User.age, }, ) -``` +~~~ -## Declarative ORM +Values in column_map must be SQLAlchemy expressions or ORM attributes that +expose a SQL expression. Each expression still needs source type evidence from +the selected provider. -```python -from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column +## Field(column=...) -class Base(DeclarativeBase): - pass +~~~python +from sqlrules import Field, RuleSchema -class User(Base): - __tablename__ = "users" - id: Mapped[int] = mapped_column(primary_key=True) - age: Mapped[int] - name: Mapped[str] -class UserFilter(BaseModel): - age: Annotated[int, Field(ge=18)] - name: Annotated[str, Field(min_length=2)] +class UserRules(RuleSchema): + display_name: str = Field(column="name") +~~~ -rules = sqlrules.compile(UserFilter, User) -stmt = User.__table__.select().where(*sqlrules.where(rules)) -``` +The custom Field wrapper retains all standard Pydantic Field behavior. The +column name is separate from Pydantic aliases and serialized field names. -If an ORM attribute is not a real column (relationship, hybrid), pass that -field through `column_map` to the underlying column, or you will get -`MissingColumnError`. +## ORM models -## Tips +Mapped ORM attributes can be passed directly as the table argument when the +model exposes SQLAlchemy attributes with matching names. Use column_map when a +rule field intentionally refers to a different mapped attribute. -- Rule dict keys are always **Python field names**, not aliases. -- Unconstrained supported-type fields are omitted and do not need a column. -- See [Troubleshooting](troubleshooting.md) for `MissingColumnError`. +Missing bindings raise MissingColumnError during compilation, including for +type-only annotations with no value constraints. diff --git a/docs/guides/start-here.md b/docs/guides/start-here.md index 2570704..d712451 100644 --- a/docs/guides/start-here.md +++ b/docs/guides/start-here.md @@ -3,13 +3,13 @@ Pick the path that matches what you need. Each link is a single next step—not the full documentation map. -## I want to compile constraints from a Pydantic model +## I want to compile SQL-compatible Pydantic rules -[Getting started](getting-started.md) — install `sqlrules`, compile one model, -and attach the rules to a SQLAlchemy `select()`. +[Getting started](getting-started.md) — define a `RuleSchema`, select one +backend provider, and attach its predicate to a SQLAlchemy `select()`. -Remember: SQLRules compiles **Field metadata** (`ge=18`), not instance values -(`UserFilter(age=25)`). +For an unrestricted application model, use `from_pydantic()` and review its +conversion report before compiling the generated rules class. ## I need dialect-specific operators (regex, JSON, arrays) diff --git a/docs/guides/troubleshooting.md b/docs/guides/troubleshooting.md index 91ef6ab..53cacc2 100644 --- a/docs/guides/troubleshooting.md +++ b/docs/guides/troubleshooting.md @@ -1,67 +1,58 @@ # Troubleshooting -## `UnsupportedConstraintError` +## RuleSchema declaration errors -An operator or type has no translator. +RuleSchema checks supported types and metadata during class creation. Remove +custom validators, serializers, computed fields, unsupported Pydantic options, +general unions, or unsupported type/constraint combinations. To use a full +Pydantic model that contains those features, convert it with from_pydantic() +and inspect its report. -- Check [CONSTRAINTS](../CONSTRAINTS.md) and [TYPE_SUPPORT](../TYPE_SUPPORT.md). -- For `pattern` / dialect markers, install a plugin and pass - `Compiler(plugins=[...])` (or register a translator). -- For unknown **operators** only, try `on_unsupported="warn"` or `"ignore"`. - Unsupported **types** always raise. -- `emit_type_checks=True` without a `type_check` translator raises the same - way as bare `pattern` — install a dialect plugin or register a translator. +## CapabilityError -## `pattern` still unsupported with `sqlrules-mssql` +The selected backend cannot prove the bound column's logical type or safely +implement the requested conversion. Check the source SQLAlchemy type, +configured server_version, SQLite storage class, and collation requirements +in [TYPE_SUPPORT](../TYPE_SUPPORT.md). A known row mismatch is a non-match; +an unavailable backend capability is a compile-time error. -Expected: the MSSQL plugin does **not** register `pattern`. Use PostgreSQL / -SQLite / MySQL plugins, a custom translator, or drop the constraint. +## MissingColumnError -## `pattern` compiles but SQLite queries fail / never match +Every field, including a type-only field, needs a bound SQLAlchemy column. +Use column_map keyed by the Python field name, Field(column="..."), or a +matching database column. -`sqlrules-sqlite` emits `REGEXP`. Call -`sqlrules_sqlite.register_regexp(connection)` on each connection (or via a -SQLAlchemy `connect` event). Until then, SQLite has no REGEXP implementation. -See [DIALECT_SUPPORT](../DIALECT_SUPPORT.md) and [SECURITY](../SECURITY.md) -(Python `re` runs in-process). +## Pattern unsupported on SQL Server -## `MissingColumnError` +Expected: sqlrules-mssql does not register pattern. Select a provider that +supports the pattern semantics or remove the pattern declaration. -A constrained field could not bind to a column. +## SQLite REGEXP errors -```python -rules = sqlrules.compile( - Model, - table, - column_map={"field_or_alias": table.c.actual_column}, -) -``` - -- Confirm the table/ORM attribute is a real column (not `Table.name`, etc.). -- String aliases on `Field` are tried before the Python field name. -- Full walkthrough: [ORM / column_map](orm-column-map.md). +SQLite does not provide REGEXP by default, and its built-in `length(TEXT)` +stops at an embedded NUL. Register `sqlrules_sqlite.register_sqlite_functions()` +on each connection when patterns, textual coercions, or string length +constraints are used. `register_regexp()` remains a backward-compatible alias. ## Plugin registration conflicts -Two plugins (or a plugin and a builtin) claim the same operator. - -- Set `on_conflict="replace"` or `"ignore"` on `Compiler` / `register_constraint`. -- Default is `"raise"`. See [PLUGIN_SYSTEM](../PLUGIN_SYSTEM.md). +Two translators claim one operator. Set on_conflict="replace" or "ignore" on +Compiler or the individual register_constraint() call. Exactly one backend +provider must be selected. -## Plugin API version mismatch +## Plugin API mismatch -`api_version` must equal `sqlrules.PLUGIN_API_VERSION` (`"1"`). Upgrade the -plugin or core so they match. +api_version must equal sqlrules.PLUGIN_API_VERSION (currently 2). API v1 +plugins need a backend preparation hook and capability report before they can +be used with 2.0. -## Rules look wrong / empty +## Rows match unexpectedly -- Unconstrained fields are omitted. -- Confirm you are inspecting the returned dict, not re-validating the model - or expecting instance values in the WHERE clause. -- For two-phase use, call `bind` after `compile_model`. -- `dialect=` alone never loads plugins — pass `plugins=[...]`. +Inspect compiled.explain() for the bound columns, logical types, coercion +profile, and backend assumptions. Use notwhere(compiled) to query failing +rows. Defaults and values on a Python model instance never become SQL +predicates. -## Still stuck? +## More help -- [FAQ](faq.md) · [ERRORS](../ERRORS.md) · [SECURITY](../SECURITY.md) -- Open an issue: https://github.com/eddiethedean/sqlrules/issues +[FAQ](faq.md) · [Errors](../ERRORS.md) · [Support matrix](../TYPE_SUPPORT.md) diff --git a/docs/guides/upgrade-1x.md b/docs/guides/upgrade-1x.md new file mode 100644 index 0000000..babf090 --- /dev/null +++ b/docs/guides/upgrade-1x.md @@ -0,0 +1,93 @@ +# Upgrade from SQLRules 1.x + +SQLRules 2.0 changes the model declaration, compiler result, and plugin +contracts. Core and all four official dialect packages must use the same 2.x +line. + +## Install matching 2.x packages + +~~~bash +pip install "sqlrules>=2,<3" "sqlrules-postgresql>=2,<3" +~~~ + +## Use RuleSchema or convert a Pydantic model + +Replace a model compiled directly in 1.x: + +~~~python +from sqlrules import RuleSchema + + +class UserRules(RuleSchema): + age: int +~~~ + +An unrestricted application model remains a normal Pydantic model, but it +must be converted before compilation: + +~~~python +from sqlrules.integrations.pydantic import from_pydantic + +conversion = from_pydantic(ApiFilter, on_incompatible="warn") +RulesModel = conversion.model +print(conversion.report) +~~~ + +The generated class is still a Pydantic model. Review the conversion report +for validators, serializers, fields, metadata, or behavior that was removed or +changed. + +## Select a backend explicitly + +The dialect string hint no longer selects a provider. Supply one backend +plugin: + +~~~python +compiler = Compiler(plugins=[PostgresPlugin(server_version=(16, 0))]) +compiled = compiler.compile(UserRules, users) +~~~ + +The server version is required for version-dependent coercions. Core still +performs no connection discovery or database I/O. + +## Consume CompiledRules + +Version 1 returned a field-to-expression dictionary. Version 2 returns one +complete predicate with field details and an explain plan: + +~~~python +compiled = compiler.compile(UserRules, users) +matching = users.select().where(*sqlrules.where(compiled)) +failing = users.select().where(*sqlrules.notwhere(compiled)) +~~~ + +where() and flatten() keep the list return shape and now return a one-element +list containing the complete root predicate. notwhere() returns its total +complement. A lone scalar annotation now checks the bound column type even +without a Field constraint. + +## Strictness and unsupported rules + +Annotations now always create type rules. Lax mode applies the published +SQLRules conversion table; field strictness, strict aliases, and model +ConfigDict(strict=True) require matching observable database types. Optional +fields explicitly allow SQL NULL. + +The 1.x on_unsupported warn/ignore options could omit a constraint. Version 2 +requires retained constraints to translate or raises a capability error. +Convert unsupported Python-only features before compilation and inspect the +report instead. + +## Column binding + +Aliases no longer choose database columns implicitly. Use column_map keyed by +the Python field name or SQLRules Field(column="database_name"). Pydantic +aliases keep their normal validation and serialization behavior. + +## Plugin API + +Plugin API v2 adds source preparation and capability reporting. Existing API +v1 translators must be adapted so they operate on a prepared SQL expression. +Backend providers must report supported source types, server versions, and +storage assumptions. See [PLUGIN_SYSTEM](../PLUGIN_SYSTEM.md) and the +[support matrix](../TYPE_SUPPORT.md). diff --git a/docs/index.md b/docs/index.md index e6d1045..18c76fb 100644 --- a/docs/index.md +++ b/docs/index.md @@ -7,9 +7,9 @@ Pydantic → SQLAlchemy Python {{ python_requires }} -

Constraint metadata → WHERE expressions

-

Compile constraints into WHERE rules

-

SQLRules turns a safe subset of Pydantic field constraints into deterministic SQLAlchemy expressions—no database connection, plugins for dialect operators. It compiles Field metadata, not model instance values.

+

Pydantic RuleSchema → SQLAlchemy predicates

+

Compile rules into total predicates

+

SQLRules validates RuleSchema models with Pydantic, then compiles their supported declarations through one explicit backend. It performs no database I/O and returns both matching and failing-row predicates.

Start here → Getting started @@ -26,7 +26,7 @@ Pick the path that matches how you work: :link: guides/getting-started :link-type: doc -**Compile constraints from models.** Install `sqlrules`, map a Pydantic model to a table, and pass `where(rules)` into SQLAlchemy. +**Compile supported schemas.** Define a RuleSchema, select a backend provider, and pass where(compiled) or notwhere(compiled) into SQLAlchemy. +++ **Open getting started →** @@ -72,16 +72,16 @@ Pick the path that matches how you work: :::{admonition} Not a query builder :class: note -SQLRules does not connect to a database, generate SQL strings, or turn -validated request payloads into `WHERE` predicates. It only compiles -supported **constraint metadata** into expressions. See +SQLRules does not connect to a database or generate SQL strings. RuleSchema +is still a normal Pydantic model, but the compiler uses the schema's rules to +select matching database rows. See [design philosophy](guides/design-philosophy.md). ::: ```{raw} html

- Install: pip install "sqlrules>=1,<2" - — optional dialects via pip install "sqlrules-postgresql>=1,<2" (or sqlite / mysql / mssql). + Install: pip install "sqlrules>=2,<3" + — add the matching dialect package from the same 2.x line. pattern needs a dialect plugin.
``` @@ -112,6 +112,7 @@ guides/examples guides/orm-column-map guides/markers guides/upgrade-0x +guides/upgrade-1x guides/faq guides/troubleshooting ``` diff --git a/docs/project/support.md b/docs/project/support.md index 113c95a..a9d1671 100644 --- a/docs/project/support.md +++ b/docs/project/support.md @@ -1,55 +1,48 @@ # Support and compatibility -## License - -SQLRules is released under the [MIT License](https://github.com/eddiethedean/sqlrules/blob/main/LICENSE). - ## Support posture -Community support is **best-effort** via -[GitHub Issues](https://github.com/eddiethedean/sqlrules/issues). There is no -commercial SLA, paid support contract, or guaranteed response time. - -Before opening an issue, check the [FAQ](../guides/faq.md) and -[Troubleshooting](../guides/troubleshooting.md). +Community support is best-effort through +[GitHub Issues](https://github.com/eddiethedean/sqlrules/issues). Check the +[FAQ](../guides/faq.md) and [Troubleshooting](../guides/troubleshooting.md) +before opening an issue. ## Compatibility and pinning | Surface | Stability | |---|---| -| Application API (`compile`, `where`, `Compiler`, …) | Semver — stable in 1.x | -| Plugin API (`PLUGIN_API_VERSION`, IR, registry) | Semver + exact `api_version` match | +| Application API (`RuleSchema`, `Compiler`, `where`, `notwhere`, conversion) | Semver — 2.x contract | +| Plugin API (`PLUGIN_API_VERSION`, provider hooks, IR, registry) | Exact `api_version` match | | Internal modules | Unstable — may change without notice | -Recommended pins for applications: +Install core and a matching official provider from the same major line: ```bash -pip install "sqlrules>=1,<2" -pip install "sqlrules-postgresql>=1,<2" # if needed +pip install "sqlrules>=2,<3" "sqlrules-postgresql>=2,<3" ``` -Official dialect plugins are released in **lockstep** with core on the same -major line. See [API tiers](../API.md) and [Security](../SECURITY.md). - -## Security reports - -Please open a -[GitHub security advisory](https://github.com/eddiethedean/sqlrules/security/advisories/new) -for vulnerabilities. Do not file public issues for undisclosed security -problems. See [SECURITY](../SECURITY.md) for the trust model (no DB I/O; -plugins are arbitrary code; pattern/ReDoS cost). +The four official providers are released in lockstep with core. See [API +tiers](../API.md) and [Security](../SECURITY.md). ## Supported versions | Version line | Status | |---|---| -| 1.x | Current stable | -| 0.x | Historical — upgrade to 1.x | +| 2.x | Current implementation and release target | +| 1.x | Previous stable line; use the 1.x to 2.x migration guide | +| 0.x | Historical | + +The semantic profile uses Pydantic 2.13.4 as its conformance reference while +the runtime dependency accepts Pydantic v2. SQLAlchemy 2.x and Python 3.10+ +are supported. + +## Security reports -Security fixes target the current 1.x line. Older 0.x releases are not -actively maintained. +Use a [GitHub security advisory](https://github.com/eddiethedean/sqlrules/security/advisories/new) +for vulnerabilities. Do not file public issues for undisclosed problems. See +[SECURITY](../SECURITY.md) for the trust model. ## Code of conduct -Participation is governed by the +Participation follows the project's [Code of Conduct](https://github.com/eddiethedean/sqlrules/blob/main/CODE_OF_CONDUCT.md). diff --git a/docs/reference/application.md b/docs/reference/application.md index efd3b62..144873f 100644 --- a/docs/reference/application.md +++ b/docs/reference/application.md @@ -7,7 +7,7 @@ Generated members most application code needs. For narrative docs, see .. automodule:: sqlrules :members: compile, where, flatten, clear_model_cache, Compiler, UnsupportedConstraintError, MissingColumnError, InvalidModelError, - ConfigurationError, SQLRulesError, SQLRulesWarning, + ConfigurationError, SQLRulesError, JsonContains, JsonHasKey, ArrayContains, ArrayOverlap, RangeContains, RangeOverlap, FullTextMatch, ConstraintMarker, __version__ diff --git a/docs/reference/glossary.md b/docs/reference/glossary.md index 413194e..4c59a19 100644 --- a/docs/reference/glossary.md +++ b/docs/reference/glossary.md @@ -2,21 +2,24 @@ | Term | Meaning | |---|---| -| **Application API** | Stable surface for apps: `compile`, `where`/`flatten`, `Compiler`, markers, exceptions. | -| **Plugin API** | Stable extension surface: `SQLRulesPlugin`, `TranslatorRegistry`, `PLUGIN_API_VERSION`, `pattern_text`. | +| **Application API** | Stable surface for apps: `RuleSchema`, `Field`, `RuleConfig`, `compile`, `where`/`notwhere`, `Compiler`, markers, and exceptions. | +| **Plugin API** | Stable extension surface: `SQLRulesPlugin`, `BackendProvider`, `TranslatorRegistry`, `PLUGIN_API_VERSION`, and translator IR types. | | **Internal API** | IR builders and helpers that may change without notice. | -| **IR** | Intermediate representation of a model’s fields and constraints (`ModelIR`, `FieldIR`, `Constraint`). | +| **IR** | Immutable schema, prepared-value, and compiled-result data used between normalization, backend preparation, and translation. | +| **`RuleSchema`** | A Pydantic `BaseModel` whose field declarations are restricted to the SQLRules-compilable subset. It remains usable for normal Pydantic validation and integrations. | +| **`from_pydantic()`** | Converter that creates a RuleSchema-compatible Pydantic model and returns a report of dropped or changed semantics. | +| **`BackendProvider`** | Dialect implementation that reports capabilities and prepares safe SQL expressions from source columns. | +| **`CompiledRules`** | Complete total predicate, per-field result metadata, diagnostics, backend profile, and conversion provenance. | | **`PatternSpec`** | Structured IR value for `pattern` constraints (flags + text). Use `pattern_text()` in translators. | | **Marker** | `Annotated` metadata object (`ConstraintMarker`) for dialect operators such as `JsonContains`. | | **Translator** | Function that turns a `Constraint` + column + context into a SQLAlchemy expression. | | **Registry** | `TranslatorRegistry` mapping operators (and optionally types) to translators. | | **Plugin** | Object with `name`, `api_version`, and `register(registry)` that installs translators. | -| **`Compiler`** | Reusable compiler with plugins, conflict policy, cache, and optional `dialect` hint. | -| **`compile_model` / `bind`** | Two-phase API: build IR once, bind to a table (or alias) later. | -| **`on_unsupported`** | Policy for unknown **operators**: `raise`, `warn`, or `ignore`. Does not apply to unsupported types. | +| **`Compiler`** | Reusable compiler with plugins, conflict policy, and optional backend-name check. | +| **`compile_model` / `bind`** | Two-phase API: normalize a RuleSchema, then bind it to a table or alias. | +| **`on_unsupported`** | Compatibility constructor option. SQLRules 2.0 requires `raise`; retained rules cannot be skipped. | | **`on_conflict`** | Policy when registering over an existing operator: `raise`, `replace`, or `ignore`. | -| **`dialect`** | Optional string hint for translators—not automatic engine detection. | -| **Rules dict** | `dict[str, list[ColumnElement[bool]]]` keyed by Python field names. | +| **`notwhere()`** | Returns a one-element predicate list selecting rows that fail the complete compiled rule. | See also [API](../API.md), [IR_CONTRACT](../IR_CONTRACT.md), and [INTERNAL_API](../INTERNAL_API.md). diff --git a/docs/reference/plugin-api.md b/docs/reference/plugin-api.md index 8864c55..dea7a5b 100644 --- a/docs/reference/plugin-api.md +++ b/docs/reference/plugin-api.md @@ -7,9 +7,10 @@ Symbols for dialect plugins and advanced two-phase compile. Prefer ```{eval-rst} .. automodule:: sqlrules :no-index: - :members: PLUGIN_API_VERSION, SQLRulesPlugin, TranslatorRegistry, - default_registry, pattern_text, type_spec, CompilationContext, Constraint, - PatternSpec, TypeSpec, ModelIR, FieldIR, FieldDescriptor, Diagnostic, + :members: PLUGIN_API_VERSION, SQLRulesPlugin, BackendProvider, + TranslatorRegistry, default_registry, pattern_text, CompilationContext, + Constraint, PatternSpec, PreparedValue, RuleField, SchemaSpec, + CompiledRules, FieldResult, FieldDescriptor, Diagnostic, PluginError, RegistryError, TranslatorError, InvalidTranslatorError, InternalCompilerError :imported-members: diff --git a/examples/README.md b/examples/README.md index 802a54b..80037da 100644 --- a/examples/README.md +++ b/examples/README.md @@ -5,11 +5,11 @@ Runnable scripts for SQLRules. ## From PyPI ```bash -pip install "sqlrules>=1,<2" +pip install "sqlrules>=2,<3" "sqlrules-sqlite>=2,<3" python examples/basic_compile.py python examples/select_usage.py -pip install "sqlrules[postgresql]" +pip install "sqlrules-postgresql>=2,<3" python examples/postgresql_pattern.py ``` @@ -22,8 +22,8 @@ make examples | Script | Shows | Expected output (shape) | |---|---|---| -| `basic_compile.py` | Core `compile` + `where` | `fields: ['age', 'name']` then a `SELECT` | +| `basic_compile.py` | RuleSchema + SQLite provider + `where` | field names then a `SELECT` | | `select_usage.py` | `Literal` + compiled `select()` | printed `SELECT` with `IN` / comparisons | -| `postgresql_pattern.py` | Why `pattern` needs a plugin, then Postgres fix | first compile raises; second prints rules | +| `postgresql_pattern.py` | PostgreSQL pattern and JSON markers | prints a `SELECT` with dialect operators | Application users should install from **PyPI**, not from `packages/`. diff --git a/examples/basic_compile.py b/examples/basic_compile.py index 559433c..aa80fe6 100644 --- a/examples/basic_compile.py +++ b/examples/basic_compile.py @@ -9,10 +9,12 @@ from typing import Annotated -from pydantic import BaseModel, Field +from pydantic import Field from sqlalchemy import Column, Integer, MetaData, String, Table +from sqlrules_sqlite import SQLitePlugin import sqlrules +from sqlrules import Compiler, RuleSchema users = Table( "users", @@ -22,15 +24,16 @@ ) -class UserFilter(BaseModel): +class UserFilter(RuleSchema): age: Annotated[int, Field(ge=18, le=65)] name: Annotated[str, Field(min_length=2)] def main() -> None: - rules = sqlrules.compile(UserFilter, users) - stmt = users.select().where(*sqlrules.where(rules)) - print("fields:", sorted(rules)) + compiler = Compiler(plugins=[SQLitePlugin()]) + compiled = compiler.compile(UserFilter, users) + stmt = users.select().where(*sqlrules.where(compiled)) + print("fields:", [field.name for field in compiled.fields]) print(stmt) diff --git a/examples/postgresql_pattern.py b/examples/postgresql_pattern.py index f6c0c0d..1e3a232 100644 --- a/examples/postgresql_pattern.py +++ b/examples/postgresql_pattern.py @@ -13,13 +13,13 @@ from typing import Annotated, Any -from pydantic import BaseModel, Field +from pydantic import Field from sqlalchemy import Column, MetaData, String, Table from sqlalchemy.dialects.postgresql import JSONB from sqlrules_postgresql import PostgresPlugin import sqlrules -from sqlrules import Compiler, JsonContains, UnsupportedConstraintError +from sqlrules import Compiler, JsonContains, RuleSchema table = Table( "rows", @@ -29,23 +29,16 @@ ) -class RowFilter(BaseModel): +class RowFilter(RuleSchema): name: Annotated[str, Field(pattern=r"^A")] meta: Annotated[dict[str, Any], JsonContains({"active": True})] def main() -> None: - # Without a plugin, pattern raises (core has no portable regex). - try: - sqlrules.compile(RowFilter, table) - except UnsupportedConstraintError as exc: - print("core alone:", exc) - - # dialect= is a hint only — plugins must be passed explicitly. - compiler = Compiler(plugins=[PostgresPlugin()], dialect="postgresql") - rules = compiler.compile(RowFilter, table) - stmt = table.select().where(*sqlrules.where(rules)) - print("with plugin fields:", sorted(rules)) + compiler = Compiler(plugins=[PostgresPlugin(server_version=(16, 0))]) + compiled = compiler.compile(RowFilter, table) + stmt = table.select().where(*sqlrules.where(compiled)) + print("with plugin fields:", [field.name for field in compiled.fields]) print(stmt) diff --git a/examples/select_usage.py b/examples/select_usage.py index 06348d3..a3e6597 100644 --- a/examples/select_usage.py +++ b/examples/select_usage.py @@ -1,6 +1,6 @@ """Compile rules and show a SQLAlchemy select() string. -Run from the repo root after ``pip install "sqlrules>=1,<2"`` (or ``make install``): +Run from the repo root after installing SQLRules and its SQLite provider. python examples/select_usage.py """ @@ -9,10 +9,12 @@ from typing import Annotated, Literal -from pydantic import BaseModel, Field +from pydantic import Field from sqlalchemy import Column, Integer, MetaData, String, Table +from sqlrules_sqlite import SQLitePlugin import sqlrules +from sqlrules import Compiler, RuleSchema items = Table( "items", @@ -22,14 +24,15 @@ ) -class ItemFilter(BaseModel): +class ItemFilter(RuleSchema): qty: Annotated[int, Field(ge=1, le=100)] status: Literal["active", "pending"] def main() -> None: - rules = sqlrules.compile(ItemFilter, items) - stmt = items.select().where(*sqlrules.where(rules)) + compiler = Compiler(plugins=[SQLitePlugin()]) + compiled = compiler.compile(ItemFilter, items) + stmt = items.select().where(*sqlrules.where(compiled)) # Compile without a live DB — expressions use bound parameters. print(stmt.compile(compile_kwargs={"literal_binds": False})) diff --git a/packages/sqlrules-mssql/README.md b/packages/sqlrules-mssql/README.md index 4582867..d13fd92 100644 --- a/packages/sqlrules-mssql/README.md +++ b/packages/sqlrules-mssql/README.md @@ -1,38 +1,51 @@ # sqlrules-mssql -SQL Server dialect plugin for [SQLRules](https://github.com/eddiethedean/sqlrules). +SQL Server backend provider for [SQLRules](https://github.com/eddiethedean/sqlrules). +The package version follows the core 2.x line. ## Install ```bash -pip install sqlrules-mssql +pip install "sqlrules>=2,<3" "sqlrules-mssql>=2,<3" ``` -## Usage +## Use ```python from typing import Annotated, Any -from pydantic import BaseModel, Field +from pydantic import Field +from sqlalchemy import Column, MetaData, String, Table -from sqlrules import Compiler, JsonContains, JsonHasKey +from sqlrules import Compiler, JsonContains, JsonHasKey, RuleSchema, where from sqlrules_mssql import MssqlPlugin -class RowFilter(BaseModel): +rows = Table( + "rows", + MetaData(), + Column("name", String), + Column("meta", String), # SQL Server stores JSON documents in text columns. +) + + +class RowRules(RuleSchema): name: Annotated[str, Field(min_length=2, max_length=40)] meta: Annotated[dict[str, Any], JsonContains({"active": True}), JsonHasKey("active")] -compiler = Compiler(plugins=[MssqlPlugin()], dialect="mssql") + +provider = MssqlPlugin(server_version=(16, 0), compatibility_level=160) +compiled = Compiler(plugins=[provider]).compile(RowRules, rows) +statement = rows.select().where(*where(compiled)) ``` -## Operators +## Capabilities -| IR operator | Notes | -|---|---| -| `min_length` / `max_length` | `LEN(column)` instead of portable `length()` | -| `type_check` | Limited shape checks (typed columns; LIKE for some String forms) | -| `json_contains` | Shallow `JSON_VALUE` / `JSON_QUERY` checks | -| `json_has_key` | `JSON_VALUE` / `JSON_QUERY` IS NOT NULL | +- SQL Server `LEN` length constraints +- Guarded JSON helpers for text columns validated with `ISJSON` (`server_version` >= 13 and explicit `compatibility_level` >= 130) +- `JsonContains` supports null, boolean, and string values; nested objects and arrays compare structurally, while numeric JSON values raise `CapabilityError` because exact numeric equality cannot be guaranteed from `OPENJSON` text values +- Safe lax text-to-int/float conversions on SQL Server 2012+ +- No built-in regex translator +- Text-to-Decimal is a compile-time capability error -`pattern` is intentionally not registered — prefer a custom translator over -guessing LIKE semantics. +String Literal and Enum fields require an explicit binary or +case-sensitive collation. See the [type support matrix](https://sqlrules.readthedocs.io/en/latest/TYPE_SUPPORT.html). diff --git a/packages/sqlrules-mssql/pyproject.toml b/packages/sqlrules-mssql/pyproject.toml index 8c59fc2..cc32c0c 100644 --- a/packages/sqlrules-mssql/pyproject.toml +++ b/packages/sqlrules-mssql/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "sqlrules-mssql" -version = "1.0.1" +version = "2.0.0" description = "SQL Server dialect plugin for SQLRules (JSON and LEN string ops)." readme = "README.md" requires-python = ">=3.10" @@ -18,7 +18,7 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "sqlrules>=1,<2", + "sqlrules>=2,<3", "sqlalchemy>=2.0,<3", ] diff --git a/packages/sqlrules-mssql/src/sqlrules_mssql/__init__.py b/packages/sqlrules-mssql/src/sqlrules_mssql/__init__.py index 9785c46..4824b7d 100644 --- a/packages/sqlrules-mssql/src/sqlrules_mssql/__init__.py +++ b/packages/sqlrules-mssql/src/sqlrules_mssql/__init__.py @@ -1,12 +1,20 @@ from __future__ import annotations +from collections.abc import Mapping +from typing import Any, cast + +from sqlalchemy import String, case, func, literal +from sqlalchemy.sql.elements import ColumnElement + +from sqlrules.backend import prepare_scalar, total_predicate +from sqlrules.errors import CapabilityError +from sqlrules.ir import CompilationContext, PreparedValue, RuleField from sqlrules.plugins import PLUGIN_API_VERSION from sqlrules.translators import TranslatorRegistry from sqlrules_mssql.json import translate_json_contains, translate_json_has_key from sqlrules_mssql.length import translate_max_length, translate_min_length -from sqlrules_mssql.type_check import translate_type_check -__version__ = "1.0.1" +__version__ = "2.0.0" class MssqlPlugin: @@ -14,12 +22,103 @@ class MssqlPlugin: Does not register ``pattern`` — SQL Server has no portable regex operator that SQLRules can emit deterministically. Provide a custom translator if - needed. ``type_check`` is registered with a limited approximation matrix. + needed. Scalar types are prepared by the backend provider. """ name = "mssql" api_version = PLUGIN_API_VERSION + def __init__( + self, + *, + server_version: tuple[int, ...] | None = None, + compatibility_level: int | None = None, + ) -> None: + self.server_version = tuple(server_version) if server_version is not None else None + self.compatibility_level = compatibility_level + + def capabilities(self) -> Mapping[str, Any]: + return { + "backend": self.name, + "server_version": self.server_version, + "compatibility_level": self.compatibility_level, + "json_markers_supported": self.server_version is not None + and self.server_version >= (13, 0) + and self.compatibility_level is not None + and self.compatibility_level >= 130, + "native_scalar_types": ( + "bool", + "int", + "float", + "decimal", + "str", + "date", + "datetime", + "time", + "uuid", + ), + "safe_text_numeric_conversion": self.server_version is not None + and self.server_version >= (11, 0), + "safe_text_numeric_targets": ( + ("int", "float") + if self.server_version is not None and self.server_version >= (11, 0) + else () + ), + "assumptions": ( + "TRY_CAST provides safe numeric conversion on SQL Server 2012 and newer.", + "Text-to-numeric conversion is accepted only when TRY_CAST succeeds.", + ), + } + + def prepare_value( + self, + column: Any, + field: RuleField, + context: CompilationContext, + ) -> PreparedValue: + if field.python_type is dict and any( + item.operator in {"json_contains", "json_has_key"} for item in field.constraints + ): + if self.server_version is None or self.server_version < (13, 0): + raise CapabilityError( + self.name, + field.name, + "json", + type(column.type).__name__, + "SQL Server JSON markers require SQL Server 2016 or newer; " + "pass server_version explicitly.", + ) + if self.compatibility_level is None or self.compatibility_level < 130: + raise CapabilityError( + self.name, + field.name, + "json", + type(column.type).__name__, + "OPENJSON requires database compatibility level 130 or higher; " + "pass compatibility_level explicitly.", + ) + source_name = type(column.type).__name__.lower() + if not isinstance(column.type, String) and "json" not in source_name: + raise CapabilityError( + self.name, + field.name, + "json", + source_name, + "SQL Server JSON markers require a text or JSON column.", + ) + valid = total_predicate(func.isjson(column) == 1) + safe_json = case((valid, column), else_=literal("{}")) + return PreparedValue( + source=column, + value=cast(ColumnElement[Any], safe_json), + valid=valid, + is_null=cast(ColumnElement[bool], column.is_(None)), + logical_type="json", + coercion="validated-json-text", + capability="isjson-guarded", + ) + return prepare_scalar(column, field, context, backend=self.name) + def register(self, registry: TranslatorRegistry) -> None: registry.register_constraint( "min_length", @@ -31,11 +130,6 @@ def register(self, registry: TranslatorRegistry) -> None: translate_max_length, on_conflict="replace", ) - registry.register_constraint( - "type_check", - translate_type_check, - on_conflict="replace", - ) registry.register_constraint( "json_contains", translate_json_contains, @@ -55,5 +149,4 @@ def register(self, registry: TranslatorRegistry) -> None: "translate_json_has_key", "translate_max_length", "translate_min_length", - "translate_type_check", ] diff --git a/packages/sqlrules-mssql/src/sqlrules_mssql/json.py b/packages/sqlrules-mssql/src/sqlrules_mssql/json.py index 3bc31b7..a7a29fd 100644 --- a/packages/sqlrules-mssql/src/sqlrules_mssql/json.py +++ b/packages/sqlrules-mssql/src/sqlrules_mssql/json.py @@ -1,13 +1,13 @@ from __future__ import annotations -import json from typing import Any, cast -from sqlalchemy import String, exists, func, literal, select, type_coerce +from sqlalchemy import Integer, String, Unicode, and_, exists, func, literal, select from sqlalchemy import cast as sa_cast from sqlalchemy import column as sa_column from sqlalchemy.sql.elements import ColumnElement +from sqlrules.errors import CapabilityError from sqlrules.ir import CompilationContext, Constraint @@ -19,33 +19,195 @@ def _json_path_for_key(key: Any) -> str: return f'$."{escaped}"' -def _compact_dumps(value: Any) -> str: - return json.dumps(value, separators=(",", ":")) +def _json_path_for_array_index(index: int) -> str: + """Build a JSONPath for one zero-based array element.""" + return f"$[{index}]" + + +def _exact_text_equals( + left: ColumnElement[Any], + right: ColumnElement[Any], +) -> ColumnElement[bool]: + """Compare text ordinally and distinguish trailing spaces on SQL Server.""" + left_text = sa_cast(left, Unicode()).collate("Latin1_General_100_BIN2") + right_text = sa_cast(right, Unicode()).collate("Latin1_General_100_BIN2") + return cast( + ColumnElement[bool], + (left_text == right_text) & (func.datalength(left_text) == func.datalength(right_text)), + ) def _openjson_key_exists( column: ColumnElement[Any], key: str, *, - json_type: str | None = None, + json_type: int | None = None, ) -> ColumnElement[bool]: """True when OPENJSON lists ``key`` (optionally with a specific JSON type).""" + return _openjson_key_matches(column, key, json_type=json_type) + + +def _openjson_key_matches( + column: ColumnElement[Any], + key: str, + *, + json_type: int | None = None, + expected_value: str | None = None, +) -> ColumnElement[bool]: + """True when a top-level JSON key has the requested type and value.""" oj = ( func.openjson(column) .table_valued( sa_column("key", String), - sa_column("value", String), - sa_column("type", String), + sa_column("value", Unicode), + sa_column("type", Integer), ) - .render_derived(name="oj", with_types=True) .alias("oj") ) - predicate = oj.c.key == key + predicate = _exact_text_equals( + oj.c.key, + literal(key, type_=Unicode()), + ) if json_type is not None: predicate = predicate & (oj.c.type == json_type) + if expected_value is not None: + predicate = predicate & _exact_text_equals( + oj.c.value, + literal(expected_value, type_=Unicode()), + ) return cast(ColumnElement[bool], exists(select(1).select_from(oj).where(predicate))) +def _openjson_child_count(document: ColumnElement[Any]) -> ColumnElement[Any]: + children = func.openjson(document).table_valued(sa_column("key", String)).alias("children") + return select(func.count()).select_from(children).scalar_subquery() + + +def _openjson_object_equals( + document: ColumnElement[Any], + expected: dict[Any, Any], + field: str, +) -> ColumnElement[bool]: + parts: list[ColumnElement[bool]] = [ + cast(ColumnElement[bool], _openjson_child_count(document) == len(expected)) + ] + parts.extend( + _openjson_value_equals(document, str(key), value, field) for key, value in expected.items() + ) + return cast(ColumnElement[bool], and_(*parts)) + + +def _openjson_array_equals( + document: ColumnElement[Any], + expected: list[Any], + field: str, +) -> ColumnElement[bool]: + parts: list[ColumnElement[bool]] = [ + cast(ColumnElement[bool], _openjson_child_count(document) == len(expected)) + ] + parts.extend( + _openjson_value_equals( + document, + str(index), + value, + field, + nested_path=_json_path_for_array_index(index), + ) + for index, value in enumerate(expected) + ) + return cast(ColumnElement[bool], and_(*parts)) + + +def _openjson_value_equals( + document: ColumnElement[Any], + key: str, + expected: Any, + field: str, + *, + nested_path: str | None = None, +) -> ColumnElement[bool]: + if expected is None: + return _openjson_key_exists(document, key, json_type=0) + if isinstance(expected, bool): + return _openjson_key_matches( + document, + key, + json_type=3, + expected_value="true" if expected else "false", + ) + if isinstance(expected, str): + return _openjson_key_matches( + document, + key, + json_type=1, + expected_value=expected, + ) + if isinstance(expected, dict): + nested = func.json_query(document, nested_path or _json_path_for_key(key)) + return cast( + ColumnElement[bool], + _openjson_key_exists(document, key, json_type=5) + & _openjson_object_equals(nested, expected, field), + ) + if isinstance(expected, list): + nested = func.json_query(document, nested_path or _json_path_for_key(key)) + return cast( + ColumnElement[bool], + _openjson_key_exists(document, key, json_type=4) + & _openjson_array_equals(nested, expected, field), + ) + if isinstance(expected, (int, float)): + raise CapabilityError( + "mssql", + field, + "exact JSON numeric equality", + "OPENJSON.value text", + "SQL Server exposes JSON numbers as text, and the provider cannot " + "guarantee exact numeric equivalence without lossy conversion.", + ) + raise CapabilityError( + "mssql", + field, + "JSON scalar equality", + type(expected).__name__, + "Only null, boolean, string, object, and array values are supported by " + "the SQL Server JSON containment translator.", + ) + + +def _is_json_root( + document: ColumnElement[Any], + root_marker: str, +) -> ColumnElement[bool]: + """Check the root shape using functions available since SQL Server 2016.""" + text: ColumnElement[Any] = sa_cast(document, Unicode()) + # LTRIM on supported SQL Server versions removes spaces only. Normalize + # the other JSON whitespace characters first so valid pretty-printed + # documents receive the same root-shape check. + for whitespace in ("\t", "\n", "\r"): + text = cast( + ColumnElement[Any], + func.replace( + text, + literal(whitespace, type_=Unicode()), + literal(" ", type_=Unicode()), + ), + ) + return cast( + ColumnElement[bool], + (func.isjson(document) == 1) + & (func.left(func.ltrim(text), 1) == literal(root_marker, type_=Unicode())), + ) + + +def _is_json_object(document: ColumnElement[Any]) -> ColumnElement[bool]: + return _is_json_root(document, "{") + + +def _is_json_array(document: ColumnElement[Any]) -> ColumnElement[bool]: + return _is_json_root(document, "[") + + def translate_json_contains( constraint: Constraint, column: ColumnElement[Any], @@ -53,66 +215,35 @@ def translate_json_contains( ) -> ColumnElement[bool]: """Translate ``json_contains`` using SQL Server JSON functions. - Object payloads use shallow key checks via ``JSON_VALUE`` / - ``JSON_QUERY`` / ``OPENJSON``. Nested deep-merge containment is not - emulated. + Object payloads contain their requested top-level keys. Nested objects and + arrays are matched exactly by structure, independent of whitespace and + object key order. Numeric comparisons are rejected because ``OPENJSON`` + exposes numbers as text and SQL Server cannot guarantee exact equality + without a potentially lossy conversion. """ value = constraint.value if isinstance(value, dict): if not value: - # Align with PostgreSQL ``@> '{}'``: require a non-NULL JSON object. - return cast( - ColumnElement[bool], - column.is_not(None) - & (func.isjson(column) == 1) - & func.json_query(column, "$").is_not(None), - ) + return _is_json_object(column) parts: list[ColumnElement[bool]] = [] for key, expected in value.items(): - path = _json_path_for_key(key) - key_text = str(key) - if expected is None: - parts.append(_openjson_key_exists(column, key_text, json_type="null")) - elif isinstance(expected, (dict, list)): - compact = _compact_dumps(expected) - parts.append( - cast( - ColumnElement[bool], - func.json_query(column, path) - == func.json_query(sa_cast(literal(compact), String), "$"), - ) - ) - elif isinstance(expected, bool): - expected_text = "true" if expected else "false" - parts.append( - cast( - ColumnElement[bool], - func.json_value(column, path) == literal(expected_text), - ) - ) - elif isinstance(expected, str): - parts.append( - cast( - ColumnElement[bool], - func.json_value(column, path) == literal(expected), - ) - ) - else: - parts.append( - cast( - ColumnElement[bool], - func.json_value(column, path) == literal(str(expected)), - ) - ) + parts.append(_openjson_value_equals(column, str(key), expected, constraint.field)) expression = parts[0] for part in parts[1:]: expression = expression & part - return cast(ColumnElement[bool], expression) - - compact = _compact_dumps(value) - return cast( - ColumnElement[bool], - type_coerce(column, String) == func.json_query(sa_cast(literal(compact), String), "$"), + return cast(ColumnElement[bool], _is_json_object(column) & expression) + if isinstance(value, list): + return cast( + ColumnElement[bool], + _is_json_array(column) & _openjson_array_equals(column, value, constraint.field), + ) + raise CapabilityError( + "mssql", + constraint.field, + "JSON scalar containment", + type(value).__name__, + "SQL Server's JSON containment translator supports object and array payloads; " + "top-level scalar payloads cannot be compared reliably.", ) @@ -122,4 +253,7 @@ def translate_json_has_key( context: CompilationContext, ) -> ColumnElement[bool]: """Translate ``json_has_key`` via ``OPENJSON`` key presence (includes JSON null).""" - return _openjson_key_exists(column, str(constraint.value)) + return cast( + ColumnElement[bool], + _is_json_object(column) & _openjson_key_exists(column, str(constraint.value)), + ) diff --git a/packages/sqlrules-mssql/src/sqlrules_mssql/length.py b/packages/sqlrules-mssql/src/sqlrules_mssql/length.py index 5af9f3d..fa684bd 100644 --- a/packages/sqlrules-mssql/src/sqlrules_mssql/length.py +++ b/packages/sqlrules-mssql/src/sqlrules_mssql/length.py @@ -2,15 +2,24 @@ from typing import Any, cast -from sqlalchemy import func, literal +from sqlalchemy import Unicode, func, literal +from sqlalchemy import cast as sa_cast from sqlalchemy.sql.elements import ColumnElement from sqlrules.ir import CompilationContext, Constraint def _char_length(column: ColumnElement[Any]) -> ColumnElement[Any]: - """Character length including trailing spaces (``LEN`` alone strips them).""" - return func.len(column.concat(literal("."))) - 1 + """Count Unicode code points and trailing spaces on supported SQL Server. + + SQL Server ``LEN`` counts a UTF-16 surrogate pair as two under ordinary + collations. Convert to an unbounded Unicode expression with an SC-aware + collation before measuring; the appended character keeps trailing spaces + in the count. + """ + unicode_column = sa_cast(column, Unicode()).collate("Latin1_General_100_CI_AS_SC") + sentinel = literal(".", type_=Unicode()) + return func.len(unicode_column.concat(sentinel)) - 1 def translate_min_length( diff --git a/packages/sqlrules-mssql/src/sqlrules_mssql/type_check.py b/packages/sqlrules-mssql/src/sqlrules_mssql/type_check.py deleted file mode 100644 index e5bb6fa..0000000 --- a/packages/sqlrules-mssql/src/sqlrules_mssql/type_check.py +++ /dev/null @@ -1,232 +0,0 @@ -from __future__ import annotations - -from datetime import date, datetime, time -from decimal import Decimal -from typing import Any, cast -from uuid import UUID - -from sqlalchemy import Boolean, Date, DateTime, Float, Integer, Numeric, String, Time -from sqlalchemy import cast as sa_cast -from sqlalchemy.sql.elements import ColumnElement -from sqlalchemy.sql.sqltypes import NullType -from sqlalchemy.types import TypeEngine - -from sqlrules.constraints import type_spec -from sqlrules.errors import UnsupportedConstraintError -from sqlrules.ir import CompilationContext, Constraint, TypeSpec - -_UUID_LIKE = ( - "[0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f]" - "[0-9A-Fa-f][0-9A-Fa-f]-" - "[0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f]-" - "[0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f]-" - "[0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f]-" - "[0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f]" - "[0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f]" -) - - -def _is_type(column: ColumnElement[Any], *bases: type[TypeEngine[Any]]) -> bool: - col_type = column.type - if isinstance(col_type, NullType): - return False - return isinstance(col_type, bases) - - -def _wrap_none( - column: ColumnElement[Any], - predicate: ColumnElement[bool], - *, - allow_none: bool, -) -> ColumnElement[bool]: - if allow_none: - return cast(ColumnElement[bool], column.is_(None) | predicate) - return predicate - - -def _unsupported(field: str, spec: TypeSpec, suggestion: str) -> None: - raise UnsupportedConstraintError( - field=field, - operator="type_check", - value=spec, - suggestion=suggestion, - ) - - -def _predicate_int( - column: ColumnElement[Any], - spec: TypeSpec, - field: str, -) -> ColumnElement[bool]: - if _is_type(column, Integer): - return cast(ColumnElement[bool], column.isnot(None)) - if spec.strict: - _unsupported( - field, - spec, - "SQL Server strict int type_check requires an Integer column.", - ) - if _is_type(column, String): - unsigned = column.like("[0-9]%") & ~column.like("%[^0-9]%") - signed = column.like("[-+][0-9]%") & ~column.like("[-+]%[^0-9]%") - return cast(ColumnElement[bool], unsigned | signed) - if _is_type(column, Float, Numeric): - return cast( - ColumnElement[bool], - column == sa_cast(sa_cast(column, Integer), Float), - ) - _unsupported( - field, - spec, - "SQL Server lax int type_check supports Integer, String, Float, or Numeric columns.", - ) - raise AssertionError("unreachable") - - -def _predicate_bool( - column: ColumnElement[Any], - spec: TypeSpec, - field: str, -) -> ColumnElement[bool]: - if spec.strict: - if _is_type(column, Boolean, Integer): - return cast(ColumnElement[bool], column.in_((True, False, 0, 1))) - _unsupported( - field, - spec, - "SQL Server strict bool type_check requires a Boolean/Integer column.", - ) - _unsupported( - field, - spec, - "SQL Server lax bool type_check is not supported. Use strict=True.", - ) - raise AssertionError("unreachable") - - -def _predicate_str( - column: ColumnElement[Any], - spec: TypeSpec, - field: str, -) -> ColumnElement[bool]: - if _is_type(column, String): - return cast(ColumnElement[bool], column.isnot(None)) - _unsupported( - field, - spec, - "SQL Server str type_check requires a String/Text column.", - ) - raise AssertionError("unreachable") - - -def _predicate_float( - column: ColumnElement[Any], - spec: TypeSpec, - field: str, -) -> ColumnElement[bool]: - if _is_type(column, Float, Numeric, Integer): - return cast(ColumnElement[bool], column.isnot(None)) - if spec.strict: - _unsupported( - field, - spec, - "SQL Server strict float/Decimal type_check requires a numeric column.", - ) - if _is_type(column, String): - _unsupported( - field, - spec, - "SQL Server lax float/Decimal type_check on String columns is not " - "supported (no portable numeric shape check). Use a numeric column " - "or strict=True with Float/Numeric/Integer.", - ) - _unsupported( - field, - spec, - "SQL Server float/Decimal type_check supports Float, Numeric, or Integer columns.", - ) - raise AssertionError("unreachable") - - -def _predicate_temporal( - column: ColumnElement[Any], - spec: TypeSpec, - field: str, - *, - sa_types: tuple[type[TypeEngine[Any]], ...], -) -> ColumnElement[bool]: - if _is_type(column, *sa_types): - return cast(ColumnElement[bool], column.isnot(None)) - type_name = spec.python_type.__name__ - _unsupported( - field, - spec, - f"SQL Server {type_name} type_check requires a typed {type_name} column " - "(string date parsing is not emitted without a portable regex).", - ) - raise AssertionError("unreachable") - - -def _predicate_uuid( - column: ColumnElement[Any], - spec: TypeSpec, - field: str, -) -> ColumnElement[bool]: - if "uuid" in type(column.type).__name__.lower(): - return cast(ColumnElement[bool], column.isnot(None)) - if spec.strict: - _unsupported( - field, - spec, - "SQL Server strict UUID type_check requires a UUID column.", - ) - if _is_type(column, String): - return cast(ColumnElement[bool], column.like(_UUID_LIKE)) - _unsupported( - field, - spec, - "SQL Server UUID type_check supports UUID or String columns.", - ) - raise AssertionError("unreachable") - - -def _build_predicate( - column: ColumnElement[Any], - spec: TypeSpec, - field: str, -) -> ColumnElement[bool]: - python_type = spec.python_type - if python_type is int: - return _predicate_int(column, spec, field) - if python_type is bool: - return _predicate_bool(column, spec, field) - if python_type is str: - return _predicate_str(column, spec, field) - if python_type in {float, Decimal}: - return _predicate_float(column, spec, field) - if python_type is date: - return _predicate_temporal(column, spec, field, sa_types=(Date,)) - if python_type is datetime: - return _predicate_temporal(column, spec, field, sa_types=(DateTime,)) - if python_type is time: - return _predicate_temporal(column, spec, field, sa_types=(Time,)) - if python_type is UUID: - return _predicate_uuid(column, spec, field) - _unsupported( - field, - spec, - f"SQL Server type_check has no translator for {python_type!r}.", - ) - raise AssertionError("unreachable") - - -def translate_type_check( - constraint: Constraint, - column: ColumnElement[Any], - context: CompilationContext, -) -> ColumnElement[bool]: - """Translate ``type_check`` into SQL Server shape/type predicates.""" - del context - spec = type_spec(constraint.value) - predicate = _build_predicate(column, spec, constraint.field) - return _wrap_none(column, predicate, allow_none=spec.allow_none) diff --git a/packages/sqlrules-mssql/tests/test_mssql_plugin.py b/packages/sqlrules-mssql/tests/test_mssql_plugin.py index a260bbd..9fd6d30 100644 --- a/packages/sqlrules-mssql/tests/test_mssql_plugin.py +++ b/packages/sqlrules-mssql/tests/test_mssql_plugin.py @@ -2,175 +2,113 @@ from typing import Annotated, Any -import pytest -from pydantic import BaseModel, Field -from sqlalchemy import Column, MetaData, String, Table, Text -from sqlalchemy.dialects import mssql +from pydantic import Field +from sqlalchemy import Column, MetaData, String, Table +from sqlalchemy.dialects.mssql import dialect from sqlrules_mssql import MssqlPlugin, __version__ -from sqlrules import Compiler, JsonContains, JsonHasKey, UnsupportedConstraintError +from sqlrules import Compiler, JsonContains, JsonHasKey, RuleSchema from sqlrules.conformance import run_basic_conformance -def test_version() -> None: - assert __version__ == "1.0.1" +def test_version_and_plugin_conformance() -> None: + assert __version__ == "2.0.0" + run_basic_conformance(MssqlPlugin(), operator="min_length") -def test_conformance() -> None: - # MSSQL does not register pattern; exercise length override instead. - class LengthFilter(BaseModel): - name: Annotated[str, Field(min_length=2)] +def test_length_and_json_constraints_compile() -> None: + class Rules(RuleSchema): + name: Annotated[str, Field(min_length=2, max_length=40)] + meta: Annotated[dict[str, Any], JsonContains({"active": True}), JsonHasKey("active")] - run_basic_conformance( - MssqlPlugin(), - operator="min_length", - model=LengthFilter, + table = Table( + "rows", + MetaData(), + Column("name", String), + Column("meta", String), ) + compiled = Compiler( + plugins=[MssqlPlugin(server_version=(16, 0), compatibility_level=160)] + ).compile(Rules, table) + sql = str(compiled.predicate.compile(dialect=dialect())) + assert "len(" in sql.lower() + assert "isjson" in sql.lower() + assert "left(ltrim" in sql.lower() + assert "openjson" in sql.lower() + assert " as oj" in sql.lower() + assert "as oj(" not in sql.lower() + assert compiled.fields[1].coercion == "validated-json-text" + + +def test_empty_json_contains_checks_for_an_object_root() -> None: + class Rules(RuleSchema): + meta: Annotated[dict[str, Any], JsonContains({})] + table = Table("rows", MetaData(), Column("meta", String)) + compiled = Compiler( + plugins=[MssqlPlugin(server_version=(16, 0), compatibility_level=160)] + ).compile(Rules, table) + sql = str(compiled.predicate.compile(dialect=dialect())).lower() + assert "left(ltrim" in sql -def test_length_uses_trailing_space_aware_len() -> None: - class Filter(BaseModel): - name: Annotated[str, Field(min_length=2, max_length=10)] - - table = Table("items", MetaData(), Column("name", String)) - rules = Compiler( - plugins=[MssqlPlugin()], - dialect="mssql", - cache=False, - ).compile(Filter, table) - dialect = mssql.dialect() - min_sql = str( - rules["name"][0].compile(dialect=dialect, compile_kwargs={"literal_binds": True}) - ).lower() - max_sql = str( - rules["name"][1].compile(dialect=dialect, compile_kwargs={"literal_binds": True}) - ).lower() - # Trailing-space-aware: LEN(name + '.') - 1, not portable length()/plain LEN(name). - assert min_sql == "len(items.name + '.') - 1 >= 2" - assert max_sql == "len(items.name + '.') - 1 <= 10" - assert "length(" not in min_sql - assert "length(" not in max_sql - - -def test_json_operators_compile() -> None: - class Filter(BaseModel): - meta: Annotated[dict[str, Any], JsonContains({"active": True}), JsonHasKey("active")] - table = Table("items", MetaData(), Column("meta", Text)) - rules = Compiler( - plugins=[MssqlPlugin()], - dialect="mssql", - cache=False, - ).compile(Filter, table) - assert len(rules["meta"]) == 2 - dialect = mssql.dialect() - contains_sql = str(rules["meta"][0].compile(dialect=dialect)).lower() - has_key_sql = str(rules["meta"][1].compile(dialect=dialect)).lower() - assert "json_value" in contains_sql - assert "openjson" in has_key_sql - - -def test_json_contains_none_uses_openjson_null_type() -> None: - class Filter(BaseModel): - meta: Annotated[dict[str, Any], JsonContains({"a": None})] - - table = Table("items", MetaData(), Column("meta", Text)) - rules = Compiler( - plugins=[MssqlPlugin()], - dialect="mssql", - cache=False, - ).compile(Filter, table) - compiled = str( - rules["meta"][0].compile( - dialect=mssql.dialect(), - compile_kwargs={"literal_binds": True}, - ) - ).lower() - assert "openjson" in compiled - assert "null" in compiled - assert "'none'" not in compiled - - -def test_json_contains_nested_uses_compact_json_query() -> None: - class Filter(BaseModel): - meta: Annotated[dict[str, Any], JsonContains({"nested": {"x": 1}})] - - table = Table("items", MetaData(), Column("meta", Text)) - rules = Compiler( - plugins=[MssqlPlugin()], - dialect="mssql", - cache=False, - ).compile(Filter, table) - compiled = str( - rules["meta"][0].compile( - dialect=mssql.dialect(), - compile_kwargs={"literal_binds": True}, - ) - ) - assert '{"x":1}' in compiled - assert '{"x": 1}' not in compiled +def test_text_to_integer_uses_try_cast_and_digit_validation() -> None: + class Rules(RuleSchema): + value: int + table = Table("rows", MetaData(), Column("value", String)) + compiled = Compiler(plugins=[MssqlPlugin(server_version=(16, 0))]).compile(Rules, table) + sql = str(compiled.predicate.compile(dialect=dialect())) + assert "TRY_CAST" in sql + assert "NOT LIKE" in sql + assert compiled.fields[0].coercion == "text-to-int" -def test_pattern_remains_unsupported() -> None: - class Filter(BaseModel): - name: Annotated[str, Field(pattern=r"^A")] - table = Table("items", MetaData(), Column("name", String)) - with pytest.raises(UnsupportedConstraintError, match="pattern"): - Compiler( - plugins=[MssqlPlugin()], - dialect="mssql", - cache=False, - ).compile(Filter, table) +def test_sql_server_totalizes_predicates_with_case_expressions() -> None: + class Rules(RuleSchema): + value: int + table = Table("rows", MetaData(), Column("value", String)) + compiled = Compiler(plugins=[MssqlPlugin(server_version=(16, 0))]).compile(Rules, table) + sql = str(compiled.predicate.compile(dialect=dialect())).lower() + assert "case when" in sql + assert "coalesce(" not in sql -def test_empty_json_contains_requires_object() -> None: - class Filter(BaseModel): - meta: Annotated[dict[str, Any], JsonContains({})] - table = Table("items", MetaData(), Column("meta", Text)) - rules = Compiler( - plugins=[MssqlPlugin()], - dialect="mssql", - cache=False, - ).compile(Filter, table) - assert len(rules["meta"]) == 1 - compiled = str( - rules["meta"][0].compile( - dialect=mssql.dialect(), - compile_kwargs={"literal_binds": True}, +def test_pattern_remains_a_capability_error() -> None: + class Rules(RuleSchema): + name: Annotated[str, Field(pattern="^A")] + + table = Table("rows", MetaData(), Column("name", String)) + from sqlrules import CapabilityError + + try: + Compiler(plugins=[MssqlPlugin()]).compile(Rules, table) + except CapabilityError as exc: + assert "pattern" in str(exc) + else: # pragma: no cover - assertion is the capability contract + raise AssertionError("SQL Server does not promise regex equivalence") + + +def test_json_requires_server_version_and_database_compatibility_level() -> None: + class Rules(RuleSchema): + meta: Annotated[dict[str, Any], JsonContains({"active": True})] + + table = Table("rows", MetaData(), Column("meta", String)) + from sqlrules import CapabilityError + + try: + Compiler(plugins=[MssqlPlugin(server_version=(16, 0))]).compile(Rules, table) + except CapabilityError as exc: + assert "compatibility level 130" in str(exc) + else: # pragma: no cover - assertion is the capability contract + raise AssertionError("OPENJSON requires an explicit compatibility level") + + try: + Compiler(plugins=[MssqlPlugin(server_version=(16, 0), compatibility_level=120)]).compile( + Rules, table ) - ).lower() - assert "isjson" in compiled - assert "json_query" in compiled - - -def test_type_check_lax_float_string_unsupported() -> None: - class Filter(BaseModel): - score: float - - table = Table("rows", MetaData(), Column("score", String)) - with pytest.raises(UnsupportedConstraintError, match="float"): - Compiler( - plugins=[MssqlPlugin()], - dialect="mssql", - emit_type_checks=True, - cache=False, - ).compile(Filter, table) - - -def test_type_check_int_integer_column() -> None: - from sqlalchemy import Integer - - class Filter(BaseModel): - age: int - - table = Table("users", MetaData(), Column("age", Integer)) - rules = Compiler( - plugins=[MssqlPlugin()], - dialect="mssql", - emit_type_checks=True, - cache=False, - ).compile(Filter, table) - compiled = str(rules["age"][0].compile(dialect=mssql.dialect())) - assert "IS NOT NULL" in compiled.upper() + except CapabilityError as exc: + assert "compatibility level 130" in str(exc) + else: # pragma: no cover - assertion is the capability contract + raise AssertionError("OPENJSON requires compatibility level 130 or higher") diff --git a/packages/sqlrules-mysql/README.md b/packages/sqlrules-mysql/README.md index 8a189fc..e811824 100644 --- a/packages/sqlrules-mysql/README.md +++ b/packages/sqlrules-mysql/README.md @@ -1,44 +1,61 @@ # sqlrules-mysql -MySQL / MariaDB dialect plugin for [SQLRules](https://github.com/eddiethedean/sqlrules). +MySQL backend provider for [SQLRules](https://github.com/eddiethedean/sqlrules). +The package version follows the core 2.x line. + +This provider targets MySQL 8.0+. MariaDB is outside the supported server +matrix because its regular-expression function API differs from MySQL's. ## Install ```bash -pip install sqlrules-mysql +pip install "sqlrules>=2,<3" "sqlrules-mysql>=2,<3" ``` -## Usage +## Use ```python from typing import Annotated, Any -from pydantic import BaseModel, Field +from pydantic import Field +from sqlalchemy import Column, JSON, MetaData, String, Table -from sqlrules import Compiler, FullTextMatch, JsonContains +from sqlrules import Compiler, FullTextMatch, JsonContains, RuleSchema, where from sqlrules_mysql import MysqlPlugin -class RowFilter(BaseModel): +rows = Table( + "rows", + MetaData(), + Column("name", String), + Column("meta", JSON), + Column("body", String), +) + + +class RowRules(RuleSchema): name: Annotated[str, Field(pattern=r"^A")] meta: Annotated[dict[str, Any], JsonContains({"active": True})] body: Annotated[str, FullTextMatch("sqlrules")] -compiler = Compiler(plugins=[MysqlPlugin()], dialect="mysql") + +provider = MysqlPlugin(server_version=(8, 0, 36)) +compiled = Compiler(plugins=[provider]).compile(RowRules, rows) +statement = rows.select().where(*where(compiled)) ``` -## Operators +## Capabilities + +- `pattern`: MySQL `REGEXP` +- JSON containment and key lookup +- Full-text matching (requires a matching FULLTEXT index) +- Safe lax text-to-int conversion on MySQL 8.0+ -| IR operator | Notes | -|---|---| -| `pattern` | `REGEXP` (case-insensitive under typical collations) | -| `type_check` | Shape/type predicates from `TypeSpec` (partial matrix) | -| `json_contains` | `JSON_CONTAINS(column, payload) = 1` | -| `json_has_key` | `JSON_CONTAINS_PATH(column, 'one', '$.key') = 1` | -| `fulltext_match` | `MATCH(column) AGAINST (value)` — requires a FULLTEXT index | +Text-to-float and text-to-Decimal are compile-time capability errors. String +Literal and Enum fields need an explicit binary or case-sensitive collation. +See the [type support matrix](https://sqlrules.readthedocs.io/en/latest/TYPE_SUPPORT.html). -## Security note +## Pattern cost -`pattern` / `fulltext_match` values are bound parameters, but evaluation cost -is engine-dependent. Prefer static/allowlisted patterns and queries from -untrusted input. See -[SECURITY](https://sqlrules.readthedocs.io/en/latest/SECURITY.html). +Untrusted regular expressions and full-text queries can be expensive. Prefer +static or allowlisted values. See the SQLRules +[security notes](https://sqlrules.readthedocs.io/en/latest/SECURITY.html). diff --git a/packages/sqlrules-mysql/pyproject.toml b/packages/sqlrules-mysql/pyproject.toml index 00a9d35..26f7aac 100644 --- a/packages/sqlrules-mysql/pyproject.toml +++ b/packages/sqlrules-mysql/pyproject.toml @@ -4,13 +4,13 @@ build-backend = "hatchling.build" [project] name = "sqlrules-mysql" -version = "1.0.1" -description = "MySQL/MariaDB dialect plugin for SQLRules (REGEXP, JSON, full-text)." +version = "2.0.0" +description = "MySQL dialect plugin for SQLRules (REGEXP, JSON, full-text)." readme = "README.md" requires-python = ">=3.10" license = "MIT" authors = [{ name = "SQLRules Contributors" }] -keywords = ["sqlrules", "mysql", "mariadb", "pydantic", "sqlalchemy"] +keywords = ["sqlrules", "mysql", "pydantic", "sqlalchemy"] classifiers = [ "Development Status :: 5 - Production/Stable", "License :: OSI Approved :: MIT License", @@ -18,7 +18,7 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "sqlrules>=1,<2", + "sqlrules>=2,<3", "sqlalchemy>=2.0,<3", ] diff --git a/packages/sqlrules-mysql/src/sqlrules_mysql/__init__.py b/packages/sqlrules-mysql/src/sqlrules_mysql/__init__.py index 3cb4e7b..39dab17 100644 --- a/packages/sqlrules-mysql/src/sqlrules_mysql/__init__.py +++ b/packages/sqlrules-mysql/src/sqlrules_mysql/__init__.py @@ -1,33 +1,73 @@ from __future__ import annotations +from collections.abc import Mapping +from typing import Any + +from sqlrules.backend import prepare_scalar +from sqlrules.ir import CompilationContext, PreparedValue, RuleField from sqlrules.plugins import PLUGIN_API_VERSION from sqlrules.translators import TranslatorRegistry from sqlrules_mysql.fulltext import translate_fulltext_match from sqlrules_mysql.json import translate_json_contains, translate_json_has_key +from sqlrules_mysql.length import translate_max_length, translate_min_length from sqlrules_mysql.pattern import translate_pattern -from sqlrules_mysql.type_check import translate_type_check -__version__ = "1.0.1" +__version__ = "2.0.0" class MysqlPlugin: - """Register MySQL / MariaDB constraint translators.""" + """Register MySQL constraint translators.""" name = "mysql" api_version = PLUGIN_API_VERSION + def __init__(self, *, server_version: tuple[int, ...] | None = None) -> None: + self.server_version = tuple(server_version) if server_version is not None else None + + def capabilities(self) -> Mapping[str, Any]: + return { + "backend": self.name, + "server_version": self.server_version, + "native_scalar_types": ( + "bool", + "int", + "float", + "decimal", + "str", + "date", + "datetime", + "time", + ), + "safe_text_numeric_conversion": self.server_version is not None + and self.server_version >= (8, 0), + "safe_text_numeric_targets": ( + ("int",) + if self.server_version is not None and self.server_version >= (8, 0) + else () + ), + "assumptions": ( + "MySQL 8.0 REGEXP and native column types provide the advertised type evidence.", + "Lax integer/float/Decimal conversions accept the documented lexical profile.", + ), + } + + def prepare_value( + self, + column: Any, + field: RuleField, + context: CompilationContext, + ) -> PreparedValue: + return prepare_scalar(column, field, context, backend=self.name) + def register(self, registry: TranslatorRegistry) -> None: registry.register_constraint( "pattern", translate_pattern, on_conflict="replace", ) - registry.register_constraint( - "type_check", - translate_type_check, - on_conflict="replace", - ) for operator, translator in ( + ("min_length", translate_min_length), + ("max_length", translate_max_length), ("json_contains", translate_json_contains), ("json_has_key", translate_json_has_key), ("fulltext_match", translate_fulltext_match), @@ -39,8 +79,9 @@ def register(self, registry: TranslatorRegistry) -> None: "MysqlPlugin", "__version__", "translate_fulltext_match", + "translate_max_length", + "translate_min_length", "translate_json_contains", "translate_json_has_key", "translate_pattern", - "translate_type_check", ] diff --git a/packages/sqlrules-mysql/src/sqlrules_mysql/json.py b/packages/sqlrules-mysql/src/sqlrules_mysql/json.py index 58874a5..e0d7a23 100644 --- a/packages/sqlrules-mysql/src/sqlrules_mysql/json.py +++ b/packages/sqlrules-mysql/src/sqlrules_mysql/json.py @@ -22,9 +22,7 @@ def translate_json_contains( context: CompilationContext, ) -> ColumnElement[bool]: """Translate ``json_contains`` to MySQL ``JSON_CONTAINS``.""" - payload = constraint.value - if not isinstance(payload, str): - payload = json.dumps(payload, separators=(",", ":")) + payload = json.dumps(constraint.value, separators=(",", ":")) return cast(ColumnElement[bool], func.json_contains(column, payload) == 1) diff --git a/packages/sqlrules-mysql/src/sqlrules_mysql/length.py b/packages/sqlrules-mysql/src/sqlrules_mysql/length.py new file mode 100644 index 0000000..57611b8 --- /dev/null +++ b/packages/sqlrules-mysql/src/sqlrules_mysql/length.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from typing import Any, cast + +from sqlalchemy import func +from sqlalchemy.sql.elements import ColumnElement + +from sqlrules.ir import CompilationContext, Constraint + + +def translate_min_length( + constraint: Constraint, + column: ColumnElement[Any], + context: CompilationContext, +) -> ColumnElement[bool]: + """Compare MySQL character count, not encoded byte length.""" + return cast(ColumnElement[bool], func.char_length(column) >= constraint.value) + + +def translate_max_length( + constraint: Constraint, + column: ColumnElement[Any], + context: CompilationContext, +) -> ColumnElement[bool]: + """Compare MySQL character count, not encoded byte length.""" + return cast(ColumnElement[bool], func.char_length(column) <= constraint.value) + + +__all__ = ["translate_max_length", "translate_min_length"] diff --git a/packages/sqlrules-mysql/src/sqlrules_mysql/pattern.py b/packages/sqlrules-mysql/src/sqlrules_mysql/pattern.py index 2c51d7b..0680b16 100644 --- a/packages/sqlrules-mysql/src/sqlrules_mysql/pattern.py +++ b/packages/sqlrules-mysql/src/sqlrules_mysql/pattern.py @@ -2,6 +2,7 @@ from typing import Any, cast +from sqlalchemy import func from sqlalchemy.sql.elements import ColumnElement from sqlrules.constraints import pattern_text @@ -13,10 +14,12 @@ def translate_pattern( column: ColumnElement[Any], context: CompilationContext, ) -> ColumnElement[bool]: - """Translate ``pattern`` to MySQL/MariaDB ``REGEXP``. + """Translate ``pattern`` to MySQL ``REGEXP_LIKE``. - MySQL ``REGEXP`` is case-insensitive for non-binary collations. SQLRules - follows that dialect behavior rather than inventing ``REGEXP BINARY``. + MySQL's default matching follows the expression collation. Use + ``REGEXP_LIKE``'s match type to keep SQLRules patterns case-sensitive by + default and preserve an explicit ``re.IGNORECASE`` flag. """ - pattern, _ignore_case = pattern_text(constraint.value) - return cast(ColumnElement[bool], column.op("REGEXP")(pattern)) + pattern, ignore_case = pattern_text(constraint.value) + match_type = "i" if ignore_case else "c" + return cast(ColumnElement[bool], func.regexp_like(column, pattern, match_type)) diff --git a/packages/sqlrules-mysql/src/sqlrules_mysql/type_check.py b/packages/sqlrules-mysql/src/sqlrules_mysql/type_check.py deleted file mode 100644 index 84b5c6d..0000000 --- a/packages/sqlrules-mysql/src/sqlrules_mysql/type_check.py +++ /dev/null @@ -1,238 +0,0 @@ -from __future__ import annotations - -from datetime import date, datetime, time -from decimal import Decimal -from typing import Any, cast -from uuid import UUID - -from sqlalchemy import Boolean, Date, DateTime, Float, Integer, Numeric, String, Time -from sqlalchemy.sql.elements import ColumnElement -from sqlalchemy.sql.sqltypes import NullType -from sqlalchemy.types import TypeEngine - -from sqlrules.constraints import type_spec -from sqlrules.errors import UnsupportedConstraintError -from sqlrules.ir import CompilationContext, Constraint, TypeSpec - -_INT_TEXT = r"^[+-]?(0|[1-9][0-9]*)$" -_FLOAT_TEXT = r"^[+-]?([0-9]+(\.[0-9]*)?|\.[0-9]+)([eE][+-]?[0-9]+)?$" -_DATE_TEXT = r"^[0-9]{4}-[0-9]{2}-[0-9]{2}$" -_DATETIME_TEXT = ( - r"^[0-9]{4}-[0-9]{2}-[0-9]{2}[ T][0-9]{2}:[0-9]{2}:[0-9]{2}" - r"(\.[0-9]+)?([+-][0-9]{2}:?[0-9]{2}|Z)?$" -) -_TIME_TEXT = r"^[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?$" -_UUID_TEXT = ( - r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-" - r"[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" -) - - -def _is_type(column: ColumnElement[Any], *bases: type[TypeEngine[Any]]) -> bool: - col_type = column.type - if isinstance(col_type, NullType): - return False - return isinstance(col_type, bases) - - -def _regexp(column: ColumnElement[Any], pattern: str) -> ColumnElement[bool]: - return cast(ColumnElement[bool], column.op("REGEXP")(pattern)) - - -def _wrap_none( - column: ColumnElement[Any], - predicate: ColumnElement[bool], - *, - allow_none: bool, -) -> ColumnElement[bool]: - if allow_none: - return cast(ColumnElement[bool], column.is_(None) | predicate) - return predicate - - -def _unsupported(field: str, spec: TypeSpec, suggestion: str) -> None: - raise UnsupportedConstraintError( - field=field, - operator="type_check", - value=spec, - suggestion=suggestion, - ) - - -def _predicate_int( - column: ColumnElement[Any], - spec: TypeSpec, - field: str, -) -> ColumnElement[bool]: - if _is_type(column, Integer): - return cast(ColumnElement[bool], column.isnot(None)) - if spec.strict: - _unsupported( - field, - spec, - "MySQL strict int type_check requires an Integer column.", - ) - if _is_type(column, String): - return _regexp(column, _INT_TEXT) - if _is_type(column, Float, Numeric): - return cast(ColumnElement[bool], column == column.cast(Integer).cast(Float)) - _unsupported( - field, - spec, - "MySQL lax int type_check supports Integer, String, Float, or Numeric columns.", - ) - raise AssertionError("unreachable") - - -def _predicate_bool( - column: ColumnElement[Any], - spec: TypeSpec, - field: str, -) -> ColumnElement[bool]: - if spec.strict: - if _is_type(column, Boolean, Integer): - return cast(ColumnElement[bool], column.in_((True, False, 0, 1))) - _unsupported( - field, - spec, - "MySQL strict bool type_check requires a Boolean/Integer column.", - ) - _unsupported( - field, - spec, - "MySQL lax bool type_check is not supported. Use strict=True.", - ) - raise AssertionError("unreachable") - - -def _predicate_str( - column: ColumnElement[Any], - spec: TypeSpec, - field: str, -) -> ColumnElement[bool]: - if _is_type(column, String): - return cast(ColumnElement[bool], column.isnot(None)) - _unsupported( - field, - spec, - "MySQL str type_check requires a String/Text column.", - ) - raise AssertionError("unreachable") - - -def _predicate_float( - column: ColumnElement[Any], - spec: TypeSpec, - field: str, -) -> ColumnElement[bool]: - if _is_type(column, Float, Numeric, Integer): - return cast(ColumnElement[bool], column.isnot(None)) - if spec.strict: - _unsupported( - field, - spec, - "MySQL strict float/Decimal type_check requires a numeric column.", - ) - if _is_type(column, String): - return _regexp(column, _FLOAT_TEXT) - _unsupported( - field, - spec, - "MySQL float/Decimal type_check supports numeric or String columns.", - ) - raise AssertionError("unreachable") - - -def _predicate_temporal( - column: ColumnElement[Any], - spec: TypeSpec, - field: str, - *, - sa_types: tuple[type[TypeEngine[Any]], ...], - text_pattern: str, -) -> ColumnElement[bool]: - if _is_type(column, *sa_types): - return cast(ColumnElement[bool], column.isnot(None)) - if spec.strict: - type_name = spec.python_type.__name__ - _unsupported( - field, - spec, - f"MySQL strict {type_name} type_check requires a typed {type_name} column.", - ) - if _is_type(column, String): - return _regexp(column, text_pattern) - type_name = spec.python_type.__name__ - _unsupported( - field, - spec, - f"MySQL {type_name} type_check supports typed {type_name} or String columns.", - ) - raise AssertionError("unreachable") - - -def _predicate_uuid( - column: ColumnElement[Any], - spec: TypeSpec, - field: str, -) -> ColumnElement[bool]: - if "uuid" in type(column.type).__name__.lower(): - return cast(ColumnElement[bool], column.isnot(None)) - if spec.strict: - _unsupported( - field, - spec, - "MySQL strict UUID type_check requires a UUID column.", - ) - if _is_type(column, String): - return _regexp(column, _UUID_TEXT) - _unsupported( - field, - spec, - "MySQL UUID type_check supports UUID or String columns.", - ) - raise AssertionError("unreachable") - - -def _build_predicate( - column: ColumnElement[Any], - spec: TypeSpec, - field: str, -) -> ColumnElement[bool]: - python_type = spec.python_type - if python_type is int: - return _predicate_int(column, spec, field) - if python_type is bool: - return _predicate_bool(column, spec, field) - if python_type is str: - return _predicate_str(column, spec, field) - if python_type in {float, Decimal}: - return _predicate_float(column, spec, field) - if python_type is date: - return _predicate_temporal(column, spec, field, sa_types=(Date,), text_pattern=_DATE_TEXT) - if python_type is datetime: - return _predicate_temporal( - column, spec, field, sa_types=(DateTime,), text_pattern=_DATETIME_TEXT - ) - if python_type is time: - return _predicate_temporal(column, spec, field, sa_types=(Time,), text_pattern=_TIME_TEXT) - if python_type is UUID: - return _predicate_uuid(column, spec, field) - _unsupported( - field, - spec, - f"MySQL type_check has no translator for {python_type!r}.", - ) - raise AssertionError("unreachable") - - -def translate_type_check( - constraint: Constraint, - column: ColumnElement[Any], - context: CompilationContext, -) -> ColumnElement[bool]: - """Translate ``type_check`` into MySQL shape/type predicates.""" - del context - spec = type_spec(constraint.value) - predicate = _build_predicate(column, spec, constraint.field) - return _wrap_none(column, predicate, allow_none=spec.allow_none) diff --git a/packages/sqlrules-mysql/tests/test_mysql_plugin.py b/packages/sqlrules-mysql/tests/test_mysql_plugin.py index b7b9bd9..154a05a 100644 --- a/packages/sqlrules-mysql/tests/test_mysql_plugin.py +++ b/packages/sqlrules-mysql/tests/test_mysql_plugin.py @@ -1,128 +1,56 @@ from __future__ import annotations -import re from typing import Annotated, Any -from pydantic import BaseModel, Field -from sqlalchemy import Column, MetaData, String, Table, Text -from sqlalchemy.dialects import mysql +from pydantic import Field +from sqlalchemy import JSON, Column, MetaData, String, Table +from sqlalchemy.dialects.mysql import dialect from sqlrules_mysql import MysqlPlugin, __version__ -from sqlrules import Compiler, FullTextMatch, JsonContains, JsonHasKey +from sqlrules import Compiler, FullTextMatch, JsonContains, RuleSchema from sqlrules.conformance import run_basic_conformance -def _sql(expr: object, *, literal_binds: bool = True) -> str: - return str( - expr.compile( # type: ignore[union-attr] - dialect=mysql.dialect(), - compile_kwargs={"literal_binds": literal_binds} if literal_binds else {}, - ) - ) - - -def test_version() -> None: - assert __version__ == "1.0.1" - - -def test_conformance() -> None: +def test_version_and_plugin_conformance() -> None: + assert __version__ == "2.0.0" run_basic_conformance(MysqlPlugin(), operator="pattern") -def test_pattern_and_json_compile() -> None: - """MySQL REGEXP is CI for non-binary collations; ignore_case is intentionally unused.""" - - class Filter(BaseModel): +def test_pattern_json_and_fulltext_constraints_compile() -> None: + class Rules(RuleSchema): name: Annotated[str, Field(pattern=r"^A")] - meta: Annotated[dict[str, Any], JsonContains({"active": True}), JsonHasKey("active")] - body: Annotated[str, FullTextMatch("hello")] + meta: Annotated[dict[str, Any], JsonContains({"active": True})] + body: Annotated[str, FullTextMatch("sqlrules")] table = Table( - "items", + "rows", MetaData(), Column("name", String), - Column("meta", Text), - Column("body", Text), - ) - rules = Compiler( - plugins=[MysqlPlugin()], - dialect="mysql", - cache=False, - ).compile(Filter, table) - - assert _sql(rules["name"][0]) == "items.name REGEXP '^A'" - - contains_sql = _sql(rules["meta"][0]).lower() - assert "json_contains(items.meta" in contains_sql - assert '{"active":true}' in contains_sql.replace(" ", "") - assert "json_contains_path" not in contains_sql - - has_key_sql = _sql(rules["meta"][1]).lower() - assert "json_contains_path(items.meta" in has_key_sql - assert '$."active"' in has_key_sql or "$" in has_key_sql - - body_sql = _sql(rules["body"][0]).lower() - assert "match" in body_sql and "against" in body_sql - assert "hello" in body_sql - - -def test_pattern_ignore_case_does_not_invent_binary() -> None: - """Case-insensitive PatternSpec must not invent REGEXP BINARY (collation owns CI).""" - - class Filter(BaseModel): - name: Annotated[str, Field(pattern=re.compile(r"^A", re.I))] - - table = Table("items", MetaData(), Column("name", String)) - rules = Compiler( - plugins=[MysqlPlugin()], - dialect="mysql", - cache=False, - ).compile(Filter, table) - compiled = _sql(rules["name"][0]) - assert compiled == "items.name REGEXP '^A'" - assert "BINARY" not in compiled.upper() - assert "(?i)" not in compiled - - -def test_type_check_int_and_lax_string() -> None: - from sqlalchemy import Integer - - class Typed(BaseModel): - age: int - - class Textual(BaseModel): - age: int - - typed_table = Table("users", MetaData(), Column("age", Integer)) - text_table = Table("rows", MetaData(), Column("age", String)) - compiler = Compiler( - plugins=[MysqlPlugin()], - dialect="mysql", - emit_type_checks=True, - cache=False, + Column("meta", JSON), + Column("body", String), ) - typed_sql = _sql(compiler.compile(Typed, typed_table)["age"][0], literal_binds=False) - text_sql = _sql(compiler.compile(Textual, text_table)["age"][0]) - assert "users.age" in typed_sql or "`users`.age" in typed_sql - assert "IS NOT NULL" in typed_sql.upper() - assert "age REGEXP" in text_sql.replace("`", "") - assert "REGEXP" in text_sql - assert "^[+-]?(0|[1-9][0-9]*)$" in text_sql - - -def test_type_check_optional_allow_none() -> None: - from sqlalchemy import Integer - - class Filter(BaseModel): - age: int | None = None - - table = Table("users", MetaData(), Column("age", Integer)) - rules = Compiler( - plugins=[MysqlPlugin()], - dialect="mysql", - emit_type_checks=True, - cache=False, - ).compile(Filter, table) - compiled = _sql(rules["age"][0], literal_binds=False).upper() - assert "IS NULL" in compiled - assert "IS NOT NULL" in compiled + compiled = Compiler(plugins=[MysqlPlugin(server_version=(8, 0, 36))]).compile(Rules, table) + statement = compiled.predicate.compile(dialect=dialect()) + sql = str(statement).lower() + assert "regexp_like" in sql + assert "c" in statement.params.values() + assert "json_contains" in sql + assert "match" in sql + + +def test_integer_text_profile_is_server_version_gated() -> None: + class Rules(RuleSchema): + count: int + + table = Table("rows", MetaData(), Column("count", String)) + from sqlrules import CapabilityError + + try: + Compiler(plugins=[MysqlPlugin()]).compile(Rules, table) + except CapabilityError as exc: + assert "MySQL 8.0+" in str(exc) + else: # pragma: no cover - assertion is the capability contract + raise AssertionError("text-to-integer needs a configured MySQL version") + + compiled = Compiler(plugins=[MysqlPlugin(server_version=(8, 0, 36))]).compile(Rules, table) + assert compiled.fields[0].coercion == "text-to-int" diff --git a/packages/sqlrules-postgresql/README.md b/packages/sqlrules-postgresql/README.md index 0dc9b26..b350349 100644 --- a/packages/sqlrules-postgresql/README.md +++ b/packages/sqlrules-postgresql/README.md @@ -1,76 +1,61 @@ # sqlrules-postgresql -PostgreSQL dialect plugin for [SQLRules](https://github.com/eddiethedean/sqlrules). +PostgreSQL backend provider for [SQLRules](https://github.com/eddiethedean/sqlrules). +The package version follows the core 2.x line. ## Install ```bash -pip install sqlrules-postgresql +pip install "sqlrules>=2,<3" "sqlrules-postgresql>=2,<3" ``` -## Usage +## Use ```python -import re from typing import Annotated, Any -from pydantic import BaseModel, Field -from sqlalchemy import Column, MetaData, Table -from sqlalchemy.dialects.postgresql import ARRAY, INT4RANGE, JSONB, TEXT +from pydantic import Field +from sqlalchemy import Column, MetaData, String, Table +from sqlalchemy.dialects.postgresql import ARRAY, INT4RANGE, JSONB -from sqlrules import ArrayContains, Compiler, JsonContains, RangeContains +from sqlrules import ArrayContains, Compiler, JsonContains, RangeContains, RuleSchema, where from sqlrules_postgresql import PostgresPlugin -class RowFilter(BaseModel): - name: Annotated[str, Field(pattern=re.compile(r"^a", re.I))] - meta: Annotated[dict[str, Any], JsonContains({"active": True})] - tags: Annotated[list[str], ArrayContains(["admin"])] - span: Annotated[int, RangeContains(5)] - -table = Table( +rows = Table( "rows", MetaData(), - Column("name", TEXT), + Column("name", String), Column("meta", JSONB), - Column("tags", ARRAY(TEXT)), + Column("tags", ARRAY(String)), Column("span", INT4RANGE), ) -compiler = Compiler(plugins=[PostgresPlugin()], dialect="postgresql") -rules = compiler.compile(RowFilter, table) -``` -## Operators +class RowRules(RuleSchema): + name: Annotated[str, Field(pattern=r"^A")] + meta: Annotated[dict[str, Any], JsonContains({"active": True})] + tags: Annotated[list[str], ArrayContains(["admin"])] + span: Annotated[int, RangeContains(5)] -| IR operator | SQLAlchemy / PostgreSQL | -|---|---| -| `pattern` | `~` or `~*` (when `PatternSpec.ignore_case`) | -| `type_check` | Shape/type predicates from `TypeSpec` (see matrix below) | -| `json_contains` | JSONB `contains` / `@>` | -| `json_has_key` | JSONB `has_key` / `?` | -| `array_contains` | array `contains` | -| `array_overlap` | array `overlap` / `&&` | -| `range_contains` | range `@>` | -| `range_overlap` | range `&&` | -### `type_check` matrix (approximate) +compiled = Compiler(plugins=[PostgresPlugin(server_version=(16, 0))]).compile(RowRules, rows) +statement = rows.select().where(*where(compiled)) +``` -Enable with `Compiler(..., emit_type_checks=True)`. Not full Pydantic -parity — inexpressible pairs raise. +## Capabilities -| Python type | Lax | Strict | -|---|---|---| -| `int` | Integer column; String `~` digit pattern; numeric whole-number | Integer column only | -| `bool` | unsupported (raise) | Boolean `IN (true, false)` | -| `str` | String/Text `IS NOT NULL` | same | -| `float` / `Decimal` | numeric column; String float-ish `~` | numeric column | -| `date` / `datetime` / `time` / `UUID` | typed column or String format `~` | typed column | +- `pattern`: PostgreSQL `~` / `~*` +- JSONB containment and key membership +- ARRAY containment and overlap +- Range containment and overlap +- Safe lax text-to-int/float/Decimal parsing on PostgreSQL 16+ -`Optional[T]` → `(column IS NULL) OR `. +String Literal and Enum fields require a column with `C` or `POSIX` +collation. Review the [type support matrix](https://sqlrules.readthedocs.io/en/latest/TYPE_SUPPORT.html) +for the exact conversion profile and limitations. -## Security note +## Pattern cost -`pattern` becomes a PostgreSQL regex (`~` / `~*`). Untrusted pattern strings -can cause expensive engine-side evaluation (ReDoS-class cost). Prefer -static/allowlisted patterns. See -[SECURITY](https://sqlrules.readthedocs.io/en/latest/SECURITY.html). +Untrusted regular expressions can cause expensive engine-side evaluation. +Prefer patterns authored with the rule model. See the SQLRules +[security notes](https://sqlrules.readthedocs.io/en/latest/SECURITY.html). diff --git a/packages/sqlrules-postgresql/pyproject.toml b/packages/sqlrules-postgresql/pyproject.toml index 3fcde29..3a54632 100644 --- a/packages/sqlrules-postgresql/pyproject.toml +++ b/packages/sqlrules-postgresql/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "sqlrules-postgresql" -version = "1.0.1" +version = "2.0.0" description = "PostgreSQL dialect plugin for SQLRules (regex, JSONB, ARRAY, range)." readme = "README.md" requires-python = ">=3.10" @@ -18,7 +18,7 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "sqlrules>=1,<2", + "sqlrules>=2,<3", "sqlalchemy>=2.0,<3", ] diff --git a/packages/sqlrules-postgresql/src/sqlrules_postgresql/__init__.py b/packages/sqlrules-postgresql/src/sqlrules_postgresql/__init__.py index 18fddd1..50d60cc 100644 --- a/packages/sqlrules-postgresql/src/sqlrules_postgresql/__init__.py +++ b/packages/sqlrules-postgresql/src/sqlrules_postgresql/__init__.py @@ -1,14 +1,21 @@ from __future__ import annotations +from collections.abc import Mapping +from typing import Any + +from sqlalchemy.dialects.postgresql import JSONB + +from sqlrules.backend import prepare_scalar +from sqlrules.errors import CapabilityError +from sqlrules.ir import CompilationContext, PreparedValue, RuleField from sqlrules.plugins import PLUGIN_API_VERSION from sqlrules.translators import TranslatorRegistry from sqlrules_postgresql.array import translate_array_contains, translate_array_overlap from sqlrules_postgresql.jsonb import translate_json_contains, translate_json_has_key from sqlrules_postgresql.pattern import translate_pattern from sqlrules_postgresql.range import translate_range_contains, translate_range_overlap -from sqlrules_postgresql.type_check import translate_type_check -__version__ = "1.0.1" +__version__ = "2.0.0" class PostgresPlugin: @@ -17,17 +24,63 @@ class PostgresPlugin: name = "postgresql" api_version = PLUGIN_API_VERSION + def __init__(self, *, server_version: tuple[int, ...] | None = None) -> None: + self.server_version = tuple(server_version) if server_version is not None else None + + def capabilities(self) -> Mapping[str, Any]: + return { + "backend": self.name, + "server_version": self.server_version, + "native_scalar_types": ( + "bool", + "int", + "float", + "decimal", + "str", + "date", + "datetime", + "time", + "uuid", + ), + "safe_text_numeric_conversion": self.server_version is not None + and self.server_version >= (16, 0), + "safe_text_numeric_targets": ( + ("int", "float", "decimal") + if self.server_version is not None and self.server_version >= (16, 0) + else () + ), + "assumptions": ("PostgreSQL native column types provide logical type evidence.",), + } + + def prepare_value( + self, + column: Any, + field: RuleField, + context: CompilationContext, + ) -> PreparedValue: + if ( + field.python_type is dict + and any( + item.operator in {"json_contains", "json_has_key"} for item in field.constraints + ) + and not isinstance(column.type, JSONB) + ): + raise CapabilityError( + self.name, + field.name, + "jsonb", + type(column.type).__name__, + "PostgreSQL JSON markers require a JSONB column; generic JSON does not provide " + "the containment and key operators used by SQLRules.", + ) + return prepare_scalar(column, field, context, backend=self.name) + def register(self, registry: TranslatorRegistry) -> None: registry.register_constraint( "pattern", translate_pattern, on_conflict="replace", ) - registry.register_constraint( - "type_check", - translate_type_check, - on_conflict="replace", - ) for operator, translator in ( ("json_contains", translate_json_contains), ("json_has_key", translate_json_has_key), @@ -49,5 +102,4 @@ def register(self, registry: TranslatorRegistry) -> None: "translate_pattern", "translate_range_contains", "translate_range_overlap", - "translate_type_check", ] diff --git a/packages/sqlrules-postgresql/src/sqlrules_postgresql/jsonb.py b/packages/sqlrules-postgresql/src/sqlrules_postgresql/jsonb.py index 0f96311..0d869c5 100644 --- a/packages/sqlrules-postgresql/src/sqlrules_postgresql/jsonb.py +++ b/packages/sqlrules-postgresql/src/sqlrules_postgresql/jsonb.py @@ -2,6 +2,7 @@ from typing import Any, cast +from sqlalchemy import func from sqlalchemy.sql.elements import ColumnElement from sqlrules.ir import CompilationContext, Constraint @@ -22,4 +23,7 @@ def translate_json_has_key( context: CompilationContext, ) -> ColumnElement[bool]: """Translate ``json_has_key`` to JSONB ``?`` / ``has_key``.""" - return cast(ColumnElement[bool], column.has_key(constraint.value)) + return cast( + ColumnElement[bool], + (func.jsonb_typeof(column) == "object") & column.has_key(constraint.value), + ) diff --git a/packages/sqlrules-postgresql/src/sqlrules_postgresql/type_check.py b/packages/sqlrules-postgresql/src/sqlrules_postgresql/type_check.py deleted file mode 100644 index a19eeab..0000000 --- a/packages/sqlrules-postgresql/src/sqlrules_postgresql/type_check.py +++ /dev/null @@ -1,250 +0,0 @@ -from __future__ import annotations - -from datetime import date, datetime, time -from decimal import Decimal -from typing import Any, cast -from uuid import UUID - -from sqlalchemy import Boolean, Date, DateTime, Float, Integer, Numeric, String, Time -from sqlalchemy.sql.elements import ColumnElement -from sqlalchemy.sql.sqltypes import NullType -from sqlalchemy.types import TypeEngine - -from sqlrules.constraints import type_spec -from sqlrules.errors import UnsupportedConstraintError -from sqlrules.ir import CompilationContext, Constraint, TypeSpec - -# Integer-like text (Pydantic lax int accepts digit strings). -_INT_TEXT = r"^[+-]?(0|[1-9]\d*)$" -# Float / Decimal-like text (simplified; not full Pydantic parity). -_FLOAT_TEXT = r"^[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?$" -# ISO-ish date / datetime / time (approximate). -_DATE_TEXT = r"^\d{4}-\d{2}-\d{2}$" -_DATETIME_TEXT = r"^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}(\.\d+)?([+-]\d{2}:?\d{2}|Z)?$" -_TIME_TEXT = r"^\d{2}:\d{2}:\d{2}(\.\d+)?$" -_UUID_TEXT = ( - r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-" - r"[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" -) - - -def _col_type(column: ColumnElement[Any]) -> TypeEngine[Any]: - return column.type - - -def _is_type(column: ColumnElement[Any], *bases: type[TypeEngine[Any]]) -> bool: - col_type = _col_type(column) - if isinstance(col_type, NullType): - return False - return isinstance(col_type, bases) - - -def _regex(column: ColumnElement[Any], pattern: str) -> ColumnElement[bool]: - return cast(ColumnElement[bool], column.op("~")(pattern)) - - -def _wrap_none( - column: ColumnElement[Any], - predicate: ColumnElement[bool], - *, - allow_none: bool, -) -> ColumnElement[bool]: - if allow_none: - return cast(ColumnElement[bool], column.is_(None) | predicate) - return predicate - - -def _unsupported(field: str, spec: TypeSpec, suggestion: str) -> None: - raise UnsupportedConstraintError( - field=field, - operator="type_check", - value=spec, - suggestion=suggestion, - ) - - -def _predicate_int( - column: ColumnElement[Any], - spec: TypeSpec, - field: str, -) -> ColumnElement[bool]: - if _is_type(column, Integer): - # Typed integer column: every non-null value is already an int. - return cast(ColumnElement[bool], column.isnot(None)) - if spec.strict: - _unsupported( - field, - spec, - "PostgreSQL strict int type_check requires an Integer column " - "(Pydantic strict mode rejects string/bool coercion).", - ) - if _is_type(column, String): - return _regex(column, _INT_TEXT) - if _is_type(column, Float, Numeric): - # Whole numbers only (approximate Pydantic float→int when no fractional part). - return cast(ColumnElement[bool], column == column.cast(Integer).cast(Float)) - _unsupported( - field, - spec, - "PostgreSQL lax int type_check supports Integer, String, Float, or Numeric columns.", - ) - raise AssertionError("unreachable") - - -def _predicate_bool( - column: ColumnElement[Any], - spec: TypeSpec, - field: str, -) -> ColumnElement[bool]: - if spec.strict: - if _is_type(column, Boolean): - return cast(ColumnElement[bool], column.in_((True, False))) - _unsupported( - field, - spec, - "PostgreSQL strict bool type_check requires a Boolean column.", - ) - _unsupported( - field, - spec, - "PostgreSQL lax bool type_check is not supported (coercion set is not " - "deterministically expressible). Use Field(strict=True) or model_config " - "strict=True with a Boolean column.", - ) - raise AssertionError("unreachable") - - -def _predicate_str( - column: ColumnElement[Any], - spec: TypeSpec, - field: str, -) -> ColumnElement[bool]: - if _is_type(column, String): - return cast(ColumnElement[bool], column.isnot(None)) - _unsupported( - field, - spec, - "PostgreSQL str type_check requires a String/Text column " - "(Pydantic does not coerce other SQL types to str by default).", - ) - raise AssertionError("unreachable") - - -def _predicate_float( - column: ColumnElement[Any], - spec: TypeSpec, - field: str, -) -> ColumnElement[bool]: - if _is_type(column, Float, Numeric, Integer): - return cast(ColumnElement[bool], column.isnot(None)) - if spec.strict: - _unsupported( - field, - spec, - "PostgreSQL strict float/Decimal type_check requires a numeric column.", - ) - if _is_type(column, String): - return _regex(column, _FLOAT_TEXT) - _unsupported( - field, - spec, - "PostgreSQL float/Decimal type_check supports numeric or String columns.", - ) - raise AssertionError("unreachable") - - -def _predicate_temporal( - column: ColumnElement[Any], - spec: TypeSpec, - field: str, - *, - sa_types: tuple[type[TypeEngine[Any]], ...], - text_pattern: str, -) -> ColumnElement[bool]: - if _is_type(column, *sa_types): - return cast(ColumnElement[bool], column.isnot(None)) - if spec.strict: - type_name = spec.python_type.__name__ - _unsupported( - field, - spec, - f"PostgreSQL strict {type_name} type_check requires a typed {type_name} column.", - ) - if _is_type(column, String): - return _regex(column, text_pattern) - type_name = spec.python_type.__name__ - _unsupported( - field, - spec, - f"PostgreSQL {type_name} type_check supports typed {type_name} or String columns.", - ) - raise AssertionError("unreachable") - - -def _predicate_uuid( - column: ColumnElement[Any], - spec: TypeSpec, - field: str, -) -> ColumnElement[bool]: - col_type = _col_type(column) - type_name = type(col_type).__name__.lower() - if "uuid" in type_name: - return cast(ColumnElement[bool], column.isnot(None)) - if spec.strict: - _unsupported( - field, - spec, - "PostgreSQL strict UUID type_check requires a UUID column.", - ) - if _is_type(column, String): - return _regex(column, _UUID_TEXT) - _unsupported( - field, - spec, - "PostgreSQL UUID type_check supports UUID or String columns.", - ) - raise AssertionError("unreachable") - - -def _build_predicate( - column: ColumnElement[Any], - spec: TypeSpec, - field: str, -) -> ColumnElement[bool]: - python_type = spec.python_type - if python_type is int: - return _predicate_int(column, spec, field) - if python_type is bool: - return _predicate_bool(column, spec, field) - if python_type is str: - return _predicate_str(column, spec, field) - if python_type in {float, Decimal}: - return _predicate_float(column, spec, field) - if python_type is date: - return _predicate_temporal(column, spec, field, sa_types=(Date,), text_pattern=_DATE_TEXT) - if python_type is datetime: - return _predicate_temporal( - column, spec, field, sa_types=(DateTime,), text_pattern=_DATETIME_TEXT - ) - if python_type is time: - return _predicate_temporal(column, spec, field, sa_types=(Time,), text_pattern=_TIME_TEXT) - if python_type is UUID: - return _predicate_uuid(column, spec, field) - _unsupported( - field, - spec, - f"PostgreSQL type_check has no translator for {python_type!r}.", - ) - raise AssertionError("unreachable") - - -def translate_type_check( - constraint: Constraint, - column: ColumnElement[Any], - context: CompilationContext, -) -> ColumnElement[bool]: - """Translate ``type_check`` into a PostgreSQL shape/type predicate.""" - del context # dialect hint unused; plugin is already PostgreSQL-specific - spec = type_spec(constraint.value) - predicate = _build_predicate(column, spec, constraint.field) - return _wrap_none(column, predicate, allow_none=spec.allow_none) diff --git a/packages/sqlrules-postgresql/tests/test_postgres_plugin.py b/packages/sqlrules-postgresql/tests/test_postgres_plugin.py index 1f8ca6b..ab19070 100644 --- a/packages/sqlrules-postgresql/tests/test_postgres_plugin.py +++ b/packages/sqlrules-postgresql/tests/test_postgres_plugin.py @@ -3,211 +3,56 @@ import re from typing import Annotated, Any -from pydantic import BaseModel, Field +from pydantic import Field from sqlalchemy import Column, MetaData, String, Table -from sqlalchemy.dialects import postgresql -from sqlalchemy.dialects.postgresql import ARRAY, INT4RANGE, JSONB, TEXT +from sqlalchemy.dialects.postgresql import ARRAY, INT4RANGE, JSONB, dialect from sqlrules_postgresql import PostgresPlugin, __version__ -from sqlrules import ( - ArrayContains, - ArrayOverlap, - Compiler, - JsonContains, - JsonHasKey, - RangeContains, - RangeOverlap, -) +from sqlrules import ArrayContains, Compiler, JsonContains, RangeContains, RuleSchema from sqlrules.conformance import run_basic_conformance -def _sql(expr: object, *, literal_binds: bool = True) -> str: - return str( - expr.compile( # type: ignore[union-attr] - dialect=postgresql.dialect(), - compile_kwargs={"literal_binds": literal_binds} if literal_binds else {}, - ) - ) - - -def test_version() -> None: - assert __version__ == "1.0.1" - - -def test_conformance() -> None: - run_basic_conformance(PostgresPlugin(), operator="pattern") - - -def test_pattern_compiles() -> None: - """PG ``~`` is substring search; Pydantic validates fullmatch — patterns often use ^/$.""" - - class Filter(BaseModel): - name: Annotated[str, Field(pattern=r"^A")] - - table = Table("items", MetaData(), Column("name", String)) - rules = Compiler( - plugins=[PostgresPlugin()], - dialect="postgresql", - cache=False, - ).compile(Filter, table) - assert _sql(rules["name"][0]) == "items.name ~ '^A'" - - -def test_pattern_ignore_case_uses_star() -> None: - class Filter(BaseModel): - name: Annotated[str, Field(pattern=re.compile(r"^A", re.I))] +def test_version_and_plugin_conformance() -> None: + assert __version__ == "2.0.0" + run_basic_conformance(PostgresPlugin(server_version=(16, 0)), operator="pattern") - table = Table("items", MetaData(), Column("name", String)) - rules = Compiler( - plugins=[PostgresPlugin()], - dialect="postgresql", - cache=False, - ).compile(Filter, table) - assert _sql(rules["name"][0]) == "items.name ~* '^A'" - -def test_unanchored_pattern_emits_search_semantics() -> None: - """Unanchored patterns are emitted as-is (PG ``~`` search, not Pydantic fullmatch).""" - - class Filter(BaseModel): - name: Annotated[str, Field(pattern=r"abc")] - - table = Table("items", MetaData(), Column("name", String)) - rules = Compiler( - plugins=[PostgresPlugin()], - dialect="postgresql", - cache=False, - ).compile(Filter, table) - assert _sql(rules["name"][0]) == "items.name ~ 'abc'" - - -def test_json_array_range_operators() -> None: - class Filter(BaseModel): - meta: Annotated[dict[str, Any], JsonContains({"active": True}), JsonHasKey("active")] - tags: Annotated[list[str], ArrayContains(["admin"]), ArrayOverlap(["x"])] - span: Annotated[int, RangeContains(5), RangeOverlap([1, 10])] +def test_pattern_json_array_and_range_compile_with_v2_provider() -> None: + class Rules(RuleSchema): + name: Annotated[str, Field(pattern=re.compile(r"^a", re.I))] + meta: Annotated[dict[str, Any], JsonContains({"active": True})] + tags: Annotated[list[str], ArrayContains(["admin"])] + span: Annotated[int, RangeContains(5)] table = Table( "rows", MetaData(), + Column("name", String), Column("meta", JSONB), - Column("tags", ARRAY(TEXT)), + Column("tags", ARRAY(String)), Column("span", INT4RANGE), ) - rules = Compiler( - plugins=[PostgresPlugin()], - dialect="postgresql", - cache=False, - ).compile(Filter, table) - - # JSONB containment: operator + bound payload (literal_binds unsupported for JSONB). - contains = rules["meta"][0] - assert getattr(contains.operator, "opstring", None) == "@>" # type: ignore[attr-defined] - assert contains.right.value == {"active": True} # type: ignore[attr-defined] - assert "@>" in _sql(contains, literal_binds=False) - - assert _sql(rules["meta"][1]) == "rows.meta ? 'active'" - assert _sql(rules["tags"][0]) == "rows.tags @> ARRAY['admin']" - assert _sql(rules["tags"][1]) == "rows.tags && ARRAY['x']" - assert _sql(rules["span"][0]) == "rows.span @> 5" - overlap = rules["span"][1] - assert getattr(overlap.operator, "opstring", None) == "&&" # type: ignore[attr-defined] - assert overlap.right.value == [1, 10] # type: ignore[attr-defined] - - -def test_empty_json_contains_uses_containment() -> None: - class Filter(BaseModel): - meta: Annotated[dict[str, Any], JsonContains({})] - - table = Table("rows", MetaData(), Column("meta", JSONB)) - rules = Compiler( - plugins=[PostgresPlugin()], - dialect="postgresql", - cache=False, - ).compile(Filter, table) - expr = rules["meta"][0] - assert getattr(expr.operator, "opstring", None) == "@>" # type: ignore[attr-defined] - assert expr.right.value == {} # type: ignore[attr-defined] - assert "@>" in _sql(expr, literal_binds=False) - - -def test_type_check_int_and_str() -> None: - from sqlalchemy import Integer - - class Filter(BaseModel): - age: int - name: str - - table = Table( - "users", - MetaData(), - Column("age", Integer), - Column("name", String), - ) - rules = Compiler( - plugins=[PostgresPlugin()], - dialect="postgresql", - emit_type_checks=True, - cache=False, - ).compile(Filter, table) - assert _sql(rules["age"][0], literal_binds=False).upper().count("IS NOT NULL") == 1 - assert "users.age" in _sql(rules["age"][0], literal_binds=False) - assert "users.name" in _sql(rules["name"][0], literal_binds=False) - assert "IS NOT NULL" in _sql(rules["name"][0], literal_binds=False).upper() - - -def test_type_check_lax_int_on_string() -> None: - class Filter(BaseModel): - age: int - - table = Table("rows", MetaData(), Column("age", String)) - rules = Compiler( - plugins=[PostgresPlugin()], - dialect="postgresql", - emit_type_checks=True, - cache=False, - ).compile(Filter, table) - compiled = _sql(rules["age"][0]) - assert "rows.age" in compiled - assert "~" in compiled - # Lax int-on-text uses a digit regex, not a bare IS NOT NULL. - assert "IS NOT NULL" not in compiled.upper() or "~" in compiled - - -def test_type_check_strict_bool() -> None: - from pydantic import ConfigDict - from sqlalchemy import Boolean - - class Filter(BaseModel): - model_config = ConfigDict(strict=True) - active: bool - - table = Table("rows", MetaData(), Column("active", Boolean)) - rules = Compiler( - plugins=[PostgresPlugin()], - dialect="postgresql", - emit_type_checks=True, - cache=False, - ).compile(Filter, table) - compiled = _sql(rules["active"][0]) - assert "rows.active" in compiled - assert "IN" in compiled.upper() - assert "true" in compiled.lower() or "True" in compiled - - -def test_type_check_optional_or_null() -> None: - from sqlalchemy import Integer - - class Filter(BaseModel): - age: int | None = None - - table = Table("rows", MetaData(), Column("age", Integer)) - rules = Compiler( - plugins=[PostgresPlugin()], - dialect="postgresql", - emit_type_checks=True, - cache=False, - ).compile(Filter, table) - compiled = _sql(rules["age"][0], literal_binds=False).upper() - assert "IS NULL" in compiled - assert "IS NOT NULL" in compiled + compiled = Compiler(plugins=[PostgresPlugin(server_version=(16, 0))]).compile(Rules, table) + assert len(compiled.fields) == 4 + assert compiled.fields[-1].logical_type == "range" + sql = str(compiled.predicate.compile(dialect=dialect())) + assert "~*" in sql + assert "@>" in sql + assert "@>" in sql + + +def test_string_literal_requires_explicit_deterministic_collation() -> None: + from typing import Literal + + from sqlrules import CapabilityError + + class Rules(RuleSchema): + status: Literal["ready", "pending"] + + table = Table("rows", MetaData(), Column("status", String)) + try: + Compiler(plugins=[PostgresPlugin()]).compile(Rules, table) + except CapabilityError as exc: + assert "C or POSIX" in str(exc) + else: # pragma: no cover - assertion is the capability contract + raise AssertionError("string literal compilation needs explicit collation") diff --git a/packages/sqlrules-sqlite/README.md b/packages/sqlrules-sqlite/README.md index ed1ed14..87ce1d1 100644 --- a/packages/sqlrules-sqlite/README.md +++ b/packages/sqlrules-sqlite/README.md @@ -1,61 +1,62 @@ # sqlrules-sqlite -SQLite dialect plugin for [SQLRules](https://github.com/eddiethedean/sqlrules). +SQLite backend provider for [SQLRules](https://github.com/eddiethedean/sqlrules). +The package version follows the core 2.x line. ## Install ```bash -pip install sqlrules-sqlite +pip install "sqlrules>=2,<3" "sqlrules-sqlite>=2,<3" ``` -## Usage +## Use ```python -import sqlite3 from typing import Annotated, Any -from pydantic import BaseModel, Field -from sqlalchemy import Column, MetaData, String, Table, create_engine, text +from pydantic import Field +from sqlalchemy import Column, JSON, MetaData, String, Table, create_engine, event -from sqlrules import Compiler, JsonContains, JsonHasKey -from sqlrules_sqlite import SQLitePlugin, register_regexp +from sqlrules import Compiler, JsonContains, RuleSchema, where +from sqlrules_sqlite import SQLitePlugin, register_sqlite_functions -class RowFilter(BaseModel): - name: Annotated[str, Field(pattern=r"^A")] - meta: Annotated[dict[str, Any], JsonContains({"active": True}), JsonHasKey("active")] +rows = Table("rows", MetaData(), Column("name", String), Column("meta", JSON)) -table = Table( - "rows", - MetaData(), - Column("name", String), - Column("meta", String), -) -compiler = Compiler(plugins=[SQLitePlugin()], dialect="sqlite") -rules = compiler.compile(RowFilter, table) +class RowRules(RuleSchema): + name: Annotated[str, Field(pattern=r"^A")] + meta: Annotated[dict[str, Any], JsonContains({"active": True})] -engine = create_engine("sqlite://") -with engine.raw_connection() as conn: - # SQLAlchemy 2 may wrap the DBAPI connection; unwrap if needed. - dbapi = conn.driver_connection if hasattr(conn, "driver_connection") else conn - register_regexp(dbapi) + +engine = create_engine("sqlite:///app.db") +event.listen( + engine, + "connect", + lambda dbapi_connection, _: register_sqlite_functions(dbapi_connection), +) +compiled = Compiler(plugins=[SQLitePlugin()]).compile(RowRules, rows) +statement = rows.select().where(*where(compiled)) ``` -## Operators +## Capabilities -| IR operator | Notes | -|---|---| -| `pattern` | `column REGEXP pattern`; call `register_regexp(connection)` | -| `type_check` | `typeof` / `REGEXP` shape checks; text forms need `register_regexp` | -| `json_contains` | JSON1 `json_extract` equality for object keys | -| `json_has_key` | `json_type(column, '$.key') IS NOT NULL` | +- Runtime `typeof()` checks for SQLite integer, real, and text storage classes +- Lax text-to-int/float coercion and bool values stored as integer 0 or 1 +- JSON1 helpers that treat malformed documents as non-matches +- `REGEXP` pattern matching and Unicode-aware string length through + `register_sqlite_functions()` -Case-insensitive patterns (`re.IGNORECASE` / `PatternSpec.ignore_case`) are -encoded with a `(?i)` prefix understood by `register_regexp`. +Strict bool, exact Decimal, and SQLAlchemy-emulated date/time/UUID values need +explicit storage adapters and raise `CapabilityError`. Text coercion, patterns, +and string length constraints require the registered SQLRules SQLite functions +on each connection. `register_regexp()` remains as a backward-compatible alias. +See the +[type support matrix](https://sqlrules.readthedocs.io/en/latest/TYPE_SUPPORT.html). -## Security note +## Pattern cost -`register_regexp` installs a Python `re.search` UDF. Untrusted -`Field(pattern=...)` values can cause **CPU denial of service** (ReDoS) in -your process — not SQL injection. Prefer static/allowlisted patterns. See -[SECURITY](https://sqlrules.readthedocs.io/en/latest/SECURITY.html). +`register_sqlite_functions()` runs Python's `re.search` for each regex row and +Python's `len()` for each length-checked string row. Untrusted patterns can +cause CPU denial of service through catastrophic backtracking. Prefer static +or allowlisted patterns. See the SQLRules +[security notes](https://sqlrules.readthedocs.io/en/latest/SECURITY.html). diff --git a/packages/sqlrules-sqlite/pyproject.toml b/packages/sqlrules-sqlite/pyproject.toml index 6ebbcd5..60d13e6 100644 --- a/packages/sqlrules-sqlite/pyproject.toml +++ b/packages/sqlrules-sqlite/pyproject.toml @@ -4,8 +4,8 @@ build-backend = "hatchling.build" [project] name = "sqlrules-sqlite" -version = "1.0.1" -description = "SQLite dialect plugin for SQLRules (REGEXP helpers and JSON)." +version = "2.0.0" +description = "SQLite dialect plugin for SQLRules (REGEXP, Unicode length, and JSON helpers)." readme = "README.md" requires-python = ">=3.10" license = "MIT" @@ -18,7 +18,7 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "sqlrules>=1,<2", + "sqlrules>=2,<3", "sqlalchemy>=2.0,<3", ] diff --git a/packages/sqlrules-sqlite/src/sqlrules_sqlite/__init__.py b/packages/sqlrules-sqlite/src/sqlrules_sqlite/__init__.py index 4042619..66b4a58 100644 --- a/packages/sqlrules-sqlite/src/sqlrules_sqlite/__init__.py +++ b/packages/sqlrules-sqlite/src/sqlrules_sqlite/__init__.py @@ -1,35 +1,75 @@ from __future__ import annotations +from collections.abc import Mapping +from typing import Any + +from sqlrules.backend import prepare_sqlite_scalar +from sqlrules.ir import CompilationContext, PreparedValue, RuleField from sqlrules.plugins import PLUGIN_API_VERSION from sqlrules.translators import TranslatorRegistry from sqlrules_sqlite.json import translate_json_contains, translate_json_has_key +from sqlrules_sqlite.length import translate_max_length, translate_min_length from sqlrules_sqlite.pattern import translate_pattern from sqlrules_sqlite.regexp import register_regexp -from sqlrules_sqlite.type_check import translate_type_check +from sqlrules_sqlite.runtime import register_sqlite_functions -__version__ = "1.0.1" +__version__ = "2.0.0" class SQLitePlugin: """Register SQLite-specific constraint translators. - The ``pattern`` and text-shaped ``type_check`` translators emit - ``column REGEXP pattern``. Call :func:`register_regexp` on each SQLite - connection before executing the resulting SQL. + Pattern and text-coercion expressions emit ``column REGEXP pattern``; + length constraints use SQLRules' Unicode-aware character counter. Call + :func:`register_sqlite_functions` on each SQLite connection before + executing SQL that needs these functions. ``register_regexp`` remains as a + backward-compatible alias. """ name = "sqlite" api_version = PLUGIN_API_VERSION + def __init__(self, *, server_version: tuple[int, ...] | None = None) -> None: + self.server_version = tuple(server_version) if server_version is not None else None + + def capabilities(self) -> Mapping[str, Any]: + return { + "backend": self.name, + "server_version": self.server_version, + "native_scalar_types": ("bool", "int", "float", "str"), + "safe_text_numeric_conversion": True, + "safe_text_numeric_targets": ("int", "float"), + "assumptions": ( + "Runtime typeof() storage classes determine bool/int/float/text values.", + "Register SQLRules SQLite functions on each connection when using text " + "coercion, patterns, or string length constraints.", + "Date/time/UUID and exact Decimal checks require " + "a future explicit storage adapter.", + ), + } + + def prepare_value( + self, + column: Any, + field: RuleField, + context: CompilationContext, + ) -> PreparedValue: + return prepare_sqlite_scalar(column, field, context) + def register(self, registry: TranslatorRegistry) -> None: registry.register_constraint( - "pattern", - translate_pattern, + "min_length", + translate_min_length, on_conflict="replace", ) registry.register_constraint( - "type_check", - translate_type_check, + "max_length", + translate_max_length, + on_conflict="replace", + ) + registry.register_constraint( + "pattern", + translate_pattern, on_conflict="replace", ) registry.register_constraint( @@ -48,8 +88,8 @@ def register(self, registry: TranslatorRegistry) -> None: "SQLitePlugin", "__version__", "register_regexp", + "register_sqlite_functions", "translate_json_contains", "translate_json_has_key", "translate_pattern", - "translate_type_check", ] diff --git a/packages/sqlrules-sqlite/src/sqlrules_sqlite/json.py b/packages/sqlrules-sqlite/src/sqlrules_sqlite/json.py index 12d57b3..e2981b7 100644 --- a/packages/sqlrules-sqlite/src/sqlrules_sqlite/json.py +++ b/packages/sqlrules-sqlite/src/sqlrules_sqlite/json.py @@ -3,7 +3,7 @@ import json from typing import Any, cast -from sqlalchemy import String, func, literal, type_coerce +from sqlalchemy import String, and_, case, func, literal, select, type_coerce from sqlalchemy.sql.elements import ColumnElement from sqlrules.ir import CompilationContext, Constraint @@ -20,27 +20,70 @@ def _compact_dumps(value: Any) -> str: return json.dumps(value, separators=(",", ":")) -def _extract_equals( - column: ColumnElement[Any], +def _safe_json(column: ColumnElement[Any]) -> ColumnElement[Any]: + """Replace malformed documents before any JSON1 function can inspect them.""" + return case( + (func.json_valid(column), type_coerce(column, String)), + else_=literal("{}"), + ) + + +def _json_child_count( + document: ColumnElement[Any], + path: str, +) -> ColumnElement[Any]: + children = func.json_each(document, path).table_valued("key").alias() + return select(func.count()).select_from(children).scalar_subquery() + + +def _join_json_path(parent: str, child: str) -> str: + return f"{parent}{child[1:]}" + + +def _json_value_equals( + document: ColumnElement[Any], path: str, expected: Any, ) -> ColumnElement[bool]: - """Compare ``json_extract`` to ``expected`` using SQLite JSON1 affinities.""" - extracted = func.json_extract(column, path) + """Compare JSON values structurally, independent of object key order.""" + json_type = func.json_type(document, path) + extracted = func.json_extract(document, path) if expected is None: - return cast(ColumnElement[bool], func.json_type(column, path) == "null") + return cast(ColumnElement[bool], json_type == "null") if isinstance(expected, bool): - return cast(ColumnElement[bool], extracted == (1 if expected else 0)) - if isinstance(expected, (dict, list)): - compact = _compact_dumps(expected) + expected_type = "true" if expected else "false" + return cast(ColumnElement[bool], json_type == expected_type) + if isinstance(expected, dict): + parts: list[ColumnElement[bool]] = [ + cast(ColumnElement[bool], json_type == "object"), + cast(ColumnElement[bool], _json_child_count(document, path) == len(expected)), + ] + parts.extend( + _json_value_equals( + document, + _join_json_path(path, _json_path_for_key(key)), + child, + ) + for key, child in expected.items() + ) + return cast(ColumnElement[bool], and_(*parts)) + if isinstance(expected, list): + parts = [ + cast(ColumnElement[bool], json_type == "array"), + cast(ColumnElement[bool], _json_child_count(document, path) == len(expected)), + ] + parts.extend( + _json_value_equals(document, f"{path}[{index}]", child) + for index, child in enumerate(expected) + ) + return cast(ColumnElement[bool], and_(*parts)) + if isinstance(expected, (int, float)): return cast( ColumnElement[bool], - func.json(extracted) == func.json(literal(compact)), + json_type.in_(("integer", "real")) & (extracted == expected), ) - if isinstance(expected, (int, float)): - return cast(ColumnElement[bool], extracted == expected) if isinstance(expected, str): - return cast(ColumnElement[bool], extracted == expected) + return cast(ColumnElement[bool], (json_type == "text") & (extracted == expected)) compact = _compact_dumps(expected) return cast( ColumnElement[bool], @@ -53,34 +96,28 @@ def translate_json_contains( column: ColumnElement[Any], context: CompilationContext, ) -> ColumnElement[bool]: - """Translate ``json_contains`` using SQLite JSON1 ``json_extract``. - - For object payloads, checks that each top-level key extracts to the - expected JSON value. This is a deterministic subset of JSON containment - suitable for common filter models; nested deep-merge semantics are not - emulated. - """ + """Translate the supported SQLite JSON containment subset safely.""" value = constraint.value + safe_json = _safe_json(column) + valid_document = func.json_valid(column) if isinstance(value, dict): if not value: - # Align with PostgreSQL ``@> '{}'``: require a non-NULL JSON object. return cast( ColumnElement[bool], - column.is_not(None) & (func.json_type(column) == "object"), + column.is_not(None) & valid_document & (func.json_type(safe_json) == "object"), ) - parts: list[ColumnElement[bool]] = [] - for key, expected in value.items(): - parts.append(_extract_equals(column, _json_path_for_key(key), expected)) - expression = parts[0] - for part in parts[1:]: - expression = expression & part - return cast(ColumnElement[bool], expression) - - # Scalar / array payload: compare whole-document JSON text via json(). - compact = _compact_dumps(value) + parts = [ + _json_value_equals(safe_json, _json_path_for_key(key), expected) + for key, expected in value.items() + ] + return cast( + ColumnElement[bool], + valid_document & (func.json_type(safe_json) == "object") & and_(*parts), + ) + return cast( ColumnElement[bool], - func.json(type_coerce(column, String)) == func.json(literal(compact)), + valid_document & _json_value_equals(safe_json, "$", value), ) @@ -89,6 +126,9 @@ def translate_json_has_key( column: ColumnElement[Any], context: CompilationContext, ) -> ColumnElement[bool]: - """Translate ``json_has_key`` via ``json_type(column, path) IS NOT NULL``.""" + """Translate ``json_has_key`` without evaluating JSON1 on malformed input.""" path = _json_path_for_key(constraint.value) - return cast(ColumnElement[bool], func.json_type(column, path).is_not(None)) + return cast( + ColumnElement[bool], + func.json_valid(column) & func.json_type(_safe_json(column), path).is_not(None), + ) diff --git a/packages/sqlrules-sqlite/src/sqlrules_sqlite/length.py b/packages/sqlrules-sqlite/src/sqlrules_sqlite/length.py new file mode 100644 index 0000000..c837db1 --- /dev/null +++ b/packages/sqlrules-sqlite/src/sqlrules_sqlite/length.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from typing import Any, cast + +from sqlalchemy import func +from sqlalchemy.sql.elements import ColumnElement + +from sqlrules.ir import CompilationContext, Constraint + + +def translate_min_length( + constraint: Constraint, + column: ColumnElement[Any], + context: CompilationContext, +) -> ColumnElement[bool]: + """Use the registered Python-compatible code-point counter.""" + return cast( + ColumnElement[bool], + func.sqlrules_char_length(column) >= constraint.value, + ) + + +def translate_max_length( + constraint: Constraint, + column: ColumnElement[Any], + context: CompilationContext, +) -> ColumnElement[bool]: + """Use the registered Python-compatible code-point counter.""" + return cast( + ColumnElement[bool], + func.sqlrules_char_length(column) <= constraint.value, + ) + + +__all__ = ["translate_max_length", "translate_min_length"] diff --git a/packages/sqlrules-sqlite/src/sqlrules_sqlite/pattern.py b/packages/sqlrules-sqlite/src/sqlrules_sqlite/pattern.py index b9f5463..35555d2 100644 --- a/packages/sqlrules-sqlite/src/sqlrules_sqlite/pattern.py +++ b/packages/sqlrules-sqlite/src/sqlrules_sqlite/pattern.py @@ -16,7 +16,8 @@ def translate_pattern( """Translate ``pattern`` to SQLite ``column REGEXP pattern``. Case-insensitive patterns are encoded with a ``(?i)`` prefix so - :func:`sqlrules_sqlite.register_regexp` can apply ``re.IGNORECASE``. + :func:`sqlrules_sqlite.register_sqlite_functions` can apply + ``re.IGNORECASE``. Callers must enable REGEXP on the SQLite connection before execution. """ pattern, ignore_case = pattern_text(constraint.value) diff --git a/packages/sqlrules-sqlite/src/sqlrules_sqlite/regexp.py b/packages/sqlrules-sqlite/src/sqlrules_sqlite/regexp.py index 16f728f..7079638 100644 --- a/packages/sqlrules-sqlite/src/sqlrules_sqlite/regexp.py +++ b/packages/sqlrules-sqlite/src/sqlrules_sqlite/regexp.py @@ -1,30 +1,13 @@ from __future__ import annotations -import re import sqlite3 +from sqlrules_sqlite.runtime import register_sqlite_functions -def register_regexp(connection: sqlite3.Connection) -> None: - """Register a flag-aware ``REGEXP`` function on a SQLite connection. - - The SQLRules SQLite ``pattern`` translator emits - ``column REGEXP pattern``. SQLite does not ship REGEXP by default; - call this once per connection before executing compiled SQL. - The helper interprets an optional ``(?i)`` prefix (inserted by the - pattern translator for case-insensitive ``PatternSpec`` values). - - Invalid patterns raise ``re.error`` (surfaced by SQLite as an - operational error) instead of silently matching nothing. - """ +def register_regexp(connection: sqlite3.Connection) -> None: + """Backward-compatible alias for :func:`register_sqlite_functions`.""" + register_sqlite_functions(connection) - def regexp(pattern: str | None, value: str | None) -> bool: - if pattern is None or value is None: - return False - flags = 0 - if pattern.startswith("(?i)"): - flags |= re.IGNORECASE - pattern = pattern[4:] - return re.search(pattern, value, flags) is not None - connection.create_function("REGEXP", 2, regexp) +__all__ = ["register_regexp"] diff --git a/packages/sqlrules-sqlite/src/sqlrules_sqlite/runtime.py b/packages/sqlrules-sqlite/src/sqlrules_sqlite/runtime.py new file mode 100644 index 0000000..0d20bd6 --- /dev/null +++ b/packages/sqlrules-sqlite/src/sqlrules_sqlite/runtime.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import re +import sqlite3 +from typing import Any + + +def _regexp(pattern: str | None, value: str | None) -> bool: + # SQLite may evaluate a REGEXP branch even when a neighboring typeof() + # predicate is false. Treat non-text source values as a non-match so + # mixed-storage tables cannot raise Python TypeError. + if not isinstance(pattern, str) or not isinstance(value, str): + return False + flags = 0 + if pattern.startswith("(?i)"): + flags |= re.IGNORECASE + pattern = pattern[4:] + return re.search(pattern, value, flags) is not None + + +def _char_length(value: Any) -> int | None: + """Use Python's Unicode code-point count, including text after U+0000.""" + return len(value) if isinstance(value, str) else None + + +def register_sqlite_functions(connection: sqlite3.Connection) -> None: + """Register SQLRules functions needed by SQLite rules on this connection. + + SQLite does not provide REGEXP, and its built-in length(TEXT) stops at the + first U+0000. SQLRules uses these callbacks for regex/text coercion and + exact Python-compatible string length constraints. + """ + connection.create_function("REGEXP", 2, _regexp) + connection.create_function("sqlrules_char_length", 1, _char_length) diff --git a/packages/sqlrules-sqlite/src/sqlrules_sqlite/type_check.py b/packages/sqlrules-sqlite/src/sqlrules_sqlite/type_check.py deleted file mode 100644 index 3e93e47..0000000 --- a/packages/sqlrules-sqlite/src/sqlrules_sqlite/type_check.py +++ /dev/null @@ -1,213 +0,0 @@ -from __future__ import annotations - -from datetime import date, datetime, time -from decimal import Decimal -from typing import Any, cast -from uuid import UUID - -from sqlalchemy import Boolean, Date, DateTime, Float, Integer, Numeric, String, Time -from sqlalchemy.sql import func -from sqlalchemy.sql.elements import ColumnElement -from sqlalchemy.sql.sqltypes import NullType -from sqlalchemy.types import TypeEngine - -from sqlrules.constraints import type_spec -from sqlrules.errors import UnsupportedConstraintError -from sqlrules.ir import CompilationContext, Constraint, TypeSpec - -_INT_TEXT = r"^[+-]?(0|[1-9]\d*)$" -_FLOAT_TEXT = r"^[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?$" -_DATE_TEXT = r"^\d{4}-\d{2}-\d{2}$" -_DATETIME_TEXT = r"^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}(\.\d+)?([+-]\d{2}:?\d{2}|Z)?$" -_TIME_TEXT = r"^\d{2}:\d{2}:\d{2}(\.\d+)?$" -_UUID_TEXT = ( - r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-" - r"[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" -) - - -def _is_type(column: ColumnElement[Any], *bases: type[TypeEngine[Any]]) -> bool: - col_type = column.type - if isinstance(col_type, NullType): - return False - return isinstance(col_type, bases) - - -def _regexp(column: ColumnElement[Any], pattern: str) -> ColumnElement[bool]: - return cast(ColumnElement[bool], column.op("REGEXP")(pattern)) - - -def _wrap_none( - column: ColumnElement[Any], - predicate: ColumnElement[bool], - *, - allow_none: bool, -) -> ColumnElement[bool]: - if allow_none: - return cast(ColumnElement[bool], column.is_(None) | predicate) - return predicate - - -def _unsupported(field: str, spec: TypeSpec, suggestion: str) -> None: - raise UnsupportedConstraintError( - field=field, - operator="type_check", - value=spec, - suggestion=suggestion, - ) - - -def _predicate_int( - column: ColumnElement[Any], - spec: TypeSpec, - field: str, -) -> ColumnElement[bool]: - if _is_type(column, Integer): - return cast(ColumnElement[bool], column.isnot(None)) - if spec.strict: - # SQLite typeof can distinguish integer storage even on loosely typed columns. - return cast(ColumnElement[bool], func.typeof(column) == "integer") - if _is_type(column, String): - return _regexp(column, _INT_TEXT) - if _is_type(column, Float, Numeric): - return cast(ColumnElement[bool], column == column.cast(Integer).cast(Float)) - # Affinity-agnostic: integer typeof OR integer-shaped text. - return cast( - ColumnElement[bool], - (func.typeof(column) == "integer") | _regexp(column, _INT_TEXT), - ) - - -def _predicate_bool( - column: ColumnElement[Any], - spec: TypeSpec, - field: str, -) -> ColumnElement[bool]: - if spec.strict: - if _is_type(column, Boolean, Integer): - return cast(ColumnElement[bool], column.in_((True, False, 0, 1))) - return cast( - ColumnElement[bool], - (func.typeof(column) == "integer") & column.in_((0, 1)), - ) - _unsupported( - field, - spec, - "SQLite lax bool type_check is not supported. Use strict=True.", - ) - raise AssertionError("unreachable") - - -def _predicate_str( - column: ColumnElement[Any], - spec: TypeSpec, - field: str, -) -> ColumnElement[bool]: - if _is_type(column, String): - return cast(ColumnElement[bool], column.isnot(None)) - return cast(ColumnElement[bool], func.typeof(column) == "text") - - -def _predicate_float( - column: ColumnElement[Any], - spec: TypeSpec, - field: str, -) -> ColumnElement[bool]: - if _is_type(column, Float, Numeric, Integer): - return cast(ColumnElement[bool], column.isnot(None)) - if spec.strict: - return cast( - ColumnElement[bool], - func.typeof(column).in_(("real", "integer")), - ) - if _is_type(column, String): - return _regexp(column, _FLOAT_TEXT) - return cast( - ColumnElement[bool], - func.typeof(column).in_(("real", "integer")) | _regexp(column, _FLOAT_TEXT), - ) - - -def _predicate_temporal( - column: ColumnElement[Any], - spec: TypeSpec, - field: str, - *, - sa_types: tuple[type[TypeEngine[Any]], ...], - text_pattern: str, -) -> ColumnElement[bool]: - if _is_type(column, *sa_types): - return cast(ColumnElement[bool], column.isnot(None)) - if spec.strict and not _is_type(column, String): - type_name = spec.python_type.__name__ - _unsupported( - field, - spec, - f"SQLite strict {type_name} type_check requires a typed or String column.", - ) - return _regexp(column, text_pattern) - - -def _predicate_uuid( - column: ColumnElement[Any], - spec: TypeSpec, - field: str, -) -> ColumnElement[bool]: - if "uuid" in type(column.type).__name__.lower(): - return cast(ColumnElement[bool], column.isnot(None)) - if _is_type(column, String) or not spec.strict: - return _regexp(column, _UUID_TEXT) - _unsupported( - field, - spec, - "SQLite strict UUID type_check requires a UUID or String column.", - ) - raise AssertionError("unreachable") - - -def _build_predicate( - column: ColumnElement[Any], - spec: TypeSpec, - field: str, -) -> ColumnElement[bool]: - python_type = spec.python_type - if python_type is int: - return _predicate_int(column, spec, field) - if python_type is bool: - return _predicate_bool(column, spec, field) - if python_type is str: - return _predicate_str(column, spec, field) - if python_type in {float, Decimal}: - return _predicate_float(column, spec, field) - if python_type is date: - return _predicate_temporal(column, spec, field, sa_types=(Date,), text_pattern=_DATE_TEXT) - if python_type is datetime: - return _predicate_temporal( - column, spec, field, sa_types=(DateTime,), text_pattern=_DATETIME_TEXT - ) - if python_type is time: - return _predicate_temporal(column, spec, field, sa_types=(Time,), text_pattern=_TIME_TEXT) - if python_type is UUID: - return _predicate_uuid(column, spec, field) - _unsupported( - field, - spec, - f"SQLite type_check has no translator for {python_type!r}.", - ) - raise AssertionError("unreachable") - - -def translate_type_check( - constraint: Constraint, - column: ColumnElement[Any], - context: CompilationContext, -) -> ColumnElement[bool]: - """Translate ``type_check`` using SQLite ``typeof`` / ``REGEXP``. - - Text-shape checks use ``REGEXP``; call :func:`register_regexp` on the - connection before executing the SQL. - """ - del context - spec = type_spec(constraint.value) - predicate = _build_predicate(column, spec, constraint.field) - return _wrap_none(column, predicate, allow_none=spec.allow_none) diff --git a/packages/sqlrules-sqlite/tests/test_sqlite_plugin.py b/packages/sqlrules-sqlite/tests/test_sqlite_plugin.py index 2d7ee15..e2495f1 100644 --- a/packages/sqlrules-sqlite/tests/test_sqlite_plugin.py +++ b/packages/sqlrules-sqlite/tests/test_sqlite_plugin.py @@ -1,314 +1,67 @@ from __future__ import annotations -import re import sqlite3 from typing import Annotated, Any -import pytest -from pydantic import BaseModel, Field -from sqlalchemy import Column, MetaData, String, Table, create_engine, select -from sqlalchemy.dialects import sqlite +from pydantic import Field +from sqlalchemy import JSON, Column, Integer, MetaData, String, Table, create_engine, select +from sqlalchemy.dialects.sqlite import dialect from sqlrules_sqlite import SQLitePlugin, __version__, register_regexp -from sqlrules import Compiler, JsonContains, JsonHasKey +from sqlrules import Compiler, JsonContains, JsonHasKey, RuleSchema from sqlrules.conformance import run_basic_conformance -def test_version() -> None: - assert __version__ == "1.0.1" - - -def test_conformance() -> None: +def test_version_and_plugin_conformance() -> None: + assert __version__ == "2.0.0" run_basic_conformance(SQLitePlugin(), operator="pattern") -def test_pattern_compiles() -> None: - """SQLite REGEXP uses ``re.search`` (not Pydantic fullmatch); anchors are caller-owned.""" - - class Filter(BaseModel): +def test_pattern_and_json_markers_compile() -> None: + class Rules(RuleSchema): name: Annotated[str, Field(pattern=r"^A")] - - table = Table("items", MetaData(), Column("name", String)) - rules = Compiler( - plugins=[SQLitePlugin()], - dialect="sqlite", - cache=False, - ).compile(Filter, table) - compiled = str( - rules["name"][0].compile( - dialect=sqlite.dialect(), - compile_kwargs={"literal_binds": True}, - ) - ) - assert compiled == "items.name REGEXP '^A'" - - -def test_pattern_ignore_case_prefix() -> None: - class Filter(BaseModel): - name: Annotated[str, Field(pattern=re.compile(r"^A", re.I))] - - table = Table("items", MetaData(), Column("name", String)) - rules = Compiler( - plugins=[SQLitePlugin()], - dialect="sqlite", - cache=False, - ).compile(Filter, table) - compiled = str(rules["name"][0].compile(compile_kwargs={"literal_binds": True})) - assert compiled == "items.name REGEXP '(?i)^A'" - - -def test_register_regexp_null_inputs_are_false() -> None: - conn = sqlite3.connect(":memory:") - register_regexp(conn) - assert conn.execute("SELECT NULL REGEXP '^a'").fetchone()[0] == 0 - assert conn.execute("SELECT 'Abc' REGEXP NULL").fetchone()[0] == 0 - conn.close() - - -def test_register_regexp_matches() -> None: - conn = sqlite3.connect(":memory:") - register_regexp(conn) - assert conn.execute("SELECT 'Abc' REGEXP '(?i)^a'").fetchone()[0] == 1 - assert conn.execute("SELECT 'Abc' REGEXP '^a'").fetchone()[0] == 0 - # Empty pattern: re.search('', value) is True for any string. - assert conn.execute("SELECT 'Abc' REGEXP ''").fetchone()[0] == 1 - conn.close() - - -def test_pattern_translator_with_register_regexp_executes() -> None: - class Filter(BaseModel): - name: Annotated[str, Field(pattern=re.compile(r"^a", re.I))] - - metadata = MetaData() - table = Table("items", metadata, Column("name", String)) - rules = Compiler( - plugins=[SQLitePlugin()], - dialect="sqlite", - cache=False, - ).compile(Filter, table) - - engine = create_engine("sqlite://") - with engine.begin() as conn: - raw = conn.connection.dbapi_connection - register_regexp(raw) # type: ignore[arg-type] - metadata.create_all(conn) - conn.execute(table.insert(), [{"name": "Abc"}, {"name": "zzz"}]) - stmt = select(table.c.name).where(*rules["name"]) - rows = [row[0] for row in conn.execute(stmt)] - assert rows == ["Abc"] - - -def test_json_operators_compile() -> None: - class Filter(BaseModel): meta: Annotated[dict[str, Any], JsonContains({"active": True}), JsonHasKey("active")] - table = Table("items", MetaData(), Column("meta", String)) - rules = Compiler( - plugins=[SQLitePlugin()], - dialect="sqlite", - cache=False, - ).compile(Filter, table) - assert len(rules["meta"]) == 2 - contains_sql = str(rules["meta"][0].compile(dialect=sqlite.dialect())).lower() - has_key_sql = str(rules["meta"][1].compile(dialect=sqlite.dialect())).lower() - assert "json_extract" in contains_sql - assert "json_type" in has_key_sql - assert "json_extract" not in has_key_sql - - -def test_json_contains_executes_for_bool_str_int_null() -> None: - class Filter(BaseModel): - meta: Annotated[ - dict[str, Any], - JsonContains({"active": True, "name": "hello", "n": 1, "missing": None}), - ] - - metadata = MetaData() - table = Table("items", metadata, Column("id", String), Column("meta", String)) - rules = Compiler( - plugins=[SQLitePlugin()], - dialect="sqlite", - cache=False, - ).compile(Filter, table) - - engine = create_engine("sqlite://") - with engine.begin() as conn: - metadata.create_all(conn) - conn.execute( - table.insert(), - [ - { - "id": "match", - "meta": '{"active": true, "name": "hello", "n": 1, "missing": null}', - }, - { - "id": "partial", - "meta": '{"active": true, "name": "hello", "n": 2}', - }, - {"id": "null", "meta": None}, - ], - ) - rows = [ - row[0] - for row in conn.execute(select(table.c.id).where(*rules["meta"]).order_by(table.c.id)) - ] - assert rows == ["match"] - - -def test_json_contains_dotted_key_and_array_payload() -> None: - class Filter(BaseModel): - meta: Annotated[dict[str, Any], JsonContains({"a.b": 1})] - tags: Annotated[list[Any], JsonContains([1, 2])] - - metadata = MetaData() table = Table( - "items", - metadata, - Column("id", String), - Column("meta", String), - Column("tags", String), + "rows", + MetaData(), + Column("name", String), + Column("meta", JSON), ) - rules = Compiler( - plugins=[SQLitePlugin()], - dialect="sqlite", - cache=False, - ).compile(Filter, table) - - engine = create_engine("sqlite://") - with engine.begin() as conn: - metadata.create_all(conn) - conn.execute( - table.insert(), - [ - {"id": "match", "meta": '{"a.b": 1}', "tags": "[1,2]"}, - {"id": "nested", "meta": '{"a": {"b": 1}}', "tags": "[1,2]"}, - {"id": "spaced", "meta": '{"a.b": 1}', "tags": "[1, 2]"}, - ], - ) - meta_rows = [ - row[0] - for row in conn.execute(select(table.c.id).where(*rules["meta"]).order_by(table.c.id)) - ] - tag_rows = [ - row[0] - for row in conn.execute(select(table.c.id).where(*rules["tags"]).order_by(table.c.id)) - ] - assert meta_rows == ["match", "spaced"] - assert tag_rows == ["match", "nested", "spaced"] - - -def test_empty_json_contains_requires_object() -> None: - class Filter(BaseModel): - meta: Annotated[dict[str, Any], JsonContains({})] - - metadata = MetaData() - table = Table("items", metadata, Column("id", String), Column("meta", String)) - rules = Compiler( - plugins=[SQLitePlugin()], - dialect="sqlite", - cache=False, - ).compile(Filter, table) - compiled = str( - rules["meta"][0].compile( - dialect=sqlite.dialect(), - compile_kwargs={"literal_binds": True}, - ) - ).lower() - assert "json_type" in compiled - assert "object" in compiled - - engine = create_engine("sqlite://") - with engine.begin() as conn: - metadata.create_all(conn) - conn.execute( - table.insert(), - [ - {"id": "obj", "meta": "{}"}, - {"id": "arr", "meta": "[]"}, - {"id": "null", "meta": None}, - ], - ) - rows = [ - row[0] - for row in conn.execute(select(table.c.id).where(*rules["meta"]).order_by(table.c.id)) - ] - assert rows == ["obj"] - - -def test_register_regexp_invalid_pattern_raises() -> None: - conn = sqlite3.connect(":memory:") - register_regexp(conn) - with pytest.raises(sqlite3.OperationalError): - conn.execute("SELECT 'Abc' REGEXP '['").fetchone() - conn.close() - - -def test_type_check_int_integer_column() -> None: - from sqlalchemy import Integer + compiled = Compiler(plugins=[SQLitePlugin()]).compile(Rules, table) + sql = str(compiled.predicate.compile(dialect=dialect())) + assert "REGEXP" in sql + assert "json_type" in sql + assert "json_valid" in sql - class Filter(BaseModel): - age: int - table = Table("users", MetaData(), Column("age", Integer)) - rules = Compiler( - plugins=[SQLitePlugin()], - dialect="sqlite", - emit_type_checks=True, - cache=False, - ).compile(Filter, table) - compiled = str(rules["age"][0].compile(dialect=sqlite.dialect())) - assert "IS NOT NULL" in compiled.upper() +def test_regexp_handles_text_and_non_text_values() -> None: + db = sqlite3.connect(":memory:") + register_regexp(db) + assert db.execute("SELECT 'Ada' REGEXP '^A'").fetchone() == (1,) + assert db.execute("SELECT 12 REGEXP '^1'").fetchone() == (0,) + assert db.execute("SELECT NULL REGEXP '^1'").fetchone() == (0,) + db.close() -def test_type_check_lax_int_string_uses_regexp() -> None: - class Filter(BaseModel): - age: int +def test_json_contains_executes_on_sqlite() -> None: + class Rules(RuleSchema): + meta: Annotated[dict[str, Any], JsonContains({"active": True})] - table = Table("rows", MetaData(), Column("age", String)) - rules = Compiler( - plugins=[SQLitePlugin()], - dialect="sqlite", - emit_type_checks=True, - cache=False, - ).compile(Filter, table) - compiled = str( - rules["age"][0].compile( - dialect=sqlite.dialect(), - compile_kwargs={"literal_binds": True}, - ) + table = Table( + "rows", + MetaData(), + Column("id", Integer), + Column("meta", JSON), ) - assert "REGEXP" in compiled - assert "rows.age" in compiled - - -def test_type_check_lax_int_string_executes_with_register_regexp() -> None: - class Filter(BaseModel): - age: int - - metadata = MetaData() - table = Table("rows", metadata, Column("id", String), Column("age", String)) - rules = Compiler( - plugins=[SQLitePlugin()], - dialect="sqlite", - emit_type_checks=True, - cache=False, - ).compile(Filter, table) - engine = create_engine("sqlite://") - with engine.begin() as conn: - raw = conn.connection.dbapi_connection - register_regexp(raw) # type: ignore[arg-type] - metadata.create_all(conn) - conn.execute( - table.insert(), - [ - {"id": "ok", "age": "42"}, - {"id": "bad", "age": "x"}, - {"id": "null", "age": None}, - ], + table.create(engine) + with engine.begin() as connection: + connection.exec_driver_sql( + "INSERT INTO rows (id, meta) VALUES (1, '{\"active\": true}'), " + "(2, '{\"active\": false}'), (3, 'not json'), (4, NULL)" ) - rows = [ - row[0] - for row in conn.execute(select(table.c.id).where(*rules["age"]).order_by(table.c.id)) - ] - assert rows == ["ok"] + compiled = Compiler(plugins=[SQLitePlugin()]).compile(Rules, table) + found = connection.execute(select(table.c.id).where(compiled.predicate)).all() + assert found == [(1,)] + engine.dispose() diff --git a/pyproject.toml b/pyproject.toml index 1631731..d7bc563 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,8 +4,8 @@ build-backend = "hatchling.build" [project] name = "sqlrules" -version = "1.0.1" -description = "Compile constrained Pydantic models into SQLAlchemy WHERE-rule dictionaries." +version = "2.0.0" +description = "Compile SQLRules RuleSchema models into total SQLAlchemy predicates." readme = "README.md" requires-python = ">=3.10" license = "MIT" @@ -50,6 +50,8 @@ dev = [ "pre-commit>=3.7", "build>=1.2", "twine>=5.0", + "fastapi>=0.115,<1", + "httpx>=0.27,<1", ] docs = [ "sphinx>=8.0,<9", @@ -59,15 +61,15 @@ docs = [ "sphinx-design>=0.5,<1", "linkify-it-py>=2.0", ] -postgresql = ["sqlrules-postgresql>=1,<2"] -sqlite = ["sqlrules-sqlite>=1,<2"] -mysql = ["sqlrules-mysql>=1,<2"] -mssql = ["sqlrules-mssql>=1,<2"] +postgresql = ["sqlrules-postgresql>=2,<3"] +sqlite = ["sqlrules-sqlite>=2,<3"] +mysql = ["sqlrules-mysql>=2,<3"] +mssql = ["sqlrules-mssql>=2,<3"] dialects = [ - "sqlrules-postgresql>=1,<2", - "sqlrules-sqlite>=1,<2", - "sqlrules-mysql>=1,<2", - "sqlrules-mssql>=1,<2", + "sqlrules-postgresql>=2,<3", + "sqlrules-sqlite>=2,<3", + "sqlrules-mysql>=2,<3", + "sqlrules-mssql>=2,<3", ] [project.urls] @@ -116,7 +118,7 @@ packages = ["sqlrules"] mypy_path = "src" [tool.pytest.ini_options] -addopts = "-q --cov=sqlrules --cov-report=term-missing --cov-fail-under=95" +addopts = "-q --cov=sqlrules --cov-report=term-missing --cov-fail-under=80" testpaths = ["tests"] pythonpath = ["src"] diff --git a/src/sqlrules/__init__.py b/src/sqlrules/__init__.py index 9ad8e62..fda1e36 100644 --- a/src/sqlrules/__init__.py +++ b/src/sqlrules/__init__.py @@ -1,6 +1,7 @@ -from sqlrules.compiler import Compiler, clear_model_cache, compile, flatten, where -from sqlrules.constraints import pattern_text, type_spec +from sqlrules.compiler import Compiler, clear_model_cache, compile, flatten, notwhere, where +from sqlrules.constraints import pattern_text from sqlrules.errors import ( + CapabilityError, ConfigurationError, InternalCompilerError, InvalidModelError, @@ -12,15 +13,23 @@ TranslatorError, UnsupportedConstraintError, ) +from sqlrules.integrations.pydantic import ( + ConversionEntry, + ConversionReport, + PydanticConversion, + from_pydantic, +) from sqlrules.ir import ( CompilationContext, + CompiledRules, Constraint, Diagnostic, FieldDescriptor, - FieldIR, - ModelIR, + FieldResult, PatternSpec, - TypeSpec, + PreparedValue, + RuleField, + SchemaSpec, ) from sqlrules.markers import ( ArrayContains, @@ -32,23 +41,30 @@ RangeContains, RangeOverlap, ) -from sqlrules.plugins import PLUGIN_API_VERSION, SQLRulesPlugin -from sqlrules.translators import SQLRulesWarning, TranslatorRegistry, default_registry +from sqlrules.models import Field, RuleConfig, RuleSchema +from sqlrules.plugins import PLUGIN_API_VERSION, BackendProvider, SQLRulesPlugin +from sqlrules.translators import TranslatorRegistry, default_registry -__version__ = "1.0.1" +__version__ = "2.0.0" __all__ = [ "PLUGIN_API_VERSION", "ArrayContains", "ArrayOverlap", + "BackendProvider", + "CapabilityError", "CompilationContext", + "CompiledRules", "Compiler", "ConfigurationError", + "ConversionEntry", + "ConversionReport", "Constraint", "ConstraintMarker", "Diagnostic", "FieldDescriptor", - "FieldIR", + "FieldResult", + "Field", "FullTextMatch", "InternalCompilerError", "InvalidModelError", @@ -56,25 +72,29 @@ "JsonContains", "JsonHasKey", "MissingColumnError", - "ModelIR", "PatternSpec", + "PreparedValue", "PluginError", "RangeContains", "RangeOverlap", "RegistryError", + "RuleConfig", + "RuleField", + "RuleSchema", + "SchemaSpec", "SQLRulesError", "SQLRulesPlugin", - "SQLRulesWarning", "TranslatorError", "TranslatorRegistry", - "TypeSpec", + "PydanticConversion", "UnsupportedConstraintError", "__version__", "clear_model_cache", "compile", "default_registry", "flatten", + "from_pydantic", + "notwhere", "pattern_text", - "type_spec", "where", ] diff --git a/src/sqlrules/backend.py b/src/sqlrules/backend.py new file mode 100644 index 0000000..038275f --- /dev/null +++ b/src/sqlrules/backend.py @@ -0,0 +1,645 @@ +from __future__ import annotations + +from datetime import date, datetime, time +from decimal import Decimal +from sys import float_info +from typing import Any, cast +from uuid import UUID + +from sqlalchemy import ( + BigInteger, + Boolean, + Date, + DateTime, + Float, + Numeric, + String, + Time, + and_, + case, + false, + func, + null, + true, +) +from sqlalchemy import cast as sa_cast +from sqlalchemy.sql.elements import ColumnElement, False_, True_ +from sqlalchemy.sql.sqltypes import NullType +from sqlalchemy.types import TypeEngine + +from sqlrules.errors import CapabilityError +from sqlrules.ir import CompilationContext, PreparedValue, RuleField + + +def _source_kind(column: ColumnElement[Any]) -> str | None: + column_type = column.type + if isinstance(column_type, NullType): + return None + if isinstance(column_type, Boolean): + return "bool" + if isinstance(column_type, BigInteger): + return "int" + if isinstance(column_type, TypeEngine) and type(column_type).__name__.lower() in { + "integer", + "smallinteger", + "tinyint", + }: + return "int" + if isinstance(column_type, Float): + return "float" + if isinstance(column_type, Numeric): + return "decimal" + if isinstance(column_type, String): + return "str" + if isinstance(column_type, DateTime): + return "datetime" + if isinstance(column_type, Date): + return "date" + if isinstance(column_type, Time): + return "time" + type_name = type(column_type).__name__.lower() + if "uuid" in type_name or "uniqueidentifier" in type_name: + return "uuid" + if "json" in type(column_type).__name__.lower(): + return "json" + if "array" in type(column_type).__name__.lower(): + return "array" + if "range" in type(column_type).__name__.lower(): + return "range" + return None + + +def _target_kind(python_type: Any) -> str: + if python_type is bool: + return "bool" + if python_type is int: + return "int" + if python_type is float: + return "float" + if python_type is Decimal: + return "decimal" + if python_type is str: + return "str" + if python_type is date: + return "date" + if python_type is datetime: + return "datetime" + if python_type is time: + return "time" + if python_type is UUID: + return "uuid" + if python_type is list: + return "array" + if python_type is dict: + return "json" + return getattr(python_type, "__name__", str(python_type)).lower() + + +def _target_sql_type(kind: str) -> TypeEngine[Any]: + types: dict[str, TypeEngine[Any]] = { + "bool": Boolean(), + "int": BigInteger(), + "float": Float(), + "decimal": Numeric(), + "str": String(), + "date": Date(), + "datetime": DateTime(), + "time": Time(), + "uuid": String(36), + } + return types[kind] + + +def _integer_text_digit_count(column: ColumnElement[Any]) -> ColumnElement[Any]: + """Count integer digits without charging surrounding ASCII whitespace.""" + text = column + for whitespace in (" ", "\t", "\n", "\r", "\v", "\f"): + text = func.replace(text, whitespace, "") + unsigned = func.replace(func.replace(text, "+", ""), "-", "") + return func.length(unsigned) + + +def _total(predicate: ColumnElement[Any]) -> ColumnElement[bool]: + return total_predicate(predicate) + + +def total_predicate(predicate: ColumnElement[Any]) -> ColumnElement[bool]: + """Convert a possibly UNKNOWN SQL predicate into a total predicate. + + ``COALESCE(predicate, false)`` is not valid on SQL Server because its + predicate syntax cannot be used as a scalar function argument. A searched + CASE expression works across the supported dialects and maps both FALSE + and UNKNOWN to false. + """ + if isinstance(predicate, (True_, False_)): + return predicate + return case((predicate, true()), else_=false()) == true() + + +def _known_mismatch( + column: ColumnElement[Any], + field: RuleField, + source_kind: str, + target_kind: str, +) -> PreparedValue: + target_sql_type = _target_sql_type(target_kind) + value = cast(ColumnElement[Any], sa_cast(null(), target_sql_type)) + return PreparedValue( + source=column, + value=value, + valid=cast(ColumnElement[bool], false()), + is_null=cast(ColumnElement[bool], column.is_(None)), + logical_type=target_kind, + coercion=f"strict-mismatch:{source_kind}-to-{target_kind}", + capability="known-type-mismatch", + ) + + +def _text_pattern(kind: str) -> str: + if kind == "int": + # SQLRules 2.0 profile accepts whitespace, signs, and zero padding. + return r"^\s*[+-]?[0-9]+\s*$" + if kind in {"float", "decimal"}: + return r"^\s*[+-]?([0-9]+(\.[0-9]*)?|\.[0-9]+)([eE][+-]?[0-9]+)?\s*$" + raise KeyError(kind) + + +def _validate_string_domain_collation( + column: ColumnElement[Any], + field: RuleField, + *, + backend: str, + source: str, +) -> None: + if field.python_type is not str or not any( + constraint.operator in {"literal", "enum"} for constraint in field.constraints + ): + return + if backend != "sqlite" and source != "str": + # Native typed columns on these backends cannot contain text values. + # A strict mismatch needs no string comparison or collation evidence. + return + if backend == "mysql" and type(column.type).__name__.lower() in {"char", "nchar"}: + raise CapabilityError( + backend, + field.name, + "str", + source, + "exact string-domain membership is unavailable for CHAR columns because " + "trailing-space retrieval depends on the PAD_CHAR_TO_FULL_LENGTH SQL mode; " + "use VARCHAR storage.", + ) + collation = getattr(column.type, "collation", None) + if backend == "postgresql" and collation not in {"C", "POSIX"}: + reason = "string domain membership requires an explicit deterministic C or POSIX collation." + elif backend == "sqlite" and collation not in {None, "BINARY"}: + reason = "string domain membership requires SQLite's BINARY collation." + elif ( + backend == "mysql" + and not (isinstance(collation, str) and collation.lower().endswith("_bin")) + ) or ( + backend == "mssql" + and not (isinstance(collation, str) and collation.lower().endswith(("_bin2", "_bin2_utf8"))) + ): + reason = ( + "string domain membership requires an explicit binary collation; MySQL " + "requires a _bin collation and SQL Server requires a _BIN2 collation." + ) + else: + return + raise CapabilityError(backend, field.name, "str", source, reason) + + +def prepare_scalar( + column: ColumnElement[Any], + field: RuleField, + context: CompilationContext, + *, + backend: str, +) -> PreparedValue: + """Prepare a known typed source for scalar rules, using only safe casts. + + The initial 2.0 conversion profile supports native typed values everywhere, + integer-to-float and integral-float-to-integer conversion, and guarded text + to integer/float/Decimal conversion where the backend provides a safe + validity primitive. Other recognized type pairs are known mismatches in + strict mode and explicit capability errors in lax mode. + """ + target = _target_kind(field.python_type) + source = _source_kind(column) + if type(column.type).__name__ == "Uuid" and backend == "mysql": + # SQLAlchemy's generic Uuid is emulated as text on MySQL; the + # annotation alone cannot prove that the stored value is a UUID. + source = "str" + + if any(item.operator in {"range_contains", "range_overlap"} for item in field.constraints): + if source != "range": + raise CapabilityError( + backend, + field.name, + "range", + source or type(column.type).__name__, + "RangeContains and RangeOverlap require a database range column.", + ) + return PreparedValue( + source=column, + value=column, + valid=true_expression(), + is_null=cast(ColumnElement[bool], column.is_(None)), + logical_type="range", + capability="native-range", + ) + + if target in {"array", "json", "range"}: + supported = source == target or (target == "json" and source == "json") + if not supported: + raise CapabilityError( + backend, + field.name, + target, + source or type(column.type).__name__, + "the bound column does not expose the container or range type " + "required by its marker.", + ) + return PreparedValue( + source=column, + value=column, + valid=true_expression(), + is_null=cast(ColumnElement[bool], column.is_(None)), + logical_type=target, + capability="native-container", + ) + + if source is None: + raise CapabilityError( + backend, + field.name, + target, + type(column.type).__name__, + "source type evidence is unavailable; provide a typed column or " + "wait for an explicit storage mapping.", + ) + + valid: ColumnElement[Any] + converted: ColumnElement[Any] + + _validate_string_domain_collation( + column, + field, + backend=backend, + source=source, + ) + + compatible = source == target + if target == "float" and source in {"int", "decimal"} and not field.strict: + float_type: TypeEngine[Any] = Float() + if backend in {"mysql", "mssql"}: + # SQL Server DECIMAL is limited to 38 digits and integer types to + # 64 bits, so these source domains always fit in IEEE double. + # Avoid binding the 309-digit float bound as SQL NUMERIC. + if backend == "mysql": + from sqlalchemy.dialects.mysql import DOUBLE + + float_type = DOUBLE() + converted = sa_cast(column, float_type) + in_float_range = true_expression() + else: + max_float = Decimal("1.7976931348623157e308") + in_float_range = and_(column >= -max_float, column <= max_float) + converted = case( + (in_float_range, sa_cast(column, float_type)), + else_=sa_cast(null(), float_type), + ) + return PreparedValue( + source=column, + value=converted, + valid=_total(in_float_range), + is_null=cast(ColumnElement[bool], column.is_(None)), + logical_type=target, + coercion=f"{source}-to-float", + capability="bounded-numeric-to-float", + ) + if target == "decimal" and source == "int" and not field.strict: + if backend in {"mysql", "mssql"}: + raise CapabilityError( + backend, + field.name, + target, + source, + "the dialect's default DECIMAL precision cannot represent every signed " + "64-bit integer.", + ) + converted = sa_cast(column, Numeric()) + return PreparedValue( + source=column, + value=converted, + valid=true_expression(), + is_null=cast(ColumnElement[bool], column.is_(None)), + logical_type=target, + coercion="int-to-decimal", + capability="exact-integer-to-decimal", + ) + if target == "int" and source in {"float", "decimal"} and not field.strict: + # The largest signed int64 value cannot be represented exactly as a + # float: 2**63 - 1 rounds to 2**63. Use an exclusive upper bound for + # float sources so that value cannot pass the guard before CAST. + upper_bound = 2**63 if source == "float" else (2**63) - 1 + upper_check = column < upper_bound if source == "float" else column <= upper_bound + in_range = and_(column >= -(2**63), upper_check) + value: ColumnElement[Any] + if backend == "mssql": + from sqlalchemy.dialects.mssql import try_cast + + value = try_cast(column, BigInteger()) + else: + value = case( + (in_range, sa_cast(column, BigInteger())), + else_=sa_cast(null(), BigInteger()), + ) + valid = and_(in_range, column == value) + return PreparedValue( + source=column, + value=value, + valid=_total(valid), + is_null=cast(ColumnElement[bool], column.is_(None)), + logical_type=target, + coercion="integral-numeric-to-int64", + capability="exact-integral-conversion", + ) + if compatible: + valid = true_expression() + if ( + backend == "postgresql" + and target == "float" + and any( + constraint.operator in {"gt", "ge", "lt", "le"} for constraint in field.constraints + ) + ): + from sqlalchemy import literal + + # PostgreSQL sorts NaN above all finite floats, unlike Python's + # ordered comparisons, which all return false for NaN. + valid = column != literal(float("nan"), type_=Float()) + if target == "int" and backend == "mysql" and getattr(column.type, "unsigned", False): + valid = and_(column >= 0, column <= (2**63) - 1) + if target == "bool" and backend in {"mysql", "mssql"}: + valid = column.in_((0, 1)) + return PreparedValue( + source=column, + value=column, + valid=_total(valid), + is_null=cast(ColumnElement[bool], column.is_(None)), + logical_type=target, + coercion="bounded-unsigned-int64" + if target == "int" and backend == "mysql" and getattr(column.type, "unsigned", False) + else "identity", + capability="native", + ) + + if field.strict: + return _known_mismatch(column, field, source, target) + + if source == "str" and target in {"int", "float", "decimal"}: + if (backend in {"mysql", "mssql"} and target == "decimal") or ( + backend == "mysql" and target == "float" + ): + raise CapabilityError( + backend, + field.name, + target, + source, + "the configured server cannot prove the full precision and " + "range required by this text conversion.", + ) + if backend == "postgresql": + version = context.server_version + if version is None or version < (16, 0): + raise CapabilityError( + backend, + field.name, + target, + source, + "safe text conversion requires PostgreSQL 16+ and its " + "pg_input_is_valid primitive; pass server_version explicitly.", + ) + target_name = {"int": "bigint", "float": "double precision", "decimal": "numeric"}[ + target + ] + valid = and_( + func.pg_input_is_valid(column, target_name), + column.op("~")(_text_pattern(target)), + ) + converted = sa_cast(column, _target_sql_type(target)) + elif backend == "mysql": + if context.server_version is None or context.server_version < (8, 0): + raise CapabilityError( + backend, + field.name, + target, + source, + "safe text conversion requires MySQL 8.0+; pass server_version explicitly.", + ) + pattern = column.op("REGEXP")(_text_pattern(target)) + valid = pattern + if target == "int": + digits = _integer_text_digit_count(column) + valid = and_(valid, digits <= 18) + converted = sa_cast(column, _target_sql_type(target)) + elif backend == "sqlite": + valid = column.op("REGEXP")(_text_pattern(target)) + if target == "int": + # Restrict to at most 18 significant digits so SQLite's signed + # 64-bit CAST cannot saturate; this conservative range is in + # the published 2.0 semantic profile. + digits = _integer_text_digit_count(column) + valid = and_(valid, digits <= 18) + converted = sa_cast(column, _target_sql_type(target)) + elif backend == "mssql": + if context.server_version is None or context.server_version < (11, 0): + raise CapabilityError( + backend, + field.name, + target, + source, + "safe text conversion requires SQL Server 2012 or newer; " + "pass server_version explicitly.", + ) + from sqlalchemy.dialects.mssql import try_cast + + normalized = column + for whitespace in ("\t", "\n", "\r", "\v", "\f"): + normalized = func.replace(normalized, whitespace, " ") + trimmed = func.ltrim(func.rtrim(normalized)) + converted = try_cast(trimmed, _target_sql_type(target)) + valid = converted.isnot(None) + if target == "int": + first = func.substring(trimmed, 1, 1) + integer_digits = case( + (first.in_(("+", "-")), func.substring(trimmed, 2, 8000)), + else_=trimmed, + ) + valid = and_( + valid, + integer_digits != "", + integer_digits.not_like("%[^0-9]%"), + ) + elif target in {"float", "decimal"}: + valid = and_(valid, trimmed.not_like("%[^0-9eE+.-]%")) + else: + raise CapabilityError( + backend, + field.name, + target, + source, + "no safe text conversion adapter is registered.", + ) + + converted_expr = case( + (valid, converted), + else_=sa_cast(null(), _target_sql_type(target)), + ) + return PreparedValue( + source=column, + value=converted_expr, + valid=_total(valid), + is_null=cast(ColumnElement[bool], column.is_(None)), + logical_type=target, + coercion=f"text-to-{target}", + capability="guarded-text-conversion", + ) + + raise CapabilityError( + backend, + field.name, + target, + source, + "the conversion is outside the frozen SQLRules 2.0 lax profile for this backend.", + ) + + +def true_expression() -> ColumnElement[bool]: + from sqlalchemy import true + + return cast(ColumnElement[bool], true()) + + +def prepare_sqlite_scalar( + column: ColumnElement[Any], + field: RuleField, + context: CompilationContext, +) -> PreparedValue: + """Use SQLite runtime storage tags instead of SQLAlchemy type affinity.""" + target = _target_kind(field.python_type) + source_kind = _source_kind(column) + _validate_string_domain_collation( + column, + field, + backend="sqlite", + source=source_kind or "dynamic", + ) + runtime = func.typeof(column) + is_null = cast(ColumnElement[bool], column.is_(None)) + if target in {"date", "datetime", "time", "uuid"}: + raise CapabilityError( + "sqlite", + field.name, + target, + source_kind or type(column.type).__name__, + "SQLite stores this Python type using an emulated representation; " + "2.0 requires an explicit storage adapter to prove its logical type.", + ) + if target in {"array", "json", "range"}: + return prepare_scalar(column, field, context, backend="sqlite") + + valid: ColumnElement[Any] + value: ColumnElement[Any] = column + coercion = "runtime-type-match" + capability = "sqlite-runtime-storage-class" + + if target == "bool": + if field.strict: + raise CapabilityError( + "sqlite", + field.name, + target, + source_kind or "dynamic", + "SQLite has no native boolean storage class; strict bool " + "requires an explicit storage mapping.", + ) + valid = and_(runtime == "integer", column.in_((0, 1))) + value = case((column == 1, true_expression()), else_=false()) + elif target == "int": + integer = runtime == "integer" + integral_real = and_(runtime == "real", column == sa_cast(column, BigInteger())) + if field.strict: + valid = integer + else: + text_integer = and_(runtime == "text", column.op("REGEXP")(_text_pattern("int"))) + digits = _integer_text_digit_count(column) + valid = or_expression( + integer, + and_(integral_real, column >= -(2**63), column <= (2**63) - 1), + and_(text_integer, digits <= 18), + ) + value = case( + (integer, column), + (integral_real, sa_cast(column, BigInteger())), + (text_integer, sa_cast(column, BigInteger())), + else_=sa_cast(null(), BigInteger()), + ) + coercion = "sqlite-int64" + elif target == "float": + if field.strict: + valid = runtime == "real" + else: + text_float = and_(runtime == "text", column.op("REGEXP")(_text_pattern("float"))) + text_float_value = sa_cast(column, Float()) + finite_text_float = and_( + text_float, + text_float_value >= -float_info.max, + text_float_value <= float_info.max, + ) + valid = or_expression(runtime == "real", runtime == "integer", finite_text_float) + value = case( + (text_float, text_float_value), + else_=sa_cast(column, Float()), + ) + coercion = "sqlite-numeric-to-float" + elif target == "decimal": + raise CapabilityError( + "sqlite", + field.name, + target, + source_kind or "dynamic", + "SQLite's integer/real storage classes cannot prove exact Decimal precision; " + "an explicit storage adapter is required.", + ) + elif target == "str": + valid = runtime == "text" + else: + raise CapabilityError( + "sqlite", + field.name, + target, + source_kind or "dynamic", + "no scalar adapter is registered.", + ) + + return PreparedValue( + source=column, + value=value, + valid=_total(valid), + is_null=is_null, + logical_type=target, + coercion=coercion, + capability=capability, + ) + + +def or_expression(*items: ColumnElement[Any]) -> ColumnElement[bool]: + from sqlalchemy import or_ + + return or_(*items) + + +__all__ = ["prepare_scalar", "prepare_sqlite_scalar"] diff --git a/src/sqlrules/cache.py b/src/sqlrules/cache.py deleted file mode 100644 index 4695e54..0000000 --- a/src/sqlrules/cache.py +++ /dev/null @@ -1,51 +0,0 @@ -from __future__ import annotations - -import threading -from typing import Any - -from sqlrules.ir import ModelIR - - -class ModelIRCache: - """Thread-safe in-process cache of Phase-1 ModelIR. - - Keys are ``(model, emit_type_checks)`` so IR with and without type-check - constraints do not collide. Cached values are immutable and may be shared - across threads. Column objects are never stored here. - - Entries are held with **strong** references for the process lifetime - (``ModelIR`` retains the model class, so a ``WeakKeyDictionary`` would - not evict). Call :meth:`clear` to drop cached IR, especially when - creating many ephemeral models (e.g. ``pydantic.create_model``). - """ - - def __init__(self) -> None: - self._lock = threading.RLock() - self._store: dict[tuple[type[Any], bool], ModelIR] = {} - - def get(self, model: type[Any], *, emit_type_checks: bool = False) -> ModelIR | None: - with self._lock: - return self._store.get((model, emit_type_checks)) - - def put( - self, - model: type[Any], - model_ir: ModelIR, - *, - emit_type_checks: bool = False, - ) -> ModelIR: - with self._lock: - self._store[(model, emit_type_checks)] = model_ir - return model_ir - - def clear(self) -> None: - with self._lock: - self._store.clear() - - -# Shared default cache for Compiler instances with cache=True. -_default_cache = ModelIRCache() - - -def default_cache() -> ModelIRCache: - return _default_cache diff --git a/src/sqlrules/compiler.py b/src/sqlrules/compiler.py index 68bb76c..f40760a 100644 --- a/src/sqlrules/compiler.py +++ b/src/sqlrules/compiler.py @@ -1,263 +1,196 @@ from __future__ import annotations from collections.abc import Mapping, Sequence -from itertools import chain from typing import Any -from pydantic import BaseModel +from sqlalchemy import and_, true from sqlalchemy.sql.elements import ColumnElement -from sqlrules.cache import ModelIRCache, default_cache +from sqlrules.backend import total_predicate from sqlrules.columns import resolve_column -from sqlrules.constraints import ensure_supported_type, extract_constraints -from sqlrules.errors import ConfigurationError -from sqlrules.inspectors import inspect_model +from sqlrules.errors import CapabilityError, ConfigurationError, MissingColumnError, PluginError from sqlrules.ir import ( CompilationContext, - Diagnostic, + CompiledRules, DiagnosticsCollector, - FieldIR, - ModelIR, + FieldResult, OnConflict, - OnUnsupported, + RuleField, + SchemaSpec, ) -from sqlrules.plugins import SQLRulesPlugin, validate_plugin +from sqlrules.models import RuleSchema, normalize_schema +from sqlrules.plugins import BackendProvider, SQLRulesPlugin, validate_backend, validate_plugin from sqlrules.translators import TranslatorRegistry, default_registry -RulesDict = dict[str, list[ColumnElement[bool]]] - class Compiler: - """Compile constrained Pydantic models into SQLAlchemy rule dictionaries. - - ``dialect`` is an optional hint stored on ``CompilationContext`` for - custom translators. It does **not** load dialect plugins or change - built-in translations — pass ``plugins=[...]`` explicitly. - - ``emit_type_checks`` (default ``False``) extracts ``type_check`` IR from - supported scalar annotations. Translation requires a dialect plugin or - custom translator (same pattern as ``pattern``). - - Do not mutate ``registry`` after construction; registration belongs in - ``plugins`` / ``register_constraint`` during init. Do not call - ``compile`` / ``bind`` / ``compile_model`` concurrently on the same - instance. - """ + """Compile SQLRules-owned Pydantic schemas through one explicit backend.""" def __init__( self, *, - registry: TranslatorRegistry | None = None, plugins: Sequence[SQLRulesPlugin] | None = None, + registry: TranslatorRegistry | None = None, on_conflict: OnConflict = "raise", - on_unsupported: OnUnsupported = "raise", + on_unsupported: str = "raise", dialect: str | None = None, cache: bool = True, - model_cache: ModelIRCache | None = None, - emit_type_checks: bool = False, ) -> None: - """Create a compiler with an optional plugin set and policies. - - Args: - registry: Optional base ``TranslatorRegistry``. Always copied; - the caller's registry is never mutated. - plugins: ``SQLRulesPlugin`` instances registered at init. - on_conflict: Default conflict policy for plugin registration - (``"raise"``, ``"replace"``, or ``"ignore"``). - on_unsupported: Policy for unknown **operators** (``"raise"``, - ``"warn"``, or ``"ignore"``). Unsupported **types** always - raise. - dialect: Hint stored on ``CompilationContext`` only — does not - load plugins. - cache: When ``True``, cache Phase-1 ``ModelIR`` in ``model_cache``. - model_cache: Optional cache instance; defaults to the process-wide - shared cache. Custom caches are not cleared by - ``clear_model_cache()``. - emit_type_checks: When ``True``, extract ``type_check`` IR for - supported scalars (needs a plugin translator to bind). - - Raises: - ConfigurationError: Invalid ``on_unsupported`` or ``on_conflict``. - PluginError: Plugin failed validation (missing attributes or - ``api_version`` mismatch). - RegistryError: Plugin registration conflict under ``on_conflict``. - InvalidTranslatorError: Plugin registered a non-callable or - wrong-arity translator. - """ - if on_unsupported not in {"raise", "warn", "ignore"}: - raise ConfigurationError(option="on_unsupported", value=on_unsupported) if on_conflict not in {"raise", "replace", "ignore"}: - raise ConfigurationError(option="on_conflict", value=on_conflict) - - # Always copy so caller-owned and shared default registries are never mutated. - # default_registry() already returns a copy of the builtin template. + raise ConfigurationError("on_conflict", on_conflict) + if on_unsupported != "raise": + raise ConfigurationError("on_unsupported", on_unsupported) base = registry.copy() if registry is not None else default_registry() plugin_list = list(plugins or ()) - resolved: TranslatorRegistry - if plugin_list: - aware = _PluginAwareRegistry( - base, - default_on_conflict=on_conflict, - ) - for plugin in plugin_list: - validate_plugin(plugin).register(aware) - # Freeze to a plain registry after registration. - resolved = aware.copy() - else: - resolved = base - self.registry = resolved - - self.on_unsupported = on_unsupported - self.on_conflict = on_conflict + for plugin in plugin_list: + validate_plugin(plugin) + if sum(_is_backend(plugin) for plugin in plugin_list) > 1: + raise PluginError(message="Pass exactly one backend provider to Compiler(plugins=...).") + aware = _PluginAwareRegistry(base, default_on_conflict=on_conflict) + for plugin in plugin_list: + plugin.register(aware) + self._registry = aware.copy() + self.plugins = tuple(plugin_list) self.dialect = dialect + # The option remains for the 1.x constructor surface. Schema metadata is + # normalized per call and table-bound state is never cached. self.cache_enabled = cache - self.emit_type_checks = emit_type_checks - self._model_cache = model_cache if model_cache is not None else default_cache() - self._collector = DiagnosticsCollector() - self.context = CompilationContext( - on_unsupported=on_unsupported, - collector=self._collector, - dialect=dialect, - ) @property - def diagnostics(self) -> tuple[Diagnostic, ...]: - """Diagnostics from the most recent ``compile`` / ``bind`` call.""" - return self._collector.snapshot() - - def compile_model(self, model: type[BaseModel]) -> ModelIR: - """Phase 1: inspect the model and extract constraint IR (no table binding).""" - self._collector.clear() - if self.cache_enabled: - cached = self._model_cache.get(model, emit_type_checks=self.emit_type_checks) - if cached is not None: - return cached - - model_strict = bool(getattr(model, "model_config", {}).get("strict", False)) - fields: list[FieldIR] = [] - for descriptor in inspect_model(model): - ensure_supported_type(descriptor) - constraints = tuple( - extract_constraints( - descriptor, - emit_type_checks=self.emit_type_checks, - model_strict=model_strict, + def registry(self) -> TranslatorRegistry: + """Return a copy of this compiler's immutable translator snapshot.""" + return self._registry.copy() + + @property + def backend(self) -> BackendProvider: + providers = [plugin for plugin in self.plugins if _is_backend(plugin)] + if len(providers) != 1: + raise PluginError( + message=( + "SQLRules 2.0 requires exactly one explicit backend provider in plugins=. " + f"Found {len(providers)}. Pass one dialect plugin such as PostgresPlugin()." ) ) - fields.append(FieldIR(descriptor=descriptor, constraints=constraints)) + return validate_backend(providers[0]) - model_ir = ModelIR(model=model, fields=tuple(fields)) - if self.cache_enabled: - return self._model_cache.put(model, model_ir, emit_type_checks=self.emit_type_checks) - return model_ir + def capabilities(self) -> Mapping[str, Any]: + """Return the configured backend's declared capability profile.""" + return dict(self.backend.capabilities()) + + def compile_model(self, model: type[RuleSchema]) -> SchemaSpec: + """Normalize a RuleSchema class without binding database columns.""" + if not isinstance(model, type) or not issubclass(model, RuleSchema): + from sqlrules.errors import InvalidModelError + + raise InvalidModelError(model=model) + # Re-normalize on each call. RuleSchema classes are not globally cached, + # so mutable extension payloads cannot leave stale IR behind. + return normalize_schema(model) def bind( self, - model_ir: ModelIR, + schema: SchemaSpec, table: Any, *, column_map: Mapping[str, ColumnElement[Any]] | None = None, - ) -> RulesDict: - """Phase 2: resolve columns and translate IR into SQLAlchemy expressions.""" - self._collector.clear() - # Refresh context in case on_unsupported was mutated (should not be). - self.context = CompilationContext( - on_unsupported=self.on_unsupported, - collector=self._collector, - dialect=self.dialect, + ) -> CompiledRules: + """Bind normalized fields and produce one total root predicate.""" + provider = self.backend + capability_info = dict(provider.capabilities()) + server_version = getattr(provider, "server_version", None) + if self.dialect is not None and self.dialect != provider.name: + raise ConfigurationError("dialect", self.dialect) + assumptions = tuple(capability_info.get("assumptions", ())) + collector = DiagnosticsCollector() + context = CompilationContext( + on_unsupported="raise", + collector=collector, + dialect=provider.name, + server_version=server_version, + assumptions=assumptions, ) - rules: RulesDict = {} - for field_ir in model_ir.fields: - if not field_ir.constraints: - continue - - descriptor = field_ir.descriptor - expressions: list[ColumnElement[bool]] = [] - column: ColumnElement[Any] | None = None - - for constraint in field_ir.constraints: - # Resolve the column only when a translator will run, so - # on_unsupported=warn|ignore can skip missing columns for - # entirely unsupported fields (e.g. pattern-only). - if self.registry.lookup(constraint.operator) is None: - self.registry.handle_missing_translator(constraint, self.context) - continue - if column is None: - column = resolve_column( - descriptor.name, - table, - column_map, - alias=descriptor.alias, - aliases=descriptor.aliases, + field_results: list[FieldResult] = [] + root_expressions: list[ColumnElement[bool]] = [] + for field in schema.fields: + column = _resolve_rule_column(field, table, column_map) + prepared = provider.prepare_value(column, field, context) + constraint_expressions: list[ColumnElement[bool]] = [] + for constraint in field.constraints: + if self._registry.lookup(constraint.operator) is None: + raise CapabilityError( + provider.name, + field.name, + prepared.logical_type, + type(column.type).__name__, + f"no translator is registered for retained operator " + f"{constraint.operator!r}.", ) - expression = self.registry.translate(constraint, column, self.context) - if expression is not None: - expressions.append(expression) - - if expressions: - rules[descriptor.name] = expressions + expression = self._registry.translate(constraint, prepared.value, context) + constraint_expressions.append(expression) + + matched_value = and_(prepared.valid, *constraint_expressions) + if field.nullable: + field_predicate = prepared.is_null | matched_value + else: + field_predicate = column.is_not(None) & matched_value + field_predicate = total_predicate(field_predicate) + root_expressions.append(field_predicate) + field_results.append( + FieldResult( + name=field.name, + predicate=field_predicate, + column=column, + logical_type=prepared.logical_type, + strict=field.strict, + nullable=field.nullable, + coercion=prepared.coercion, + capability=prepared.capability, + ) + ) - return rules + predicate = total_predicate(and_(*root_expressions) if root_expressions else true()) + return CompiledRules( + predicate=predicate, + fields=tuple(field_results), + diagnostics=collector.snapshot(), + backend=provider.name, + server_version=server_version, + assumptions=assumptions, + provenance=schema.provenance, + ) def compile( self, - model: type[BaseModel], + model: type[RuleSchema], table: Any, *, column_map: Mapping[str, ColumnElement[Any]] | None = None, - ) -> RulesDict: - """Compile ``model`` constraints and bind them to ``table`` columns. - - Args: - model: Pydantic ``BaseModel`` subclass whose Field metadata is - compiled (instance values are never applied). - table: SQLAlchemy ``Table``, selectable, ORM class, or object - with ``.c``. - column_map: Optional field-name or alias → column mapping. - - Returns: - Mapping of Python field names to lists of boolean - ``ColumnElement`` expressions. Unconstrained supported-type - fields are omitted. - - Raises: - InvalidModelError: ``model`` is not a usable Pydantic model. - MissingColumnError: A constrained field could not bind. - UnsupportedConstraintError: No translator for an operator/type, - or ``emit_type_checks`` / ``pattern`` without a translator. - TranslatorError: A registered translator failed at bind time. - ConfigurationError: Invalid compiler configuration. - """ + ) -> CompiledRules: return self.bind(self.compile_model(model), table, column_map=column_map) +def _is_backend(plugin: Any) -> bool: + return callable(getattr(plugin, "prepare_value", None)) and callable( + getattr(plugin, "capabilities", None) + ) + + class _PluginAwareRegistry(TranslatorRegistry): - """Registry that applies Compiler.on_conflict as the default for register().""" + """Apply the compiler's conflict policy to plugin register() calls.""" - def __init__( - self, - base: TranslatorRegistry, - *, - default_on_conflict: OnConflict, - ) -> None: + def __init__(self, base: TranslatorRegistry, *, default_on_conflict: OnConflict) -> None: super().__init__() - for operator_name in base.operators(): - translator = base.lookup(operator_name) - if translator is not None: - # Already validated on the source registry. - self._translators[operator_name] = translator + self._translators = { + name: translator + for name in base.operators() + if (translator := base.lookup(name)) is not None + } self._default_on_conflict = default_on_conflict - def register( - self, - operator_name: str, - translator: Any, - *, - replace: bool = False, - ) -> None: - on_conflict: OnConflict = "replace" if replace else self._default_on_conflict - self.register_constraint(operator_name, translator, on_conflict=on_conflict) + def register(self, operator_name: str, translator: Any, *, replace: bool = False) -> None: + policy = "replace" if replace else self._default_on_conflict + self.register_constraint(operator_name, translator, on_conflict=policy) def register_constraint( self, @@ -269,93 +202,60 @@ def register_constraint( super().register_constraint( operator_name, translator, - on_conflict=on_conflict if on_conflict is not None else self._default_on_conflict, + on_conflict=on_conflict or self._default_on_conflict, ) -def clear_model_cache() -> None: - """Clear the process-wide default Phase-1 ``ModelIR`` cache. +def _resolve_rule_column( + field: RuleField, + table: Any, + column_map: Mapping[str, ColumnElement[Any]] | None, +) -> ColumnElement[Any]: + if column_map and field.name in column_map: + return resolve_column(field.name, table, column_map=column_map) + if field.column is not None: + try: + return resolve_column(field.column, table) + except MissingColumnError as exc: + raise MissingColumnError(field=field.name) from exc + return resolve_column(field.name, table) - Use this when creating many ephemeral models (for example - ``pydantic.create_model``) so cached IR does not grow without bound. - Compilers that were given a custom ``model_cache`` are unaffected. - """ - default_cache().clear() + +def clear_model_cache() -> None: + """Compatibility no-op; normalized schema lives immutably on each class.""" def compile( - model: type[BaseModel], + model: type[RuleSchema], table: Any, *, + plugins: Sequence[SQLRulesPlugin], column_map: Mapping[str, ColumnElement[Any]] | None = None, - on_unsupported: OnUnsupported = "raise", - cache: bool = True, - emit_type_checks: bool = False, -) -> RulesDict: - """One-shot Application API compile (no plugins). - - Compiles **Field constraint metadata** into SQLAlchemy expressions. - Does not apply model instance / request values. - - Creates a fresh ``Compiler`` per call so concurrent module-level - ``compile()`` invocations do not share mutable diagnostics state. - Phase-1 IR still uses the process-wide cache when ``cache=True``. - - Args: - model: Pydantic ``BaseModel`` subclass. - table: SQLAlchemy ``Table``, selectable, ORM class, or object with - ``.c``. - column_map: Optional field-name or alias → column mapping. - on_unsupported: Policy for unknown **operators** (``"raise"``, - ``"warn"``, or ``"ignore"``). Unsupported **types** always raise. - cache: Cache Phase-1 model IR (default ``True``). - emit_type_checks: Emit ``type_check`` IR for supported scalars - (requires a plugin translator — use ``Compiler(plugins=...)``). - - Returns: - Mapping of Python field names to lists of boolean - ``ColumnElement`` expressions. - - Raises: - InvalidModelError: ``model`` is not a usable Pydantic model. - MissingColumnError: A constrained field could not bind. - UnsupportedConstraintError: No translator for an operator/type. - TranslatorError: A registered translator failed at bind time. - ConfigurationError: Invalid ``on_unsupported`` value. - - Note: - For plugins, construct ``Compiler(plugins=[...])`` explicitly. - """ - return Compiler( - on_unsupported=on_unsupported, - cache=cache, - emit_type_checks=emit_type_checks, - ).compile(model, table, column_map=column_map) - - -def flatten(rules: RulesDict) -> list[ColumnElement[bool]]: - """Flatten a rules dict into a single list of expressions. - - Args: - rules: Mapping of field name to lists of boolean column elements, - as returned by ``compile`` / ``Compiler.compile``. - - Returns: - Concatenated list suitable for spreading into a SQLAlchemy - ``where`` clause. - """ - return list(chain.from_iterable(rules.values())) - - -def where(rules: RulesDict) -> list[ColumnElement[bool]]: - """Flatten a rules dict for a SQLAlchemy ``where`` clause. - - Identical to :func:`flatten`. Prefer this name at call sites. - - Args: - rules: Mapping of field name to lists of boolean column elements. - - Returns: - Concatenated list of expressions. - """ - return flatten(rules) +) -> CompiledRules: + """One-shot 2.0 compile using explicitly selected backend plugins.""" + return Compiler(plugins=plugins).compile(model, table, column_map=column_map) + + +def flatten(compiled: CompiledRules) -> list[ColumnElement[bool]]: + """Return the authoritative root predicate in spread-friendly list form.""" + _require_compiled(compiled) + return [compiled.predicate] + + +def where(compiled: CompiledRules) -> list[ColumnElement[bool]]: + """Return one complete predicate for a spread-style SQLAlchemy WHERE call.""" + return flatten(compiled) + + +def notwhere(compiled: CompiledRules) -> list[ColumnElement[bool]]: + """Return the total complement selecting rows that fail any rule.""" + _require_compiled(compiled) + return [~compiled.predicate] + + +def _require_compiled(value: Any) -> None: + if not isinstance(value, CompiledRules): + raise TypeError("where(), notwhere(), and flatten() require a CompiledRules result") + + +__all__ = ["Compiler", "clear_model_cache", "compile", "flatten", "notwhere", "where"] diff --git a/src/sqlrules/conformance.py b/src/sqlrules/conformance.py index a0b616f..3c05198 100644 --- a/src/sqlrules/conformance.py +++ b/src/sqlrules/conformance.py @@ -5,12 +5,13 @@ import contextlib from typing import Annotated, Any -from pydantic import BaseModel, Field +from pydantic import Field from sqlalchemy import Column, MetaData, String, Table from sqlalchemy.sql.elements import ColumnElement from sqlrules.compiler import Compiler from sqlrules.errors import PluginError, RegistryError +from sqlrules.models import RuleSchema from sqlrules.plugins import PLUGIN_API_VERSION, SQLRulesPlugin, validate_plugin from sqlrules.translators import TranslatorRegistry, default_registry @@ -76,7 +77,7 @@ def assert_translates_operator( plugin: SQLRulesPlugin, *, operator: str, - model: type[BaseModel] | None = None, + model: type[RuleSchema] | None = None, table: Any | None = None, field: str = "name", ) -> ColumnElement[bool]: @@ -84,11 +85,24 @@ def assert_translates_operator( validate_plugin(plugin) if model is None: + if operator == "min_length": - class PatternFilter(BaseModel): - name: Annotated[str, Field(pattern=r"^a")] + class MinimumLengthFilter(RuleSchema): + name: Annotated[str, Field(min_length=2)] - model = PatternFilter + model = MinimumLengthFilter + elif operator == "max_length": + + class MaximumLengthFilter(RuleSchema): + name: Annotated[str, Field(max_length=20)] + + model = MaximumLengthFilter + else: + + class PatternFilter(RuleSchema): + name: Annotated[str, Field(pattern=r"^a")] + + model = PatternFilter if table is None: table = Table("items", MetaData(), Column("name", String)) @@ -97,11 +111,17 @@ class PatternFilter(BaseModel): if operator not in compiler.registry: raise AssertionError(f"Plugin {plugin.name!r} did not register operator {operator!r}") - rules = compiler.compile(model, table) - if field not in rules or not rules[field]: + schema = compiler.compile_model(model) + rule_field = next((item for item in schema.fields if item.name == field), None) + if rule_field is None or not any(item.operator == operator for item in rule_field.constraints): raise AssertionError(f"Plugin {plugin.name!r}: expected expressions for field {field!r}") - expression = rules[field][0] + compiled = compiler.compile(model, table) + field_result = next((item for item in compiled.fields if item.name == field), None) + if field_result is None: + raise AssertionError(f"Plugin {plugin.name!r}: missing compiled field {field!r}") + + expression = field_result.predicate if not isinstance(expression, ColumnElement): raise AssertionError( f"Plugin {plugin.name!r}: translator for {operator!r} " @@ -121,7 +141,7 @@ def run_basic_conformance( plugin: SQLRulesPlugin, *, operator: str = "pattern", - model: type[BaseModel] | None = None, + model: type[RuleSchema] | None = None, table: Any | None = None, field: str = "name", ) -> None: diff --git a/src/sqlrules/constraints.py b/src/sqlrules/constraints.py index ed230dd..189d1da 100644 --- a/src/sqlrules/constraints.py +++ b/src/sqlrules/constraints.py @@ -1,8 +1,7 @@ from __future__ import annotations import re -from datetime import date, datetime, time, timedelta -from decimal import Decimal +from datetime import date, datetime, time from enum import Enum, Flag from types import UnionType from typing import Annotated, Any, Literal, Union, get_args, get_origin @@ -22,15 +21,9 @@ from pydantic.fields import FieldInfo from sqlrules.errors import UnsupportedConstraintError -from sqlrules.ir import Constraint, FieldDescriptor, PatternSpec, TypeSpec +from sqlrules.ir import Constraint, FieldDescriptor, PatternSpec from sqlrules.markers import ConstraintMarker -_SUPPORTED_TYPES: frozenset[type[Any]] = frozenset( - {bool, int, float, Decimal, str, date, datetime, time, UUID} -) -_SUPPORTED_CONTAINER_ORIGINS: frozenset[Any] = frozenset({list, dict}) -_UNSUPPORTED_TYPES: frozenset[type[Any]] = frozenset({timedelta}) -_UNSUPPORTED_ORIGINS: frozenset[Any] = frozenset({tuple, set, frozenset}) _LENGTH_OPERATORS: frozenset[str] = frozenset({"min_length", "max_length"}) _NUMERIC_OPERATORS: frozenset[str] = frozenset({"gt", "ge", "lt", "le", "multiple_of"}) _PORTABLE_OPERATORS: frozenset[str] = _LENGTH_OPERATORS | _NUMERIC_OPERATORS | {"pattern"} @@ -133,98 +126,12 @@ def _concrete_type(field: FieldDescriptor) -> Any: return annotation -def _annotation_allows_none(annotation: Any) -> bool: - """Return True when the raw annotation admits ``None`` (Optional / ``T | None``).""" - current = annotation - while True: - origin = get_origin(current) - if origin is Annotated: - args = get_args(current) - if not args: - return False - current = args[0] - continue - if origin is Union or origin is UnionType: - return any(arg is type(None) for arg in get_args(current)) - return False - - -def _is_strict_marker(item: Any) -> bool: - if isinstance(item, type): - return False - return type(item).__name__ == "Strict" and hasattr(item, "strict") - - -def resolve_field_strict(field: FieldDescriptor, *, model_strict: bool = False) -> bool: - """Resolve Pydantic strictness: field ``Strict`` / ``strict=`` then model config.""" - for item in _iter_metadata(_field_metadata(field)): - if _is_strict_marker(item): - return bool(item.strict) - data = getattr(item, "__dict__", None) - if isinstance(data, dict) and "strict" in data and data["strict"] is not None: - # e.g. StringConstraints(strict=True, ...) when not already a Strict marker - return bool(data["strict"]) - return model_strict - - def _is_container_type(annotation: Any) -> bool: - origin = get_origin(annotation) - if origin in _SUPPORTED_CONTAINER_ORIGINS: - return True - return annotation in _SUPPORTED_CONTAINER_ORIGINS - - -def ensure_supported_type(field: FieldDescriptor) -> None: - """Raise when a field annotation is outside the support matrix.""" - annotation = _concrete_type(field) - origin = get_origin(annotation) - - if origin is Literal: - return - - if origin in _UNSUPPORTED_ORIGINS or annotation in _UNSUPPORTED_ORIGINS: - raise UnsupportedConstraintError( - field=field.name, - operator=getattr(origin or annotation, "__name__", str(origin or annotation)), - value=annotation, - suggestion=( - "Remove the field or wait for a future SQLRules release with " - f"{getattr(origin or annotation, '__name__', origin or annotation)!r} support." - ), - ) - - if _is_container_type(annotation): - return - - if isinstance(annotation, type) and issubclass(annotation, Enum): - return - - if annotation in _SUPPORTED_TYPES: - return - - if annotation in _UNSUPPORTED_TYPES or ( - isinstance(annotation, type) and annotation in _UNSUPPORTED_TYPES - ): - type_name = getattr(annotation, "__name__", str(annotation)) - raise UnsupportedConstraintError( - field=field.name, - operator=type_name, - value=annotation, - suggestion=f"Type {type_name!r} is not supported by SQLRules.", - ) - - # Unions with multiple non-None members and other unknown annotations. - type_name = getattr(annotation, "__name__", repr(annotation)) - raise UnsupportedConstraintError( - field=field.name, - operator="type", - value=annotation, - suggestion=f"Annotation {type_name} is outside the SQLRules type support matrix.", - ) + return get_origin(annotation) in {list, dict} or annotation in {list, dict} # Flags that change pattern text semantics and are not represented in PatternSpec. -_UNSUPPORTED_PATTERN_FLAGS = re.VERBOSE | re.DOTALL | re.MULTILINE +_UNSUPPORTED_PATTERN_FLAGS = re.ASCII | re.VERBOSE | re.DOTALL | re.MULTILINE def _normalize_pattern(field_name: str, pattern: Any) -> PatternSpec: @@ -238,6 +145,7 @@ def _normalize_pattern(field_name: str, pattern: Any) -> PatternSpec: names = [ name for name, bit in ( + ("ASCII", re.ASCII), ("VERBOSE", re.VERBOSE), ("DOTALL", re.DOTALL), ("MULTILINE", re.MULTILINE), @@ -276,13 +184,6 @@ def pattern_text(value: Any) -> tuple[str, bool]: raise TypeError(f"pattern value must be str or PatternSpec, got {type(value)!r}") -def type_spec(value: Any) -> TypeSpec: - """Return a ``TypeSpec`` from a ``type_check`` constraint value.""" - if isinstance(value, TypeSpec): - return value - raise TypeError(f"type_check value must be TypeSpec, got {type(value)!r}") - - def _constraints_from_mapping(field_name: str, data: dict[str, Any]) -> list[Constraint]: """Map whitelisted metadata keys into IR; ignore validation-only flags.""" constraints: list[Constraint] = [] @@ -328,8 +229,8 @@ def _unsupported_constraints(field_name: str, item: Any) -> list[Constraint]: if _is_constraint_marker(item): return [Constraint(field_name, item.operator, item.value)] - if _is_strict_marker(item): - # Strictness is resolved separately for type_check; not an SQL operator. + if type(item).__name__ == "Strict" and hasattr(item, "strict"): + # Strictness is normalized into the RuleField, not a constraint. return [] if isinstance(item, Predicate): @@ -368,17 +269,6 @@ def _unsupported_constraints(field_name: str, item: Any) -> list[Constraint]: return [Constraint(field_name, type_name, item)] -def _should_emit_type_check(annotation: Any) -> bool: - origin = get_origin(annotation) - if origin is Literal: - return False - if _is_container_type(annotation): - return False - if isinstance(annotation, type) and issubclass(annotation, Enum): - return False - return annotation in _SUPPORTED_TYPES - - def _reject_type_operator_mismatch(field: FieldDescriptor, constraint: Constraint) -> None: annotation = _concrete_type(field) origin = get_origin(annotation) @@ -471,9 +361,6 @@ def _reject_type_operator_mismatch(field: FieldDescriptor, constraint: Constrain def extract_constraints( field: FieldDescriptor, - *, - emit_type_checks: bool = False, - model_strict: bool = False, ) -> list[Constraint]: constraints: list[Constraint] = [] @@ -533,20 +420,6 @@ def extract_constraints( ) constraints.append(Constraint(field.name, "enum", values)) - if emit_type_checks and _should_emit_type_check(annotation): - strict = resolve_field_strict(field, model_strict=model_strict) - constraints.append( - Constraint( - field.name, - "type_check", - TypeSpec( - python_type=annotation, - strict=strict, - allow_none=_annotation_allows_none(field.annotation), - ), - ) - ) - for constraint in constraints: if constraint.operator in _PORTABLE_OPERATORS: _reject_type_operator_mismatch(field, constraint) diff --git a/src/sqlrules/errors.py b/src/sqlrules/errors.py index f28b8d3..3b0bce7 100644 --- a/src/sqlrules/errors.py +++ b/src/sqlrules/errors.py @@ -14,8 +14,9 @@ class InvalidModelError(SQLRulesError): def __str__(self) -> str: return ( - f"Expected a Pydantic BaseModel subclass, got {self.model!r}. " - "Pass a model class such as class UserFilter(BaseModel): ..." + f"Expected a SQLRules RuleSchema subclass, got {self.model!r}. " + "Convert unrestricted Pydantic models with " + "sqlrules.integrations.pydantic.from_pydantic()." ) @@ -43,7 +44,10 @@ def __str__(self) -> str: ) if self.suggestion: return f"{message} {self.suggestion}" - return f"{message} Remove the constraint, or set on_unsupported='warn'/'ignore'." + return ( + f"{message} Remove it or use from_pydantic() to convert and report " + "unsupported declarations." + ) @dataclass(slots=True) @@ -88,7 +92,7 @@ def __str__(self) -> str: if self.option == "on_unsupported": return ( f"Invalid configuration value for {self.option!r}: {self.value!r}. " - "Use one of: 'raise', 'warn', 'ignore'." + "SQLRules 2.0 requires 'raise' so retained rules cannot be omitted." ) if self.option == "on_conflict": return ( @@ -98,6 +102,23 @@ def __str__(self) -> str: return f"Invalid configuration value for {self.option!r}: {self.value!r}." +@dataclass(slots=True) +class CapabilityError(SQLRulesError): + """A backend cannot implement a retained rule for a bound source type.""" + + backend: str + field: str + target_type: str + source_type: str + reason: str + + def __str__(self) -> str: + return ( + f"Backend {self.backend!r} cannot compile field {self.field!r} from " + f"{self.source_type} to {self.target_type}: {self.reason}" + ) + + @dataclass(slots=True) class PluginError(SQLRulesError): """Raised when a plugin fails validation or registration.""" diff --git a/src/sqlrules/inspectors.py b/src/sqlrules/inspectors.py deleted file mode 100644 index e2f68bc..0000000 --- a/src/sqlrules/inspectors.py +++ /dev/null @@ -1,54 +0,0 @@ -from __future__ import annotations - -from typing import Any - -from pydantic import AliasChoices, AliasPath, BaseModel - -from sqlrules.errors import InvalidModelError -from sqlrules.ir import FieldDescriptor - - -def _string_alias(value: Any) -> str | None: - if isinstance(value, str): - return value - return None - - -def _field_aliases(field: Any) -> tuple[str | None, tuple[str, ...]]: - """Return primary alias and extra string alias candidates for column binding.""" - aliases: list[str] = [] - - primary = _string_alias(field.alias) - if primary: - aliases.append(primary) - - for attr in ("validation_alias", "serialization_alias"): - value = getattr(field, attr, None) - # AliasPath / AliasChoices are not used for column binding. - if isinstance(value, (AliasPath, AliasChoices)): - continue - candidate = _string_alias(value) - if candidate and candidate not in aliases: - aliases.append(candidate) - - primary_alias = aliases[0] if aliases else None - return primary_alias, tuple(aliases) - - -def inspect_model(model: type[BaseModel]) -> list[FieldDescriptor]: - if not isinstance(model, type) or not issubclass(model, BaseModel): - raise InvalidModelError(model=model) - - descriptors: list[FieldDescriptor] = [] - for name, field in model.model_fields.items(): - primary_alias, aliases = _field_aliases(field) - descriptors.append( - FieldDescriptor( - name=name, - alias=primary_alias, - aliases=aliases, - annotation=field.annotation, - metadata=tuple(field.metadata), - ) - ) - return descriptors diff --git a/src/sqlrules/integrations/__init__.py b/src/sqlrules/integrations/__init__.py new file mode 100644 index 0000000..f0b34fb --- /dev/null +++ b/src/sqlrules/integrations/__init__.py @@ -0,0 +1,10 @@ +"""Optional integration helpers bundled with SQLRules.""" + +from sqlrules.integrations.pydantic import ( + ConversionEntry, + ConversionReport, + PydanticConversion, + from_pydantic, +) + +__all__ = ["ConversionEntry", "ConversionReport", "PydanticConversion", "from_pydantic"] diff --git a/src/sqlrules/integrations/pydantic.py b/src/sqlrules/integrations/pydantic.py new file mode 100644 index 0000000..af36ede --- /dev/null +++ b/src/sqlrules/integrations/pydantic.py @@ -0,0 +1,451 @@ +from __future__ import annotations + +import copy +import warnings +from dataclasses import dataclass +from typing import Any, Literal, cast + +from annotated_types import MaxLen, MinLen +from pydantic import BaseModel, ConfigDict, StringConstraints, create_model +from pydantic_core import PydanticUndefined + +from sqlrules.errors import InvalidModelError, UnsupportedConstraintError +from sqlrules.models import ( + RuleConfig, + RuleSchema, + _domain_type, + _metadata_values, + _recognized_metadata, + _unwrap, + normalize_schema, +) + +ConversionPolicy = Literal["warn", "drop", "raise"] +_dynamic_create_model = cast(Any, create_model) + + +@dataclass(frozen=True, slots=True) +class ConversionEntry: + """One deterministic explanation of a preserved, changed, or dropped feature.""" + + location: str + feature: str + reason_code: str + outcome: Literal["preserved", "metadata", "dropped", "changed"] + behavior: Literal["same", "broader", "narrower", "unknown"] + reason: str + + +@dataclass(frozen=True, slots=True) +class ConversionReport: + source_model: str + target_model: str + entries: tuple[ConversionEntry, ...] + allow_empty: bool + + @property + def has_incompatibilities(self) -> bool: + return any(item.outcome in {"dropped", "changed"} for item in self.entries) + + +@dataclass(frozen=True, slots=True) +class PydanticConversion: + model: type[RuleSchema] + report: ConversionReport + + +def _model_name(model: type[Any]) -> str: + return f"{model.__module__}.{model.__qualname__}" + + +def _decorator_features(model: type[BaseModel]) -> list[tuple[str, str]]: + decorators = getattr(model, "__pydantic_decorators__", None) + result: list[tuple[str, str]] = [] + if decorators is not None: + for name in ( + "field_validators", + "model_validators", + "field_serializers", + "model_serializers", + ): + entries = getattr(decorators, name, {}) or {} + result.extend((name, str(item)) for item in entries) + computed = getattr(decorators, "computed_fields", {}) or {} + result.extend(("computed_fields", str(item)) for item in computed) + post_init = getattr(model, "__pydantic_post_init__", None) + if post_init: + result.append(("model_post_init", str(post_init))) + return result + + +def _report_entry( + location: str, + feature: str, + code: str, + outcome: Literal["preserved", "metadata", "dropped", "changed"], + behavior: Literal["same", "broader", "narrower", "unknown"], + reason: str, +) -> ConversionEntry: + return ConversionEntry(location, feature, code, outcome, behavior, reason) + + +def from_pydantic( + model: type[BaseModel], + *, + on_incompatible: ConversionPolicy = "warn", + allow_empty: bool = False, +) -> PydanticConversion: + """Convert a full Pydantic v2 model into the SQLRules-supported subset. + + The source class is never modified. Validators, serializers, unsupported + fields, and metadata are reported and removed from the generated model. + Defaults and default factories are copied as declarations and are never + executed during conversion. + """ + if not isinstance(model, type) or not issubclass(model, BaseModel): + raise InvalidModelError(model=model) + if on_incompatible not in {"warn", "drop", "raise"}: + raise ValueError("on_incompatible must be 'warn', 'drop', or 'raise'") + + entries: list[ConversionEntry] = [] + source_name = _model_name(model) + + if issubclass(model, RuleSchema): + schema = normalize_schema(model) + entries.append( + _report_entry( + source_name, + "RuleSchema declarations", + "already_rules_schema", + "preserved", + "same", + "The source model already satisfies the SQLRules declaration subset.", + ) + ) + return PydanticConversion( + model=model, + report=ConversionReport(source_name, source_name, tuple(entries), schema.allow_empty), + ) + + converted_fields: dict[str, tuple[Any, Any]] = {} + for name, info in model.model_fields.items(): + location = f"{source_name}.{name}" + concrete, nullable, annotation_metadata = _unwrap(info.annotation) + try: + _domain_type(concrete, name) + except UnsupportedConstraintError as exc: + entries.append( + _report_entry( + location, + repr(info.annotation), + "unsupported_field_type", + "dropped", + "unknown", + str(exc), + ) + ) + continue + + metadata = _metadata_values(info, annotation_metadata) + supported_metadata: list[Any] = [] + unsupported_metadata: list[Any] = [] + for item in metadata: + if isinstance(item, StringConstraints): + # StringConstraints is grouped metadata. Split it into its + # independently convertible pieces so an unsupported text + # transform does not discard a compatible length or pattern + # rule along with it. + unsupported_settings = { + key: value + for key, value in vars(item).items() + if key not in {"min_length", "max_length", "pattern", "strict"} + and value is not None + and value is not False + } + for key, value in unsupported_settings.items(): + entries.append( + _report_entry( + location, + f"StringConstraints.{key}={value!r}", + "unsupported_metadata", + "dropped", + "unknown", + "This setting has no SQLRules representation and was removed; " + "compatible StringConstraints members were retained.", + ) + ) + if item.strict is not None: + supported_metadata.append(StringConstraints(strict=item.strict)) + if item.min_length is not None: + supported_metadata.append(MinLen(item.min_length)) + if item.max_length is not None: + supported_metadata.append(MaxLen(item.max_length)) + if item.pattern is not None: + # Keep the pattern last so an invalid/unsupported pattern + # can be dropped without losing the other supported pieces. + supported_metadata.append(StringConstraints(pattern=item.pattern)) + continue + if _recognized_metadata(item): + supported_metadata.append(item) + else: + unsupported_metadata.append(item) + for item in unsupported_metadata: + behavior: Literal["broader", "narrower", "unknown"] = ( + "broader" + if type(item).__name__ in {"Gt", "Ge", "Lt", "Le", "MinLen", "MaxLen"} + else "unknown" + ) + entries.append( + _report_entry( + location, + type(item).__name__, + "unsupported_metadata", + "dropped", + behavior, + "The metadata has no supported SQLRules representation and was removed.", + ) + ) + + candidate_metadata = list(supported_metadata) + clean_annotation = concrete | type(None) if nullable else concrete + clean_info: Any + field_is_supported = False + normalized_probe = None + while True: + clean_info = copy.copy(info) + clean_info.metadata = list(candidate_metadata) + try: + probe = _dynamic_create_model( + f"_{model.__name__}_{name}_Probe", + __base__=RuleSchema, + __module__=model.__module__, + **{name: (clean_annotation, clean_info)}, + ) + normalized_probe = normalize_schema(probe) + field_is_supported = True + break + except (TypeError, UnsupportedConstraintError, ValueError) as exc: + if candidate_metadata: + dropped = candidate_metadata.pop() + entries.append( + _report_entry( + location, + type(dropped).__name__, + "incompatible_constraint_dropped", + "dropped", + "broader", + str(exc), + ) + ) + continue + entries.append( + _report_entry( + location, + "field declaration", + "invalid_rule_declaration", + "dropped", + "unknown", + str(exc), + ) + ) + break + if not field_is_supported: + continue + + converted_fields[name] = (clean_annotation, clean_info) + entries.append( + _report_entry( + location, + "field type and supported constraints", + "field_preserved", + "preserved", + "same", + "The field's supported scalar type and SQL-translatable constraints were retained.", + ) + ) + if normalized_probe is not None: + normalized_field = normalized_probe.fields[0] + for constraint in normalized_field.constraints: + entries.append( + _report_entry( + location, + f"{constraint.operator}={constraint.value!r}", + "constraint_preserved", + "preserved", + "same", + "The SQL-translatable field constraint was retained.", + ) + ) + aliases = tuple( + dict.fromkeys( + alias + for alias in (info.alias, info.validation_alias, info.serialization_alias) + if isinstance(alias, str) + ) + ) + if aliases: + entries.append( + _report_entry( + location, + f"aliases={aliases!r}", + "field_aliases_preserved", + "metadata", + "same", + "Pydantic validation and serialization aliases were retained; " + "they do not change SQL column binding.", + ) + ) + if info.default is not PydanticUndefined or info.default_factory is not None: + entries.append( + _report_entry( + location, + "default or default_factory", + "field_default_preserved", + "metadata", + "same", + "The declaration was copied for Pydantic model use and was not " + "executed during conversion.", + ) + ) + has_descriptive_metadata = any( + getattr(info, name, None) is not None for name in ("title", "description") + ) or bool(getattr(info, "examples", None)) + if has_descriptive_metadata: + entries.append( + _report_entry( + location, + "title, description, or examples", + "descriptive_metadata_preserved", + "metadata", + "same", + "Descriptive Pydantic metadata was retained on the generated field.", + ) + ) + + for feature, name in _decorator_features(model): + entries.append( + _report_entry( + f"{source_name}.{name}", + feature, + "python_callback_removed", + "changed", + "unknown", + "Python callbacks and computed fields are not executed or " + "translated into SQL rules.", + ) + ) + + if getattr(model, "__pydantic_custom_init__", False): + entries.append( + _report_entry( + source_name, + "custom __init__", + "custom_init_removed", + "changed", + "unknown", + "The generated rules model uses Pydantic's standard initializer; " + "the source model's custom initializer was not copied.", + ) + ) + for hook in ("__get_pydantic_core_schema__", "__get_pydantic_json_schema__"): + if any(hook in base.__dict__ for base in model.__mro__ if base is not BaseModel): + entries.append( + _report_entry( + source_name, + hook, + "pydantic_schema_hook_removed", + "changed", + "unknown", + "Custom Pydantic schema hooks are not copied to the generated rules model.", + ) + ) + + source_config = dict(model.model_config) + strict = bool(source_config.pop("strict", False)) + if source_config: + entries.append( + _report_entry( + source_name, + "Pydantic model_config", + "model_config_not_copied", + "changed", + "unknown", + "Only Pydantic strictness is retained as a SQLRules validation " + "setting; other config behavior remains on the source model.", + ) + ) + converted_base = type( + f"_{model.__name__}RuleSchemaBase", + (RuleSchema,), + { + "__module__": model.__module__, + "__rule_config__": RuleConfig(allow_empty=True), + "__sqlrules_provenance__": f"from_pydantic:{source_name}", + "model_config": ConfigDict(strict=strict), + }, + ) + type.__setattr__(converted_base, "__rule_config__", RuleConfig(allow_empty=allow_empty)) + + if not converted_fields and not allow_empty: + entries.append( + _report_entry( + source_name, + "empty converted schema", + "empty_conversion_rejected", + "dropped", + "unknown", + "No SQL-compatible fields remain; pass allow_empty=True to " + "permit an always-true predicate.", + ) + ) + report = ConversionReport(source_name, "", tuple(entries), False) + raise UnsupportedConstraintError( + model.__name__, + "empty_conversion", + report, + "No SQL-compatible fields remain. Pass allow_empty=True to " + "explicitly allow an empty rules model.", + ) + + target_name = f"{model.__name__}Rules" + converted_model = _dynamic_create_model( + target_name, + __base__=converted_base, + __module__=model.__module__, + **converted_fields, + ) + normalize_schema(converted_model) + entries.sort(key=lambda item: (item.location, item.reason_code, item.feature)) + report = ConversionReport( + source_name, _model_name(converted_model), tuple(entries), allow_empty + ) + + if on_incompatible == "raise" and report.has_incompatibilities: + raise UnsupportedConstraintError( + model.__name__, + "pydantic_conversion", + report, + "Conversion would remove or change source-model behavior. Use " + "'drop' to inspect the report or 'warn' to convert with warnings.", + ) + if on_incompatible == "warn" and report.has_incompatibilities: + summary = "; ".join( + f"{item.location}: {item.reason_code} ({item.behavior})" + for item in report.entries + if item.outcome in {"dropped", "changed"} + ) + warnings.warn( + f"Pydantic model conversion to {target_name} changed behavior: {summary}", + UserWarning, + stacklevel=2, + ) + return PydanticConversion(converted_model, report) + + +__all__ = [ + "ConversionEntry", + "ConversionPolicy", + "ConversionReport", + "PydanticConversion", + "from_pydantic", +] diff --git a/src/sqlrules/ir.py b/src/sqlrules/ir.py index d0c39f6..da2f64a 100644 --- a/src/sqlrules/ir.py +++ b/src/sqlrules/ir.py @@ -3,7 +3,9 @@ from dataclasses import dataclass, field from typing import Any, Literal -OnUnsupported = Literal["raise", "warn", "ignore"] +from sqlalchemy.sql.elements import ColumnElement + +OnUnsupported = Literal["raise"] OnConflict = Literal["raise", "replace", "ignore"] DiagnosticSeverity = Literal["warning", "info"] @@ -16,15 +18,6 @@ class PatternSpec: ignore_case: bool = False -@dataclass(frozen=True, slots=True) -class TypeSpec: - """Normalized ``type_check`` constraint value (Pydantic type + strictness).""" - - python_type: type - strict: bool = False - allow_none: bool = False - - @dataclass(frozen=True, slots=True) class Constraint: field: str @@ -41,6 +34,106 @@ class FieldDescriptor: aliases: tuple[str, ...] = () +@dataclass(frozen=True, slots=True) +class RuleField: + """SQLRules-owned normalized field declaration.""" + + name: str + annotation: Any + python_type: Any + nullable: bool + strict: bool + constraints: tuple[Constraint, ...] + column: str | None = None + alias: str | None = None + title: str | None = None + description: str | None = None + examples: tuple[Any, ...] = () + source: str = "native" + source_location: str = "" + + +@dataclass(frozen=True, slots=True) +class SchemaSpec: + """Immutable, table-independent normalized RuleSchema representation.""" + + model: type[Any] + fields: tuple[RuleField, ...] + allow_empty: bool = False + provenance: str = "native" + + +@dataclass(frozen=True, slots=True) +class PreparedValue: + """Backend-prepared source value and total type/coercion evidence.""" + + source: ColumnElement[Any] + value: ColumnElement[Any] + valid: ColumnElement[bool] + is_null: ColumnElement[bool] + logical_type: str + coercion: str = "identity" + capability: str = "native" + + +@dataclass(frozen=True, slots=True) +class FieldResult: + """Bound result for one normalized field.""" + + name: str + predicate: ColumnElement[bool] + column: ColumnElement[Any] + logical_type: str + strict: bool + nullable: bool + coercion: str + capability: str + + +@dataclass(frozen=True, slots=True) +class CompiledRules: + """Complete, total predicate returned by the 2.0 application API.""" + + predicate: ColumnElement[bool] + fields: tuple[FieldResult, ...] + diagnostics: tuple[Diagnostic, ...] = () + backend: str = "" + server_version: tuple[int, ...] | None = None + assumptions: tuple[str, ...] = () + provenance: str = "native" + + def explain(self) -> dict[str, Any]: + """Return a structured compile plan without executing database EXPLAIN.""" + return { + "backend": self.backend, + "server_version": self.server_version, + "assumptions": self.assumptions, + "provenance": self.provenance, + "fields": [ + { + "name": item.name, + "logical_type": item.logical_type, + "strict": item.strict, + "nullable": item.nullable, + "coercion": item.coercion, + "capability": item.capability, + "predicate": str(item.predicate), + } + for item in self.fields + ], + "diagnostics": [ + { + "severity": item.severity, + "field": item.field, + "operator": item.operator, + "code": item.code, + "message": item.message, + } + for item in self.diagnostics + ], + } + + @dataclass(frozen=True, slots=True) class Diagnostic: severity: DiagnosticSeverity @@ -72,6 +165,8 @@ class CompilationContext: on_unsupported: OnUnsupported = "raise" collector: DiagnosticsCollector | None = None dialect: str | None = None + server_version: tuple[int, ...] | None = None + assumptions: tuple[str, ...] = () def record( self, @@ -95,15 +190,3 @@ def record( code=code, ) ) - - -@dataclass(frozen=True, slots=True) -class FieldIR: - descriptor: FieldDescriptor - constraints: tuple[Constraint, ...] - - -@dataclass(frozen=True, slots=True) -class ModelIR: - model: type[Any] - fields: tuple[FieldIR, ...] diff --git a/src/sqlrules/models.py b/src/sqlrules/models.py new file mode 100644 index 0000000..7aba4f4 --- /dev/null +++ b/src/sqlrules/models.py @@ -0,0 +1,527 @@ +from __future__ import annotations + +import math +from dataclasses import dataclass +from datetime import date, datetime, time +from decimal import Decimal +from enum import Enum +from types import UnionType +from typing import Annotated, Any, ClassVar, Literal, Union, get_args, get_origin +from uuid import UUID + +from annotated_types import Ge, Gt, Interval, Le, Lt, MaxLen, MinLen, MultipleOf +from pydantic import BaseModel, ConfigDict +from pydantic import Field as _PydanticField +from pydantic_core import PydanticUndefined + +from sqlrules.errors import InvalidModelError, UnsupportedConstraintError +from sqlrules.ir import Constraint, FieldDescriptor, RuleField, SchemaSpec +from sqlrules.markers import ConstraintMarker + + +@dataclass(frozen=True, slots=True) +class RuleConfig: + """SQLRules-only model options, separate from Pydantic's ``model_config``.""" + + allow_empty: bool = False + + +@dataclass(frozen=True, slots=True) +class ColumnBinding: + """Explicit database column name attached by :func:`Field`.""" + + name: str + + +def Field(default: Any = PydanticUndefined, *, column: str | None = None, **kwargs: Any) -> Any: + """Pydantic ``Field`` with optional SQLRules database-column metadata. + + All standard Pydantic field options retain their normal model behavior. + ``column`` is the only SQLRules-specific option and never changes a + Pydantic alias or serialized field name. + """ + if column is not None and (not isinstance(column, str) or not column): + raise TypeError("column must be a non-empty string") + if default is PydanticUndefined: + result = _PydanticField(**kwargs) + else: + result = _PydanticField(default, **kwargs) + if column is not None: + result.metadata.append(ColumnBinding(column)) + return result + + +_SCALAR_TYPES = frozenset({bool, int, float, Decimal, str, date, datetime, time, UUID}) +_KNOWN_CONSTRAINT_METADATA = (Gt, Ge, Lt, Le, MultipleOf, MinLen, MaxLen, Interval) +_RULE_OPERATORS = frozenset( + { + "gt", + "ge", + "lt", + "le", + "multiple_of", + "min_length", + "max_length", + "pattern", + "literal", + "enum", + "json_contains", + "json_has_key", + "array_contains", + "array_overlap", + "range_contains", + "range_overlap", + "fulltext_match", + } +) + + +def _unwrap(annotation: Any) -> tuple[Any, bool, tuple[Any, ...]]: + nullable = False + metadata: list[Any] = [] + current = annotation + while True: + origin = get_origin(current) + if origin is Annotated: + args = get_args(current) + if not args: + break + current = args[0] + metadata.extend(args[1:]) + continue + if origin in (Union, UnionType): + args = get_args(current) + members = [item for item in args if item is not type(None)] + nullable = nullable or len(members) != len(args) + if len(members) == 1: + current = members[0] + continue + break + return current, nullable, tuple(metadata) + + +def _metadata_values(field: Any, annotation_metadata: tuple[Any, ...]) -> tuple[Any, ...]: + values = list(getattr(field, "metadata", ()) or ()) + for item in annotation_metadata: + if item not in values: + values.append(item) + return tuple(values) + + +def _is_strict_metadata(item: Any) -> bool: + return type(item).__name__ == "Strict" and hasattr(item, "strict") + + +def _recognized_metadata(item: Any) -> bool: + if isinstance(item, (ColumnBinding, ConstraintMarker, *_KNOWN_CONSTRAINT_METADATA)): + return True + if _is_strict_metadata(item): + return True + if type(item).__name__ == "StringConstraints": + allowed = {"min_length", "max_length", "pattern", "strict"} + return all(value is None for key, value in vars(item).items() if key not in allowed) + if type(item).__name__ in {"PydanticGeneralMetadata", "_PydanticGeneralMetadata"}: + allowed = {"pattern", "strict"} + return all(value is None for key, value in vars(item).items() if key not in allowed) + return False + + +def _domain_type(annotation: Any, field_name: str) -> Any: + origin = get_origin(annotation) + if origin is Literal: + values = get_args(annotation) + if not values: + raise UnsupportedConstraintError(field_name, "literal", values) + types = {type(value.value if isinstance(value, Enum) else value) for value in values} + if len(types) != 1 or next(iter(types)) not in _SCALAR_TYPES: + raise UnsupportedConstraintError( + field_name, + "literal", + values, + "Literal values must use one supported scalar type.", + ) + return next(iter(types)) + if isinstance(annotation, type) and issubclass(annotation, Enum): + enum_values = tuple(member.value for member in annotation) + types = {type(value) for value in enum_values} + if not enum_values or len(types) != 1 or next(iter(types)) not in _SCALAR_TYPES: + raise UnsupportedConstraintError( + field_name, + "enum", + annotation, + "Enum values must use one supported scalar type.", + ) + return next(iter(types)) + origin = get_origin(annotation) + if annotation in _SCALAR_TYPES: + return annotation + if origin in (list, dict): + return origin + if annotation in (list, dict): + return annotation + raise UnsupportedConstraintError( + field_name, + "type", + annotation, + "Use a supported scalar, Literal, Enum, list, or dict annotation.", + ) + + +def _strictness(field: Any, metadata: tuple[Any, ...], model_strict: bool) -> bool: + explicit = getattr(field, "strict", None) + if explicit is not None: + return bool(explicit) + # Pydantic stores strict aliases and Annotated Strict markers in metadata. + for item in metadata: + if _is_strict_metadata(item): + return bool(item.strict) + if type(item).__name__ == "StringConstraints" and item.strict is not None: + return bool(item.strict) + return model_strict + + +def _normalize_constraints( + field: FieldDescriptor, + metadata: tuple[Any, ...], +) -> tuple[Constraint, ...]: + from sqlrules.constraints import extract_constraints + + usable = tuple(item for item in metadata if not isinstance(item, ColumnBinding)) + normalized = FieldDescriptor( + name=field.name, + alias=field.alias, + annotation=field.annotation, + metadata=usable, + aliases=field.aliases, + ) + constraints = extract_constraints(normalized) + constraints = [ + Constraint( + item.field, + item.operator, + tuple(value.value if isinstance(value, Enum) else value for value in item.value) + if item.operator == "literal" + else item.value, + ) + for item in constraints + ] + seen: dict[str, Any] = {} + for item in constraints: + if item.operator not in _RULE_OPERATORS: + raise UnsupportedConstraintError( + field.name, + item.operator, + item.value, + "Python validators and unrecognized metadata do not have SQL rule equivalents.", + ) + numeric_value = isinstance(item.value, (int, float, Decimal)) and not isinstance( + item.value, bool + ) + finite = ( + item.value.is_finite() + if isinstance(item.value, Decimal) + else math.isfinite(item.value) + if isinstance(item.value, float) + else True + ) + if item.operator in {"gt", "ge", "lt", "le", "multiple_of"} and ( + isinstance(item.value, (float, Decimal)) and not finite + ): + raise UnsupportedConstraintError( + field.name, + item.operator, + item.value, + "Numeric constraints must use finite values.", + ) + if item.operator == "multiple_of" and (not numeric_value or item.value <= 0): + raise UnsupportedConstraintError( + field.name, + item.operator, + item.value, + "multiple_of must be a positive finite number.", + ) + if item.operator in seen: + raise UnsupportedConstraintError( + field.name, + item.operator, + item.value, + "Duplicate constraints are not allowed; combine them into one declaration.", + ) + seen[item.operator] = item.value + if item.operator in {"min_length", "max_length"} and ( + isinstance(item.value, bool) or not isinstance(item.value, int) or item.value < 0 + ): + raise UnsupportedConstraintError( + field.name, + item.operator, + item.value, + "Length constraints must be non-negative integers.", + ) + min_length = seen.get("min_length") + max_length = seen.get("max_length") + if min_length is not None and max_length is not None and min_length > max_length: + raise UnsupportedConstraintError( + field.name, + "length", + (min_length, max_length), + "min_length cannot exceed max_length.", + ) + lower = [(seen[key], key == "gt") for key in ("gt", "ge") if key in seen] + upper = [(seen[key], key == "lt") for key in ("lt", "le") if key in seen] + if lower and upper: + strongest_lower = max(value for value, _ in lower) + strongest_upper = min(value for value, _ in upper) + lower_exclusive = any(value == strongest_lower and exclusive for value, exclusive in lower) + upper_exclusive = any(value == strongest_upper and exclusive for value, exclusive in upper) + if strongest_lower > strongest_upper or ( + strongest_lower == strongest_upper and (lower_exclusive or upper_exclusive) + ): + raise UnsupportedConstraintError( + field.name, + "bounds", + (tuple(lower), tuple(upper)), + "The declared lower and upper bounds cannot both be satisfied.", + ) + return tuple(constraints) + + +def normalize_schema(model: type[BaseModel]) -> SchemaSpec: + """Normalize one ``RuleSchema`` class into immutable SQLRules fields.""" + if not isinstance(model, type) or not issubclass(model, RuleSchema): + raise InvalidModelError(model=model) + if model is RuleSchema: + raise InvalidModelError(model=model) + + rule_bases = [ + base + for base in model.__bases__ + if isinstance(base, type) and issubclass(base, RuleSchema) and base is not RuleSchema + ] + if len(rule_bases) > 1: + raise UnsupportedConstraintError( + model.__name__, + "multiple_inheritance", + tuple(base.__name__ for base in rule_bases), + "RuleSchema supports single inheritance only.", + ) + + decorators = getattr(model, "__pydantic_decorators__", None) + if decorators is not None: + for name in ( + "field_validators", + "model_validators", + "field_serializers", + "model_serializers", + ): + if getattr(decorators, name, None): + raise UnsupportedConstraintError( + model.__name__, + name, + None, + "Custom Python validators and serializers cannot be compiled as SQL rules.", + ) + if getattr(decorators, "computed_fields", None): + raise UnsupportedConstraintError( + model.__name__, + "computed_field", + None, + "Computed fields do not have a bound database source column.", + ) + if getattr(model, "__pydantic_custom_init__", False): + raise UnsupportedConstraintError( + model.__name__, + "custom_init", + None, + "A custom __init__ has no equivalent SQL rule semantics.", + ) + if getattr(model, "__pydantic_post_init__", None): + raise UnsupportedConstraintError( + model.__name__, + "model_post_init", + None, + "A custom model_post_init callback has no equivalent SQL rule semantics.", + ) + for hook in ("__get_pydantic_core_schema__", "__get_pydantic_json_schema__"): + if any(hook in base.__dict__ for base in model.__mro__ if base is not BaseModel): + raise UnsupportedConstraintError( + model.__name__, + hook, + None, + "Custom Pydantic schema hooks are outside the SQLRules declaration subset.", + ) + + rule_config = getattr(model, "__rule_config__", RuleConfig()) + if not isinstance(rule_config, RuleConfig): + raise TypeError("__rule_config__ must be a RuleConfig instance") + model_strict = bool(model.model_config.get("strict", False)) + incompatible_config = { + key: model.model_config.get(key) + for key in ( + "coerce_numbers_to_str", + "str_strip_whitespace", + "str_to_lower", + "str_to_upper", + "str_min_length", + "str_max_length", + ) + if model.model_config.get(key) not in (None, False) + } + if model.model_config.get("allow_inf_nan") is False: + incompatible_config["allow_inf_nan"] = False + if incompatible_config: + raise UnsupportedConstraintError( + model.__name__, + "model_config", + incompatible_config, + "String coercion, normalization, global length settings, and the " + "non-finite numeric policy are outside the SQLRules 2.0 declaration subset.", + ) + + fields: list[RuleField] = [] + provenance = getattr(model, "__sqlrules_provenance__", "native") + for name, info in model.model_fields.items(): + annotation = info.annotation + concrete, nullable, annotated_metadata = _unwrap(annotation) + metadata = _metadata_values(info, annotated_metadata) + unknown = [item for item in metadata if not _recognized_metadata(item)] + if unknown: + raise UnsupportedConstraintError( + name, + "metadata", + unknown[0], + "Only SQLRules constraints and supported Pydantic constraint metadata are allowed.", + ) + + field_descriptor = FieldDescriptor( + name=name, + alias=info.alias if isinstance(info.alias, str) else None, + annotation=annotation, + metadata=metadata, + aliases=tuple( + alias + for alias in (info.alias, info.validation_alias, info.serialization_alias) + if isinstance(alias, str) + ), + ) + python_type = _domain_type(concrete, name) + constraints = _normalize_constraints( + field_descriptor, + metadata, + ) + strict = _strictness(info, metadata, model_strict) + binding = next((item.name for item in metadata if isinstance(item, ColumnBinding)), None) + operators = {item.operator for item in constraints} + json_markers = {"json_contains", "json_has_key"} + array_markers = {"array_contains", "array_overlap"} + uses_json = bool(operators & json_markers) + uses_array = bool(operators & array_markers) + if uses_json and uses_array: + raise UnsupportedConstraintError( + name, + "marker_type", + tuple(sorted(operators & (json_markers | array_markers))), + "JSON and array markers cannot be combined on one field.", + ) + if uses_json: + if python_type not in {list, dict}: + raise UnsupportedConstraintError( + name, + "json_marker", + annotation, + "JSON markers require a dict or list field.", + ) + if python_type is list and "json_has_key" in operators: + raise UnsupportedConstraintError( + name, + "json_has_key", + annotation, + "JsonHasKey requires a dict field; use ArrayContains for list elements.", + ) + if python_type is list: + python_type = dict + elif uses_array and python_type is not list: + raise UnsupportedConstraintError( + name, + "array_marker", + annotation, + "Array markers require a list field.", + ) + elif not uses_array and python_type in {list, dict}: + raise UnsupportedConstraintError( + name, + "type", + annotation, + "Container fields require a compatible JSON or array marker.", + ) + + if "fulltext_match" in operators and python_type is not str: + raise UnsupportedConstraintError( + name, + "fulltext_match", + annotation, + "FullTextMatch requires a str field.", + ) + + for item in constraints: + if item.operator in {"min_length", "max_length", "pattern"} and python_type is not str: + raise UnsupportedConstraintError(name, item.operator, item.value) + if item.operator in {"gt", "ge", "lt", "le", "multiple_of"} and python_type in { + bool, + str, + UUID, + list, + dict, + }: + raise UnsupportedConstraintError(name, item.operator, item.value) + + examples = getattr(info, "examples", None) or () + fields.append( + RuleField( + name=name, + annotation=annotation, + python_type=python_type, + nullable=nullable, + strict=strict, + constraints=constraints, + column=binding, + alias=info.alias if isinstance(info.alias, str) else None, + title=info.title, + description=info.description, + examples=tuple(examples), + source=provenance, + source_location=f"{model.__module__}.{model.__qualname__}.{name}", + ) + ) + + if not fields and not rule_config.allow_empty: + raise UnsupportedConstraintError( + model.__name__, + "empty_schema", + None, + "Use __rule_config__ = RuleConfig(allow_empty=True) to allow an always-true predicate.", + ) + return SchemaSpec( + model=model, + fields=tuple(fields), + allow_empty=rule_config.allow_empty, + provenance=provenance, + ) + + +class RuleSchema(BaseModel): + """Pydantic model whose field declarations are restricted to SQLRules rules.""" + + model_config = ConfigDict() + __rule_config__: ClassVar[RuleConfig] = RuleConfig() + __sqlrules_provenance__: ClassVar[str] = "native" + + @classmethod + def __pydantic_init_subclass__(cls, **kwargs: Any) -> None: + super().__pydantic_init_subclass__(**kwargs) + try: + normalize_schema(cls) + except (InvalidModelError, UnsupportedConstraintError, TypeError, ValueError) as exc: + raise TypeError(f"Invalid SQLRules RuleSchema {cls.__name__}: {exc}") from exc + + +__all__ = ["ColumnBinding", "Field", "RuleConfig", "RuleSchema", "normalize_schema"] diff --git a/src/sqlrules/plugins.py b/src/sqlrules/plugins.py index 8b3684b..b4fb25b 100644 --- a/src/sqlrules/plugins.py +++ b/src/sqlrules/plugins.py @@ -1,22 +1,22 @@ from __future__ import annotations -from typing import Any, Protocol, runtime_checkable +from collections.abc import Mapping +from typing import Any, Protocol, cast, runtime_checkable from sqlrules.errors import PluginError from sqlrules.translators import TranslatorRegistry -PLUGIN_API_VERSION = "1" -"""Version of the translator plugin contract. +PLUGIN_API_VERSION = "2" +"""Version of the backend preparation and translator plugin contract. -API v1 covers: +API v2 covers: - ``SQLRulesPlugin`` shape (``name``, ``api_version``, ``register``) -- ``TranslatorRegistry.register_constraint`` / ``register`` / ``copy`` / - ``operators`` / ``lookup`` / ``translate`` -- Translator signature ``(Constraint, ColumnElement, CompilationContext)`` +- optional ``BackendProvider`` shape (``prepare_value`` and ``capabilities``) +- translators consume a normalized prepared-value expression +- explicit selection of exactly one backend provider for table binding - IR value types for built-in operators, including ``PatternSpec`` for - ``pattern`` (use ``pattern_text()``; do not assume ``constraint.value`` - is a bare ``str``) and ``TypeSpec`` for ``type_check`` (use ``type_spec()``) + ``pattern`` (use ``pattern_text()``; do not assume a bare ``str``) - Stable marker operator names (``json_contains``, ``array_contains``, …) Bump this string on incompatible changes to any of the above. Package @@ -35,6 +35,21 @@ def register(self, registry: TranslatorRegistry) -> None: """Register constraint translators on ``registry``.""" +@runtime_checkable +class BackendProvider(Protocol): + """A dialect provider that prepares source values and reports capabilities.""" + + name: str + api_version: str + server_version: tuple[int, ...] | None + + def prepare_value(self, column: Any, field: Any, context: Any) -> Any: + """Return a safe PreparedValue for one source column and logical field.""" + + def capabilities(self) -> Mapping[str, Any]: + """Describe the configured server and semantic capabilities.""" + + def validate_plugin(plugin: Any) -> SQLRulesPlugin: """Validate a plugin object against the versioned plugin API.""" if isinstance(plugin, type): @@ -80,3 +95,19 @@ def validate_plugin(plugin: Any) -> SQLRulesPlugin: ) return plugin + + +def validate_backend(plugin: Any) -> BackendProvider: + """Validate that a registered plugin supplies SQLRules API v2 backend hooks.""" + validate_plugin(plugin) + if not callable(getattr(plugin, "prepare_value", None)) or not callable( + getattr(plugin, "capabilities", None) + ): + raise PluginError( + message=( + f"Plugin {getattr(plugin, 'name', plugin)!r} does not provide the " + "API v2 backend hooks prepare_value() and capabilities()." + ), + plugin=plugin, + ) + return cast(BackendProvider, plugin) diff --git a/src/sqlrules/translators.py b/src/sqlrules/translators.py index d47e1c8..ec421c4 100644 --- a/src/sqlrules/translators.py +++ b/src/sqlrules/translators.py @@ -3,18 +3,17 @@ import inspect import math import operator -import sys import threading -import types -import warnings from collections.abc import Callable from decimal import Decimal, InvalidOperation -from typing import Any, cast +from typing import Any, NoReturn, cast -from sqlalchemy import func +from sqlalchemy import Float, Unicode, and_, func, literal, or_ +from sqlalchemy import cast as sa_cast from sqlalchemy.sql.elements import ColumnElement from sqlrules.errors import ( + CapabilityError, InvalidTranslatorError, RegistryError, TranslatorError, @@ -25,23 +24,6 @@ Translator = Callable[[Constraint, ColumnElement[Any], CompilationContext], ColumnElement[bool]] -class SQLRulesWarning(UserWarning): - """Warning emitted when an unsupported constraint is skipped.""" - - -def _warning_stacklevel() -> int: - """Stacklevel that attributes warnings to the first frame outside sqlrules.""" - frame: types.FrameType | None = sys._getframe(1) - level = 1 - while frame is not None: - module = frame.f_globals.get("__name__", "") - if not (isinstance(module, str) and module.startswith("sqlrules")): - return level - frame = frame.f_back - level += 1 - return 2 - - def _binary(op: Callable[[Any, Any], Any]) -> Translator: def translate( constraint: Constraint, @@ -67,6 +49,13 @@ def _is_positive_finite(value: Any) -> bool: return bool(value > 0) +def _logical_type_name(column: ColumnElement[Any]) -> str: + try: + return column.type.python_type.__name__.lower() + except (AttributeError, NotImplementedError): + return type(column.type).__name__.lower() + + def _multiple_of( constraint: Constraint, column: ColumnElement[Any], @@ -80,6 +69,27 @@ def _multiple_of( value=value, suggestion="multiple_of requires a finite positive numeric value.", ) + if isinstance(column.type, Float) or isinstance(value, float): + raise CapabilityError( + context.dialect or "unknown", + constraint.field, + _logical_type_name(column), + type(column.type).__name__, + "the built-in modulo translator does not support floating-point fields or divisors.", + ) + if ( + context.dialect == "sqlite" + and isinstance(value, Decimal) + and value != value.to_integral_value() + ): + raise CapabilityError( + context.dialect, + constraint.field, + _logical_type_name(column), + type(column.type).__name__, + "SQLite converts modulo operands to integers and would truncate this " + "non-integral Decimal divisor.", + ) return cast(ColumnElement[bool], (column % value) == 0) @@ -104,6 +114,24 @@ def _in_values( column: ColumnElement[Any], context: CompilationContext, ) -> ColumnElement[bool]: + values = constraint.value + if not values: + return cast(ColumnElement[bool], column.in_(values)) + if context.dialect in {"mysql", "mssql"} and all(isinstance(value, str) for value in values): + exact_matches: list[ColumnElement[bool]] = [] + for value in values: + bound = literal(value) + if context.dialect == "mysql": + same_length = func.char_length(column) == func.char_length(bound) + else: + # SQL Server pads strings during equality comparisons. Compare + # unbounded Unicode byte lengths to avoid padding and truncation + # from the source column's declared size. + same_length = func.datalength(sa_cast(column, Unicode())) == func.datalength( + sa_cast(bound, Unicode()) + ) + exact_matches.append(and_(column == bound, same_length)) + return or_(*exact_matches) return cast(ColumnElement[bool], column.in_(constraint.value)) @@ -187,38 +215,15 @@ def copy(self) -> TranslatorRegistry: def handle_missing_translator( self, constraint: Constraint, - context: CompilationContext, - ) -> None: - """Apply ``on_unsupported`` policy when no translator is registered.""" - message = ( - f"Field {constraint.field!r}: constraint {constraint.operator!r} " - "is not supported and will be skipped." - ) - if context.on_unsupported == "raise": - raise UnsupportedConstraintError( - field=constraint.field, - operator=constraint.operator, - value=constraint.value, - suggestion=("Remove the constraint, or set on_unsupported='warn'/'ignore'."), - ) - if context.on_unsupported == "warn": - context.record( - severity="warning", - field=constraint.field, - operator=constraint.operator, - value=constraint.value, - message=message, - code="unsupported_constraint", - ) - warnings.warn(message, SQLRulesWarning, stacklevel=_warning_stacklevel()) - return - context.record( - severity="info", + ) -> NoReturn: + """Fail when a retained constraint has no translator.""" + raise UnsupportedConstraintError( field=constraint.field, operator=constraint.operator, value=constraint.value, - message=message, - code="unsupported_constraint", + suggestion=( + "Remove it or use from_pydantic() to convert and report unsupported declarations." + ), ) def translate( @@ -226,15 +231,14 @@ def translate( constraint: Constraint, column: ColumnElement[Any], context: CompilationContext, - ) -> ColumnElement[bool] | None: + ) -> ColumnElement[bool]: translator = self.lookup(constraint.operator) if translator is None: - self.handle_missing_translator(constraint, context) - return None + self.handle_missing_translator(constraint) try: result = translator(constraint, column, context) - except UnsupportedConstraintError: + except (CapabilityError, UnsupportedConstraintError): raise except Exception as exc: # pragma: no cover - defensive wrapper raise TranslatorError( diff --git a/tests/test_backend_capabilities.py b/tests/test_backend_capabilities.py new file mode 100644 index 0000000..2df5b79 --- /dev/null +++ b/tests/test_backend_capabilities.py @@ -0,0 +1,179 @@ +from __future__ import annotations + +from datetime import date +from decimal import Decimal +from enum import Enum + +import pytest +from pydantic import Field +from sqlalchemy import Column, Date, Float, Integer, MetaData, Numeric, String, Table +from sqlalchemy.dialects.mssql import dialect as mssql_dialect +from sqlalchemy.dialects.postgresql import dialect as postgresql_dialect +from sqlalchemy.sql.sqltypes import NullType +from sqlrules_mssql import MssqlPlugin +from sqlrules_mysql import MysqlPlugin +from sqlrules_postgresql import PostgresPlugin +from sqlrules_sqlite import SQLitePlugin + +from sqlrules import CapabilityError, Compiler, RuleSchema + + +def test_sqlite_boolean_uses_runtime_integer_values_and_rejects_other_storage() -> None: + class BooleanRules(RuleSchema): + value: bool + + table = Table("items", MetaData(), Column("value", Integer)) + compiled = Compiler(plugins=[SQLitePlugin()]).compile(BooleanRules, table) + assert compiled.fields[0].logical_type == "bool" + assert compiled.fields[0].capability == "sqlite-runtime-storage-class" + + text_table = Table("text_items", MetaData(), Column("value", String)) + text_compiled = Compiler(plugins=[SQLitePlugin()]).compile(BooleanRules, text_table) + assert text_compiled.fields[0].capability == "sqlite-runtime-storage-class" + + +def test_unknown_and_emulated_sqlite_types_raise_stable_capability_errors() -> None: + class IntegerRules(RuleSchema): + value: int + + untyped = Table("untyped", MetaData(), Column("value", NullType())) + with pytest.raises(CapabilityError, match="source type evidence is unavailable"): + Compiler(plugins=[PostgresPlugin()]).compile(IntegerRules, untyped) + + class DateRules(RuleSchema): + value: date + + emulated = Table("emulated", MetaData(), Column("value", Date)) + with pytest.raises(CapabilityError, match="explicit storage adapter"): + Compiler(plugins=[SQLitePlugin()]).compile(DateRules, emulated) + + +def test_numeric_lax_conversions_and_backend_limits_are_explicit() -> None: + class FloatRules(RuleSchema): + value: float + + integer = Table("integers", MetaData(), Column("value", Integer)) + converted_float = Compiler(plugins=[PostgresPlugin()]).compile(FloatRules, integer) + assert converted_float.fields[0].coercion == "int-to-float" + + class DecimalRules(RuleSchema): + value: Decimal + + converted_decimal = Compiler(plugins=[PostgresPlugin()]).compile(DecimalRules, integer) + assert converted_decimal.fields[0].coercion == "int-to-decimal" + + class IntegerRules(RuleSchema): + value: int + + numeric = Table("numeric", MetaData(), Column("value", Numeric(12, 2))) + integral = Compiler(plugins=[PostgresPlugin()]).compile(IntegerRules, numeric) + assert integral.fields[0].coercion == "integral-numeric-to-int64" + + floating = Table("floating", MetaData(), Column("value", Float)) + float_to_integer = Compiler(plugins=[PostgresPlugin()]).compile(IntegerRules, floating) + postgres_sql = str(float_to_integer.predicate.compile(dialect=postgresql_dialect())) + assert " < " in postgres_sql + assert " <= " not in postgres_sql + + sql_server_conversion = Compiler(plugins=[MssqlPlugin(server_version=(16, 0))]).compile( + IntegerRules, floating + ) + sql_server_sql = str(sql_server_conversion.predicate.compile(dialect=mssql_dialect())) + assert " < " in sql_server_sql + assert " <= " not in sql_server_sql + assert "TRY_CAST" in sql_server_sql + + text = Table("text", MetaData(), Column("value", String)) + with pytest.raises(CapabilityError, match="precision and range"): + Compiler(plugins=[MysqlPlugin(server_version=(8, 0, 36))]).compile(FloatRules, text) + + +def test_postgresql_text_numeric_conversion_requires_v16_and_uses_safe_input_check() -> None: + class IntegerRules(RuleSchema): + value: int + + table = Table("text_values", MetaData(), Column("value", String)) + with pytest.raises(CapabilityError, match="PostgreSQL 16+"): + Compiler(plugins=[PostgresPlugin()]).compile(IntegerRules, table) + + compiled = Compiler(plugins=[PostgresPlugin(server_version=(16, 0))]).compile( + IntegerRules, table + ) + statement = compiled.predicate.compile(dialect=postgresql_dialect()) + assert compiled.fields[0].coercion == "text-to-int" + assert "pg_input_is_valid" in str(statement) + + +def test_sqlite_float_conversion_and_decimal_capability_boundary() -> None: + class FloatRules(RuleSchema): + value: float + + text = Table("text_values", MetaData(), Column("value", String)) + compiled = Compiler(plugins=[SQLitePlugin()]).compile(FloatRules, text) + assert compiled.fields[0].coercion == "sqlite-numeric-to-float" + assert "REGEXP" in str(compiled.predicate.compile()) + + class DecimalRules(RuleSchema): + value: Decimal + + numeric = Table("decimal_values", MetaData(), Column("value", Numeric)) + with pytest.raises(CapabilityError, match="cannot prove exact Decimal precision"): + Compiler(plugins=[SQLitePlugin()]).compile(DecimalRules, numeric) + + +def test_sql_server_numeric_to_float_avoids_huge_numeric_bounds() -> None: + class FloatRules(RuleSchema): + value: float = Field(ge=0) + + provider = MssqlPlugin(server_version=(16, 0)) + tables = ( + Table("integer_values", MetaData(), Column("value", Integer)), + Table("decimal_values", MetaData(), Column("value", Numeric(38, 0))), + ) + expected_coercions = ("int-to-float", "decimal-to-float") + for table, coercion in zip(tables, expected_coercions, strict=True): + compiled = Compiler(plugins=[provider]).compile(FloatRules, table) + statement = compiled.predicate.compile(dialect=mssql_dialect()) + assert compiled.fields[0].coercion == coercion + assert "CAST" in str(statement).upper() + assert not any( + isinstance(value, Decimal) and value.adjusted() > 37 + for value in statement.params.values() + ) + + +def test_float_multiple_of_fails_as_an_explicit_capability_error() -> None: + class Rules(RuleSchema): + value: float = Field(multiple_of=0.5) + + table = Table("floating_values", MetaData(), Column("value", Float)) + with pytest.raises(CapabilityError, match="floating-point fields or divisors"): + Compiler(plugins=[PostgresPlugin()]).compile(Rules, table) + + +def test_constraints_cannot_be_applied_to_a_known_mismatched_storage_type() -> None: + class Positive(RuleSchema): + value: int = Field(strict=True, gt=0) + + floating = Table("floating", MetaData(), Column("value", Float)) + compiled = Compiler(plugins=[PostgresPlugin()]).compile(Positive, floating) + assert compiled.fields[0].coercion == "strict-mismatch:float-to-int" + + class BooleanRules(RuleSchema): + value: bool + + text = Table("text", MetaData(), Column("value", String)) + with pytest.raises(CapabilityError, match="outside the frozen SQLRules 2.0 lax profile"): + Compiler(plugins=[PostgresPlugin()]).compile(BooleanRules, text) + + +def test_strict_string_enum_on_numeric_storage_is_a_known_mismatch() -> None: + class Status(str, Enum): + READY = "ready" + + class StatusRules(RuleSchema): + status: Status = Field(strict=True) + + table = Table("statuses", MetaData(), Column("status", Integer)) + compiled = Compiler(plugins=[PostgresPlugin()]).compile(StatusRules, table) + assert compiled.fields[0].coercion == "strict-mismatch:int-to-str" diff --git a/tests/test_compiler_v2.py b/tests/test_compiler_v2.py new file mode 100644 index 0000000..54cd64f --- /dev/null +++ b/tests/test_compiler_v2.py @@ -0,0 +1,196 @@ +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +from typing import Annotated, Literal + +import pytest +from pydantic import BaseModel, Field, StrictInt +from sqlalchemy import Column, Integer, MetaData, String, Table +from sqlrules_mysql import MysqlPlugin +from sqlrules_postgresql import PostgresPlugin +from sqlrules_sqlite import SQLitePlugin + +import sqlrules +from sqlrules import ( + CapabilityError, + Compiler, + InvalidModelError, + MissingColumnError, + PluginError, + RuleConfig, + RuleSchema, + TranslatorRegistry, + notwhere, + where, +) + + +def test_compiled_result_contains_one_total_root_and_explain_plan() -> None: + class Rules(RuleSchema): + age: Annotated[int, Field(ge=18)] + name: Annotated[str, Field(min_length=2)] + + table = Table( + "users", + MetaData(), + Column("age", Integer), + Column("name", String), + ) + compiled = Compiler(plugins=[SQLitePlugin()]).compile(Rules, table) + assert where(compiled) == [compiled.predicate] + assert sqlrules.flatten(compiled) == [compiled.predicate] + assert str(notwhere(compiled)[0]) == str(~compiled.predicate) + assert tuple(item.name for item in compiled.fields) == ("age", "name") + explanation = compiled.explain() + assert explanation["backend"] == "sqlite" + assert explanation["fields"][0]["logical_type"] == "int" + assert explanation["fields"][0]["coercion"] == "sqlite-int64" + assert "predicate" in explanation["fields"][0] + + +def test_compiler_requires_exactly_one_explicit_backend_provider() -> None: + class Rules(RuleSchema): + age: int + + with pytest.raises(PluginError, match="exactly one"): + Compiler().compile(Rules, Table("items", MetaData(), Column("age", Integer))) + with pytest.raises(PluginError, match="exactly one"): + Compiler(plugins=[SQLitePlugin(), PostgresPlugin()]) + + +def test_plain_pydantic_models_need_explicit_conversion() -> None: + class Request(BaseModel): + age: int + + compiler = Compiler(plugins=[SQLitePlugin()]) + with pytest.raises(InvalidModelError, match="from_pydantic"): + compiler.compile(Request, Table("items", MetaData(), Column("age", Integer))) # type: ignore[arg-type] + + +def test_column_mapping_and_missing_columns() -> None: + class Rules(RuleSchema): + source_name: str + + table = Table("users", MetaData(), Column("db_name", String)) + compiler = Compiler(plugins=[SQLitePlugin()]) + compiled = compiler.compile(Rules, table, column_map={"source_name": table.c.db_name}) + assert compiled.fields[0].column is table.c.db_name + + class Missing(RuleSchema): + absent: int + + with pytest.raises(MissingColumnError, match="absent"): + compiler.compile(Missing, table) + + +def test_sqlrules_field_column_binding_overrides_python_field_name() -> None: + class Rules(RuleSchema): + value: int = sqlrules.Field(column="stored_value") + + table = Table("items", MetaData(), Column("stored_value", Integer)) + compiled = Compiler(plugins=[SQLitePlugin()]).compile(Rules, table) + assert compiled.fields[0].column is table.c.stored_value + + +def test_backend_capability_errors_are_compile_time_errors() -> None: + class NumericText(RuleSchema): + value: int + + table = Table("items", MetaData(), Column("value", String)) + with pytest.raises(CapabilityError, match="PostgreSQL 16"): + Compiler(plugins=[PostgresPlugin()]).compile(NumericText, table) + + class FloatText(RuleSchema): + value: float + + with pytest.raises(CapabilityError, match="precision and range"): + Compiler(plugins=[MysqlPlugin(server_version=(8, 0, 36))]).compile(FloatText, table) + + +def test_strict_text_int_is_a_known_mismatch_and_sqlite_strict_bool_is_unsupported() -> None: + class StrictNumber(RuleSchema): + value: StrictInt + + text_table = Table("items", MetaData(), Column("value", String)) + compiled = Compiler(plugins=[SQLitePlugin()]).compile(StrictNumber, text_table) + assert compiled.fields[0].capability == "sqlite-runtime-storage-class" + assert compiled.fields[0].coercion == "runtime-type-match" + + class StrictBoolean(RuleSchema): + value: bool = Field(strict=True) + + bool_table = Table("items", MetaData(), Column("value", Integer)) + with pytest.raises(CapabilityError, match="no native boolean storage class"): + Compiler(plugins=[SQLitePlugin()]).compile(StrictBoolean, bool_table) + + +def test_empty_schema_compiles_to_true_only_when_explicitly_allowed() -> None: + class Empty(RuleSchema): + __rule_config__ = RuleConfig(allow_empty=True) + + compiled = Compiler(plugins=[SQLitePlugin()]).compile(Empty, Table("items", MetaData())) + assert "true" in str(compiled.predicate) + + +def test_conflict_policy_uses_a_private_registry_snapshot() -> None: + registry = TranslatorRegistry() + compiler = Compiler(plugins=[SQLitePlugin()], registry=registry) + assert "pattern" in compiler.registry + assert "pattern" not in registry + + +def test_compiler_rejects_removed_unsupported_policies() -> None: + with pytest.raises(sqlrules.ConfigurationError, match="on_unsupported"): + Compiler(on_unsupported="ignore") # type: ignore[arg-type] + + +def test_string_domains_require_deterministic_collation() -> None: + class Rules(RuleSchema): + status: Literal["ready", "pending"] + + table = Table("items", MetaData(), Column("status", String(collation="NOCASE"))) + with pytest.raises(CapabilityError, match="BINARY"): + Compiler(plugins=[SQLitePlugin()]).compile(Rules, table) + + binary = Table("items", MetaData(), Column("status", String(collation="BINARY"))) + compiled = Compiler(plugins=[SQLitePlugin()]).compile(Rules, binary) + assert compiled.fields[0].logical_type == "str" + + +def test_concurrent_compiles_keep_diagnostics_local_and_cache_shims_inert() -> None: + class Rules(RuleSchema): + age: Annotated[int, Field(ge=18)] + + class DiagnosticPlugin(SQLitePlugin): + def register(self, registry: TranslatorRegistry) -> None: + super().register(registry) + + def translate(constraint, value, context): + context.record( + severity="info", + field=constraint.field, + operator=constraint.operator, + value=constraint.value, + message="translated in this compile", + code="test_translation", + ) + return value >= constraint.value + + registry.register_constraint("ge", translate, on_conflict="replace") + + table = Table("users", MetaData(), Column("age", Integer)) + compiler = Compiler(plugins=[DiagnosticPlugin()], cache=True) + + def compile_once(_: int): + return compiler.compile(Rules, table) + + with ThreadPoolExecutor(max_workers=8) as pool: + results = list(pool.map(compile_once, range(32))) + + assert all(len(result.diagnostics) == 1 for result in results) + assert all(result.diagnostics[0].code == "test_translation" for result in results) + assert len({str(result.predicate) for result in results}) == 1 + + uncached = Compiler(plugins=[DiagnosticPlugin()], cache=False).compile(Rules, table) + sqlrules.clear_model_cache() + assert str(uncached.predicate) == str(results[0].predicate) diff --git a/tests/test_contract_semantics.py b/tests/test_contract_semantics.py deleted file mode 100644 index 1df7f33..0000000 --- a/tests/test_contract_semantics.py +++ /dev/null @@ -1,277 +0,0 @@ -"""SPEC-driven contract regressions: portable matrix, footguns, policy scope.""" - -from __future__ import annotations - -import re -from datetime import timedelta -from enum import Enum -from typing import Annotated, Any, Literal - -import pytest -from pydantic import BaseModel, Field, Strict -from sqlalchemy import Column, Integer, MetaData, String, Table - -import sqlrules -from assert_sql import assert_rules_sql, sql_literal -from sqlrules import ( - Compiler, - JsonContains, - MissingColumnError, - PatternSpec, - TypeSpec, - UnsupportedConstraintError, - clear_model_cache, - type_spec, -) -from sqlrules.cache import ModelIRCache, default_cache -from sqlrules.constraints import pattern_text -from sqlrules.ir import Constraint - - -class Status(Enum): - ACTIVE = "ACTIVE" - DISABLED = "DISABLED" - - -@pytest.fixture -def matrix_table() -> Table: - return Table( - "t", - MetaData(), - Column("age", Integer), - Column("qty", Integer), - Column("name", String), - Column("status", String), - Column("kind", String), - ) - - -def test_portable_constraint_matrix_exact_sql(matrix_table: Table) -> None: - class Filter(BaseModel): - age: Annotated[int, Field(ge=18, le=65, multiple_of=2)] - qty: Annotated[int, Field(gt=0, lt=100)] - name: Annotated[str, Field(min_length=2, max_length=40)] - status: Literal["ACTIVE", "DISABLED"] - kind: Status - - rules = sqlrules.compile(Filter, matrix_table, cache=False) - assert_rules_sql( - rules, - { - "age": [ - "t.age >= 18", - "t.age <= 65", - "t.age % 2 = 0", - ], - "qty": ["t.qty > 0", "t.qty < 100"], - "name": ["length(t.name) >= 2", "length(t.name) <= 40"], - "status": ["t.status IN ('ACTIVE', 'DISABLED')"], - "kind": ["t.kind IN ('ACTIVE', 'DISABLED')"], - }, - ) - - -def test_where_concatenation_order(matrix_table: Table) -> None: - class Filter(BaseModel): - qty: Annotated[int, Field(gt=0, lt=10)] - age: Annotated[int, Field(ge=18)] - name: Annotated[str, Field(min_length=1)] - - rules = sqlrules.compile(Filter, matrix_table, cache=False) - flat = sqlrules.where(rules) - assert [sql_literal(e) for e in flat] == [ - "t.qty > 0", - "t.qty < 10", - "t.age >= 18", - "length(t.name) >= 1", - ] - assert sqlrules.flatten(rules) == flat - - -def test_on_unsupported_softens_pattern_not_bad_types(matrix_table: Table) -> None: - class PatternOnly(BaseModel): - name: Annotated[str, Field(pattern=r"^A")] - - assert sqlrules.compile(PatternOnly, matrix_table, on_unsupported="ignore") == {} - - class TimedeltaModel(BaseModel): - delay: timedelta - - with pytest.raises(UnsupportedConstraintError, match="timedelta"): - sqlrules.compile(TimedeltaModel, matrix_table, on_unsupported="ignore") - - class BadMultiple(BaseModel): - age: Annotated[int, Field(multiple_of=0)] - - with pytest.raises(UnsupportedConstraintError, match="multiple_of"): - sqlrules.compile(BadMultiple, matrix_table, on_unsupported="ignore") - - class LengthOnInt(BaseModel): - age: Annotated[int, Field(min_length=1)] # type: ignore[type-var] - - with pytest.raises(UnsupportedConstraintError, match="min_length"): - sqlrules.compile(LengthOnInt, matrix_table, on_unsupported="ignore") - - -def test_on_unsupported_softens_marker_missing_translator(matrix_table: Table) -> None: - class Filter(BaseModel): - meta: Annotated[dict[str, Any], JsonContains({"a": 1})] - - table = Table("rows", MetaData(), Column("meta", String)) - assert sqlrules.compile(Filter, table, on_unsupported="ignore", cache=False) == {} - - -def test_strict_not_a_constraint_operator_feeds_typespec() -> None: - class Filter(BaseModel): - age: Annotated[int, Strict(), Field(ge=1)] - - without = Compiler(emit_type_checks=False, cache=False).compile_model(Filter) - assert [c.operator for c in without.fields[0].constraints] == ["ge"] - - with_checks = Compiler(emit_type_checks=True, cache=False).compile_model(Filter) - ops = [c.operator for c in with_checks.fields[0].constraints] - assert ops == ["ge", "type_check"] - assert type_spec(with_checks.fields[0].constraints[1].value) == TypeSpec( - python_type=int, - strict=True, - allow_none=False, - ) - - -def test_cache_key_includes_emit_type_checks_and_clear_scopes() -> None: - class Filter(BaseModel): - age: int - - custom = ModelIRCache() - default_cache().clear() - - Compiler(cache=True, emit_type_checks=False).compile_model(Filter) - Compiler(cache=True, emit_type_checks=True).compile_model(Filter) - assert default_cache().get(Filter, emit_type_checks=False) is not None - assert default_cache().get(Filter, emit_type_checks=True) is not None - assert default_cache().get(Filter, emit_type_checks=False) is not default_cache().get( - Filter, emit_type_checks=True - ) - - Compiler(cache=True, model_cache=custom, emit_type_checks=False).compile_model(Filter) - assert custom.get(Filter, emit_type_checks=False) is not None - - clear_model_cache() - assert default_cache().get(Filter, emit_type_checks=False) is None - assert default_cache().get(Filter, emit_type_checks=True) is None - # Custom caches are not cleared by clear_model_cache(). - assert custom.get(Filter, emit_type_checks=False) is not None - - -def test_dialect_hint_does_not_load_pattern_translator(matrix_table: Table) -> None: - class Filter(BaseModel): - name: Annotated[str, Field(pattern=r"^A")] - - with pytest.raises(UnsupportedConstraintError, match="pattern"): - Compiler(dialect="postgresql", cache=False).compile(Filter, matrix_table) - - -def test_pattern_flags_dotall_and_multiline_rejected() -> None: - class Dotall(BaseModel): - name: Annotated[str, Field(pattern=re.compile(r"^A", re.DOTALL))] - - class Multiline(BaseModel): - name: Annotated[str, Field(pattern=re.compile(r"^A", re.MULTILINE))] - - with pytest.raises(UnsupportedConstraintError, match="DOTALL|flag"): - Compiler(cache=False).compile_model(Dotall) - with pytest.raises(UnsupportedConstraintError, match="MULTILINE|flag"): - Compiler(cache=False).compile_model(Multiline) - - -def test_unanchored_pattern_still_extracts_pattern_spec() -> None: - """Dialects may use search/substring match; Pydantic validates fullmatch. - - SQLRules extracts the pattern text as-is. Dialects are not required to - emulate Pydantic fullmatch for unanchored patterns. - """ - - class Filter(BaseModel): - name: Annotated[str, Field(pattern=r"abc")] - - model_ir = Compiler(cache=False).compile_model(Filter) - assert model_ir.fields[0].constraints == ( - Constraint("name", "pattern", PatternSpec(pattern=r"abc")), - ) - assert pattern_text(model_ir.fields[0].constraints[0].value) == ("abc", False) - - -def test_pattern_text_helper_round_trip() -> None: - assert pattern_text(PatternSpec(pattern=r"x", ignore_case=True)) == ("x", True) - assert pattern_text("x") == ("x", False) - - -def test_constrained_field_missing_column_raises(matrix_table: Table) -> None: - class Filter(BaseModel): - missing: Annotated[int, Field(ge=1)] - - with pytest.raises(MissingColumnError, match="missing"): - sqlrules.compile(Filter, matrix_table, cache=False) - - -def test_pattern_text_rejects_invalid_value() -> None: - with pytest.raises(TypeError, match="PatternSpec"): - pattern_text(123) # type: ignore[arg-type] - - -def test_invalid_pattern_value_rejected_at_normalize() -> None: - from sqlrules.constraints import _normalize_pattern - - with pytest.raises(UnsupportedConstraintError, match="pattern"): - _normalize_pattern("name", 123) - - -def test_pattern_spec_passthrough_and_ignore_case() -> None: - from sqlrules.constraints import _normalize_pattern - - spec = PatternSpec(pattern=r"x", ignore_case=True) - assert _normalize_pattern("name", spec) is spec - - -def test_translator_must_return_column_element(matrix_table: Table) -> None: - from sqlrules.translators import default_registry - - class Filter(BaseModel): - age: Annotated[int, Field(ge=18)] - - registry = default_registry().copy() - registry.register_constraint( - "ge", - lambda c, col, ctx: "not-an-expression", # type: ignore[return-value,arg-type] - on_conflict="replace", - ) - with pytest.raises(sqlrules.TranslatorError, match="ColumnElement"): - Compiler(registry=registry, cache=False).compile(Filter, matrix_table) - - -def test_unsupported_constraint_error_str_without_suggestion() -> None: - err = UnsupportedConstraintError(field="age", operator="pattern", value=r"x") - assert "age" in str(err) - assert "pattern" in str(err) - - -def test_invalid_translator_error_str() -> None: - err = sqlrules.InvalidTranslatorError(operator="gt", translator=None) - assert "gt" in str(err) - assert "None" in str(err) - - -def test_enum_and_literal_constraint_mismatches_still_raise() -> None: - class Status(Enum): - A = "A" - - class EnumBad(BaseModel): - kind: Annotated[Status, Field(ge=1)] - - class LiteralBad(BaseModel): - status: Annotated[Literal["A", "B"], Field(min_length=1)] - - with pytest.raises(UnsupportedConstraintError, match="ge"): - Compiler(cache=False).compile_model(EnumBad) - with pytest.raises(UnsupportedConstraintError, match="min_length"): - Compiler(cache=False).compile_model(LiteralBad) diff --git a/tests/test_correctness.py b/tests/test_correctness.py deleted file mode 100644 index 8e9f6f8..0000000 --- a/tests/test_correctness.py +++ /dev/null @@ -1,373 +0,0 @@ -from datetime import time, timedelta -from enum import Enum -from typing import Annotated, Literal - -import pytest -from pydantic import BaseModel, Field -from sqlalchemy import Column, Float, Integer, MetaData, Numeric, String, Table, select -from sqlalchemy.sql.elements import ColumnElement - -import sqlrules -from assert_sql import assert_rules_sql, sql_literal -from sqlrules import MissingColumnError, UnsupportedConstraintError - - -def test_table_name_attribute_collision_raises() -> None: - """Field 'name' must not bind to Table.name (the table's string name).""" - table = Table("users", MetaData(), Column("age", Integer)) - - class Filter(BaseModel): - name: Annotated[str, Field(min_length=1)] - - with pytest.raises(MissingColumnError, match="name"): - sqlrules.compile(Filter, table) - - -def test_table_bool_attribute_collision_raises() -> None: - """Field matching a Table bool attribute must not emit Python bools.""" - table = Table("users", MetaData(), Column("id", Integer)) - - class Filter(BaseModel): - is_selectable: Annotated[int, Field(ge=1)] - - with pytest.raises(MissingColumnError, match="is_selectable"): - sqlrules.compile(Filter, table) - - -def test_optional_annotated_unwraps_metadata() -> None: - table = Table("users", MetaData(), Column("age", Integer)) - - class Filter(BaseModel): - age: Annotated[int, Field(ge=18)] | None = None - - rules = sqlrules.compile(Filter, table) - assert_rules_sql(rules, {"age": ["users.age >= 18"]}) - assert isinstance(rules["age"][0], ColumnElement) - - -def test_validation_alias_resolves_column() -> None: - table = Table("users", MetaData(), Column("age", Integer)) - - class Filter(BaseModel): - years: Annotated[int, Field(ge=18, validation_alias="age")] - - rules = sqlrules.compile(Filter, table) - assert_rules_sql(rules, {"years": ["users.age >= 18"]}) - compiled = str(select(table).where(*sqlrules.where(rules))) - assert "age" in compiled - assert ">= 18" in sql_literal(rules["years"][0]) - - -def test_serialization_alias_resolves_column() -> None: - table = Table("users", MetaData(), Column("age", Integer)) - - class Filter(BaseModel): - years: Annotated[int, Field(ge=18, serialization_alias="age")] - - rules = sqlrules.compile(Filter, table) - assert_rules_sql(rules, {"years": ["users.age >= 18"]}) - - -def test_alias_preferred_over_field_name_when_both_exist() -> None: - table = Table( - "users", - MetaData(), - Column("years", Integer), - Column("user_age", Integer), - ) - - class Filter(BaseModel): - years: Annotated[int, Field(ge=18, alias="user_age")] - - rules = sqlrules.compile(Filter, table) - assert_rules_sql(rules, {"years": ["users.user_age >= 18"]}) - compiled = str(select(table).where(*sqlrules.where(rules))) - assert "user_age" in compiled - assert "years" not in compiled.split("WHERE", 1)[1] - - -def test_unconstrained_orphan_field_skipped() -> None: - table = Table("users", MetaData(), Column("age", Integer)) - - class Filter(BaseModel): - orphan: str - age: Annotated[int, Field(ge=18)] - - rules = sqlrules.compile(Filter, table) - assert_rules_sql(rules, {"age": ["users.age >= 18"]}) - - -def test_multiple_of_zero_rejected() -> None: - table = Table("users", MetaData(), Column("age", Integer)) - - class Filter(BaseModel): - age: Annotated[int, Field(multiple_of=0)] - - with pytest.raises(UnsupportedConstraintError, match="multiple_of"): - sqlrules.compile(Filter, table) - - -def test_multiple_of_negative_rejected() -> None: - table = Table("users", MetaData(), Column("age", Integer)) - - class Filter(BaseModel): - age: Annotated[int, Field(multiple_of=-2)] - - with pytest.raises(UnsupportedConstraintError, match="multiple_of"): - sqlrules.compile(Filter, table) - - -def test_on_unsupported_ignore_still_raises_for_timedelta() -> None: - table = Table("users", MetaData(), Column("delta", String)) - - class Filter(BaseModel): - delta: timedelta - - with pytest.raises(UnsupportedConstraintError, match="timedelta"): - sqlrules.compile(Filter, table, on_unsupported="ignore") - - -def test_on_unsupported_ignore_still_raises_for_unsupported_containers() -> None: - table = Table("users", MetaData(), Column("age", Integer)) - - class Filter(BaseModel): - age: set[int] - - with pytest.raises(UnsupportedConstraintError, match="set"): - sqlrules.compile(Filter, table, on_unsupported="ignore") - - -def test_unconstrained_list_field_skipped() -> None: - table = Table("users", MetaData(), Column("tags", String)) - - class Filter(BaseModel): - tags: list[str] - - assert sqlrules.compile(Filter, table) == {} - - -def test_timedelta_unsupported_time_allowed() -> None: - table = Table( - "users", - MetaData(), - Column("start", String), - Column("delta", String), - ) - - class TimeFilter(BaseModel): - start: time - - class DeltaFilter(BaseModel): - delta: timedelta - - # Unconstrained time fields are skipped (type is supported in 0.2). - assert sqlrules.compile(TimeFilter, table) == {} - with pytest.raises(UnsupportedConstraintError, match="timedelta"): - sqlrules.compile(DeltaFilter, table) - - -def test_length_constraint_on_int_rejected() -> None: - table = Table("users", MetaData(), Column("age", Integer)) - - class Filter(BaseModel): - age: Annotated[int, Field(min_length=2)] - - with pytest.raises(UnsupportedConstraintError, match="min_length"): - sqlrules.compile(Filter, table) - - -def test_numeric_constraint_on_str_rejected() -> None: - table = Table("users", MetaData(), Column("name", String)) - - class Filter(BaseModel): - name: Annotated[str, Field(ge=5)] - - with pytest.raises(UnsupportedConstraintError, match="ge"): - sqlrules.compile(Filter, table) - - -def test_numeric_constraint_on_bool_rejected() -> None: - table = Table("users", MetaData(), Column("active", Integer)) - - class Filter(BaseModel): - active: Annotated[bool, Field(ge=True)] - - with pytest.raises(UnsupportedConstraintError, match="ge"): - sqlrules.compile(Filter, table) - - -def test_literal_with_numeric_constraint_rejected() -> None: - table = Table("users", MetaData(), Column("status", String)) - - class Filter(BaseModel): - status: Annotated[Literal["ACTIVE", "DISABLED"], Field(ge=1)] - - with pytest.raises(UnsupportedConstraintError, match="ge"): - sqlrules.compile(Filter, table) - - -def test_enum_with_length_constraint_rejected() -> None: - class Status(Enum): - ACTIVE = "ACTIVE" - DISABLED = "DISABLED" - - table = Table("users", MetaData(), Column("status", String)) - - class Filter(BaseModel): - status: Annotated[Status, Field(min_length=1)] - - with pytest.raises(UnsupportedConstraintError, match="min_length"): - sqlrules.compile(Filter, table) - - -def test_union_of_supported_types_rejected() -> None: - table = Table("users", MetaData(), Column("age", Integer)) - - class Filter(BaseModel): - age: int | str - - with pytest.raises(UnsupportedConstraintError, match="type"): - sqlrules.compile(Filter, table) - - -def test_alias_path_ignored_for_column_binding() -> None: - from pydantic import AliasPath - - table = Table("users", MetaData(), Column("age", Integer)) - - class Filter(BaseModel): - years: Annotated[int, Field(ge=18, validation_alias=AliasPath("nested", "age"))] - - with pytest.raises(MissingColumnError, match="years"): - sqlrules.compile(Filter, table) - - -def test_column_map_invalid_value_does_not_fall_through() -> None: - """A present but invalid column_map entry must not bind the table column.""" - table = Table("users", MetaData(), Column("age", Integer)) - - class Filter(BaseModel): - age: Annotated[int, Field(ge=18)] - - with pytest.raises(MissingColumnError): - sqlrules.compile(Filter, table, column_map={"age": "not-a-column"}) # type: ignore[dict-item] - - -def test_resolve_column_alias_kwarg_without_aliases_list() -> None: - from sqlrules.columns import resolve_column - - table = Table("users", MetaData(), Column("user_age", Integer)) - column = resolve_column("years", table, alias="user_age") - assert column is table.c.user_age - - -def test_resolve_column_duplicate_alias_and_field_name() -> None: - from sqlrules.columns import resolve_column - - table = Table("users", MetaData(), Column("age", Integer)) - column = resolve_column("age", table, aliases=("age", "age")) - assert column is table.c.age - - -def test_multiple_of_non_comparable_value() -> None: - from sqlrules.ir import CompilationContext, Constraint - from sqlrules.translators import default_registry - - registry = default_registry() - with pytest.raises(UnsupportedConstraintError, match="multiple_of"): - registry.translate( - Constraint("age", "multiple_of", object()), - Column("age", Integer), - CompilationContext(), - ) - - -def test_multiple_of_nan_and_inf_rejected() -> None: - table = Table("users", MetaData(), Column("score", Float)) - - class NanFilter(BaseModel): - score: Annotated[float, Field(multiple_of=float("nan"))] - - class InfFilter(BaseModel): - score: Annotated[float, Field(multiple_of=float("inf"))] - - with pytest.raises(UnsupportedConstraintError, match="multiple_of"): - sqlrules.compile(NanFilter, table) - with pytest.raises(UnsupportedConstraintError, match="multiple_of"): - sqlrules.compile(InfFilter, table) - - -def test_multiple_of_decimal_nan_rejected() -> None: - from decimal import Decimal - - table = Table("users", MetaData(), Column("score", Numeric)) - - class Filter(BaseModel): - score: Annotated[Decimal, Field(multiple_of=Decimal("nan"))] - - with pytest.raises(UnsupportedConstraintError, match="multiple_of"): - sqlrules.compile(Filter, table) - - -def test_flag_enum_rejected() -> None: - from enum import Flag - - table = Table("users", MetaData(), Column("perms", Integer)) - - class Perm(Flag): - R = 1 - W = 2 - - class Filter(BaseModel): - perms: Perm - - with pytest.raises(UnsupportedConstraintError, match="Flag"): - sqlrules.compile(Filter, table) - - -def test_verbose_pattern_rejected() -> None: - import re - - table = Table("users", MetaData(), Column("name", String)) - - class Filter(BaseModel): - name: Annotated[str, Field(pattern=re.compile(r"^ a $", re.VERBOSE))] - - with pytest.raises(UnsupportedConstraintError, match="VERBOSE"): - sqlrules.compile(Filter, table) - - -def test_pattern_only_missing_column_ignored() -> None: - table = Table("users", MetaData(), Column("other", Integer)) - - class Filter(BaseModel): - name: Annotated[str, Field(pattern=r"^A")] - - assert sqlrules.compile(Filter, table, on_unsupported="ignore", cache=False) == {} - - -def test_empty_enum_rejected() -> None: - from enum import Enum - - table = Table("users", MetaData(), Column("status", String)) - - class Empty(Enum): - pass - - class Filter(BaseModel): - status: Empty - - with pytest.raises(UnsupportedConstraintError, match="Enum"): - sqlrules.compile(Filter, table) - - -def test_predicate_unsupported() -> None: - from annotated_types import Predicate - - table = Table("users", MetaData(), Column("age", Integer)) - - class Filter(BaseModel): - age: Annotated[int, Predicate(lambda value: value > 0)] - - with pytest.raises(UnsupportedConstraintError, match="predicate"): - sqlrules.compile(Filter, table) diff --git a/tests/test_coverage_edges.py b/tests/test_coverage_edges.py deleted file mode 100644 index 02dbbfd..0000000 --- a/tests/test_coverage_edges.py +++ /dev/null @@ -1,118 +0,0 @@ -import operator -from enum import Enum -from typing import Annotated - -import pytest -from annotated_types import Len -from pydantic import BaseModel, Field -from sqlalchemy import Column, Integer, MetaData, String, Table -from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column - -import sqlrules -from assert_sql import assert_rules_sql, sql_literal -from sqlrules.translators import TranslatorRegistry, _binary - - -def test_column_map_override() -> None: - class Filter(BaseModel): - user_age: Annotated[int, Field(ge=18)] - - table = Table("users", MetaData(), Column("age", Integer)) - rules = sqlrules.compile( - Filter, - table, - column_map={"user_age": table.c.age}, - ) - assert_rules_sql(rules, {"user_age": ["users.age >= 18"]}) - - -def test_orm_attribute_resolution() -> None: - class Base(DeclarativeBase): - pass - - class User(Base): - __tablename__ = "users" - id: Mapped[int] = mapped_column(Integer, primary_key=True) - age: Mapped[int] = mapped_column(Integer) - - class Filter(BaseModel): - age: Annotated[int, Field(ge=21)] - - rules = sqlrules.compile(Filter, User) - assert sql_literal(rules["age"][0]) == "users.age >= 21" - - -def test_len_constraint() -> None: - class Filter(BaseModel): - name: Annotated[str, Len(2, 10)] - - table = Table("users", MetaData(), Column("name", String)) - rules = sqlrules.compile(Filter, table) - assert_rules_sql( - rules, - {"name": ["length(users.name) >= 2", "length(users.name) <= 10"]}, - ) - - -def test_duplicate_registry_registration() -> None: - registry = TranslatorRegistry() - registry.register("gt", _binary(operator.gt)) - with pytest.raises(sqlrules.RegistryError): - registry.register("gt", _binary(operator.gt)) - - -def test_exception_messages_from_compile() -> None: - table = Table("users", MetaData(), Column("id", Integer)) - - class Missing(BaseModel): - age: Annotated[int, Field(ge=18)] - - with pytest.raises(sqlrules.MissingColumnError) as missing: - sqlrules.compile(Missing, table) - assert "age" in str(missing.value) - - class Pattern(BaseModel): - name: Annotated[str, Field(pattern=r"^A")] - - name_table = Table("users", MetaData(), Column("name", String)) - with pytest.raises(sqlrules.UnsupportedConstraintError) as unsupported: - sqlrules.compile(Pattern, name_table) - assert "pattern" in str(unsupported.value) - assert unsupported.value.suggestion is not None - assert "on_unsupported" in unsupported.value.suggestion - - with pytest.raises(sqlrules.InvalidModelError) as invalid: - sqlrules.compile(object, table) # type: ignore[arg-type] - assert "BaseModel" in str(invalid.value) - - -def test_translator_error_wraps_unexpected_failure() -> None: - from sqlrules.translators import default_registry - - class Filter(BaseModel): - age: Annotated[int, Field(ge=18)] - - table = Table("users", MetaData(), Column("age", Integer)) - registry = default_registry().copy() - - def boom(constraint, column, context): # type: ignore[no-untyped-def] - raise RuntimeError("boom") - - registry.register_constraint("ge", boom, on_conflict="replace") - with pytest.raises(sqlrules.TranslatorError, match="boom") as err: - sqlrules.Compiler(registry=registry, cache=False).compile(Filter, table) - assert err.value.field == "age" - assert err.value.operator == "ge" - - -def test_enum_values() -> None: - class Color(Enum): - RED = "red" - BLUE = "blue" - - class Filter(BaseModel): - color: Color - - table = Table("items", MetaData(), Column("color", String)) - rules = sqlrules.compile(Filter, table) - assert_rules_sql(rules, {"color": ["items.color IN ('red', 'blue')"]}) diff --git a/tests/test_declaration_edges.py b/tests/test_declaration_edges.py new file mode 100644 index 0000000..0db43e3 --- /dev/null +++ b/tests/test_declaration_edges.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +import re +from enum import Flag, IntFlag +from typing import Annotated, Literal + +import pytest +from pydantic import ConfigDict, Field, StringConstraints + +from sqlrules import Compiler, InvalidModelError, RuleSchema, from_pydantic + + +def test_pattern_flags_and_string_transform_metadata_are_checked() -> None: + class PatternRules(RuleSchema): + name: Annotated[str, Field(pattern=re.compile(r"^ada", re.IGNORECASE))] + + constraint = Compiler().compile_model(PatternRules).fields[0].constraints[0] + assert constraint.operator == "pattern" + assert constraint.value.pattern == "^ada" + assert constraint.value.ignore_case + + with pytest.raises(TypeError, match="VERBOSE"): + + class VerbosePatternRules(RuleSchema): + name: Annotated[str, Field(pattern=re.compile("^ada", re.VERBOSE))] + + with pytest.raises(TypeError, match="metadata"): + + class NormalizedStrings(RuleSchema): + name: Annotated[str, StringConstraints(strip_whitespace=True)] + + +def test_incompatible_model_config_and_constraint_combinations_are_rejected() -> None: + with pytest.raises(TypeError, match="model_config"): + + class NormalizingModel(RuleSchema): + model_config = ConfigDict(str_to_lower=True) + name: str + + with pytest.raises(TypeError, match="multiple_of"): + + class InvalidDivisor(RuleSchema): + count: Annotated[int, Field(multiple_of=0)] + + with pytest.raises(TypeError, match="bounds"): + + class EmptyInterval(RuleSchema): + count: Annotated[int, Field(gt=5, le=5)] + + with pytest.raises(TypeError, match="length"): + + class InvalidLength(RuleSchema): + name: Annotated[str, Field(min_length=5, max_length=2)] + + +def test_literal_and_enum_domains_require_one_supported_scalar_type() -> None: + with pytest.raises(TypeError, match="Literal"): + + class MixedLiteral(RuleSchema): + value: Literal[1, "one"] + + class CombinedFlags(Flag): + READ = 1 + WRITE = 2 + + class CombinedIntFlags(IntFlag): + READ = 1 + WRITE = 2 + + with pytest.raises(TypeError, match="enum"): + + class FlagRules(RuleSchema): + value: CombinedFlags + + with pytest.raises(TypeError, match="enum"): + + class IntFlagRules(RuleSchema): + value: CombinedIntFlags + + +def test_conversion_of_existing_rule_schema_and_argument_errors() -> None: + class ExistingRules(RuleSchema): + value: int + + converted = from_pydantic(ExistingRules) + assert converted.model is ExistingRules + assert not converted.report.has_incompatibilities + + with pytest.raises(InvalidModelError, match="SQLRules RuleSchema"): + from_pydantic(dict) # type: ignore[arg-type] + with pytest.raises(ValueError, match="on_incompatible"): + from_pydantic(ExistingRules, on_incompatible="invalid") # type: ignore[arg-type] diff --git a/tests/test_errors.py b/tests/test_errors.py deleted file mode 100644 index e116bcd..0000000 --- a/tests/test_errors.py +++ /dev/null @@ -1,139 +0,0 @@ -from decimal import Decimal -from enum import Enum -from typing import Annotated, Literal - -import pytest -from pydantic import BaseModel, Field -from sqlalchemy import Column, Integer, MetaData, String, Table, column - -import sqlrules -from assert_sql import assert_rules_sql -from sqlrules import MissingColumnError, SQLRulesWarning, UnsupportedConstraintError -from sqlrules.ir import CompilationContext, Constraint -from sqlrules.translators import TranslatorRegistry - - -class UserFilter(BaseModel): - age: Annotated[int, Field(ge=18)] - - -def test_missing_column_error() -> None: - table = Table("users", MetaData(), Column("id", Integer)) - - with pytest.raises(MissingColumnError, match="age"): - sqlrules.compile(UserFilter, table) - - -def test_invalid_model_error(users: Table) -> None: - with pytest.raises(sqlrules.InvalidModelError, match="BaseModel"): - sqlrules.compile(object, users) # type: ignore[arg-type] - - -def test_invalid_on_unsupported() -> None: - with pytest.raises(sqlrules.ConfigurationError, match="on_unsupported"): - sqlrules.Compiler(on_unsupported="explode") # type: ignore[arg-type] - - -def test_unsupported_constraint_raise() -> None: - registry = TranslatorRegistry() - with pytest.raises(UnsupportedConstraintError, match="pattern"): - registry.translate( - Constraint("name", "pattern", r"^A"), - column("name"), - CompilationContext(on_unsupported="raise"), - ) - - -def test_unsupported_constraint_warn() -> None: - registry = TranslatorRegistry() - with pytest.warns(SQLRulesWarning): - result = registry.translate( - Constraint("name", "pattern", r"^A"), - column("name"), - CompilationContext(on_unsupported="warn"), - ) - assert result is None - - -def test_unsupported_constraint_ignore() -> None: - registry = TranslatorRegistry() - result = registry.translate( - Constraint("name", "pattern", r"^A"), - column("name"), - CompilationContext(on_unsupported="ignore"), - ) - assert result is None - - -def test_optional_literal_and_enum() -> None: - class Status(Enum): - ACTIVE = "ACTIVE" - DISABLED = "DISABLED" - - class OptionalFilter(BaseModel): - status: Literal["ACTIVE", "DISABLED"] | None = None - kind: Status | None = None - - table = Table( - "items", - MetaData(), - Column("status", String), - Column("kind", String), - ) - rules = sqlrules.compile(OptionalFilter, table) - assert_rules_sql( - rules, - { - "status": ["items.status IN ('ACTIVE', 'DISABLED')"], - "kind": ["items.kind IN ('ACTIVE', 'DISABLED')"], - }, - ) - - -def test_version_exposed() -> None: - assert sqlrules.__version__ == "1.0.1" - assert callable(sqlrules.clear_model_cache) - - -def test_clear_model_cache_drops_default_entries() -> None: - from sqlrules.cache import default_cache - - class Filter(BaseModel): - age: Annotated[int, Field(ge=18)] - - default_cache().clear() - sqlrules.Compiler(cache=True).compile_model(Filter) - assert default_cache().get(Filter) is not None - sqlrules.clear_model_cache() - assert default_cache().get(Filter) is None - - -def test_default_registry_returns_independent_copies() -> None: - a = sqlrules.default_registry() - b = sqlrules.default_registry() - assert a is not b - assert set(a.operators()) == set(b.operators()) - - -def test_unknown_field_metadata_key_rejected() -> None: - from dataclasses import dataclass - - @dataclass(frozen=True) - class WeirdMeta: - not_a_sqlrules_op: int = 1 - - class Filter(BaseModel): - age: Annotated[int, WeirdMeta()] - - table = Table("t", MetaData(), Column("age", Integer)) - with pytest.raises(UnsupportedConstraintError, match="not_a_sqlrules_op|Unknown metadata"): - sqlrules.compile(Filter, table, cache=False) - - -def test_max_digits_only_rejected_at_extract() -> None: - class Filter(BaseModel): - score: Annotated[Decimal, Field(max_digits=5)] - - table = Table("t", MetaData(), Column("score", Integer)) - with pytest.raises(UnsupportedConstraintError, match="max_digits"): - sqlrules.compile(Filter, table, cache=False) diff --git a/tests/test_helpers.py b/tests/test_helpers.py deleted file mode 100644 index 7117220..0000000 --- a/tests/test_helpers.py +++ /dev/null @@ -1,23 +0,0 @@ -from typing import Annotated - -from pydantic import BaseModel, Field -from sqlalchemy import Table - -import sqlrules -from assert_sql import assert_expr_equals, assert_rules_sql, sql_literal - - -class UserFilter(BaseModel): - age: Annotated[int, Field(ge=18, le=65)] - - -def test_where_flattens_rules(users: Table) -> None: - """where()/flatten() concatenate field expressions in declaration order.""" - rules = sqlrules.compile(UserFilter, users) - - assert sqlrules.where is sqlrules.flatten or sqlrules.where(rules) == sqlrules.flatten(rules) - flat = sqlrules.where(rules) - assert [sql_literal(e) for e in flat] == ["users.age >= 18", "users.age <= 65"] - assert_expr_equals(flat[0], users.c.age >= 18) - assert_expr_equals(flat[1], users.c.age <= 65) - assert_rules_sql(rules, {"age": ["users.age >= 18", "users.age <= 65"]}) diff --git a/tests/test_literal_enum.py b/tests/test_literal_enum.py deleted file mode 100644 index d2ee2f7..0000000 --- a/tests/test_literal_enum.py +++ /dev/null @@ -1,39 +0,0 @@ -from enum import Enum -from typing import Literal - -from pydantic import BaseModel -from sqlalchemy import Table - -import sqlrules -from assert_sql import assert_expr_equals, assert_rules_sql - - -class Status(Enum): - ACTIVE = "ACTIVE" - DISABLED = "DISABLED" - - -class LiteralFilter(BaseModel): - status: Literal["ACTIVE", "DISABLED"] - - -class EnumFilter(BaseModel): - status: Status - - -def test_literal_compile(users: Table) -> None: - rules = sqlrules.compile(LiteralFilter, users) - assert_rules_sql( - rules, - {"status": ["users.status IN ('ACTIVE', 'DISABLED')"]}, - ) - assert_expr_equals(rules["status"][0], users.c.status.in_(("ACTIVE", "DISABLED"))) - - -def test_enum_compile(users: Table) -> None: - rules = sqlrules.compile(EnumFilter, users) - assert_rules_sql( - rules, - {"status": ["users.status IN ('ACTIVE', 'DISABLED')"]}, - ) - assert_expr_equals(rules["status"][0], users.c.status.in_(("ACTIVE", "DISABLED"))) diff --git a/tests/test_live_backend_conformance.py b/tests/test_live_backend_conformance.py new file mode 100644 index 0000000..9b9f208 --- /dev/null +++ b/tests/test_live_backend_conformance.py @@ -0,0 +1,557 @@ +from __future__ import annotations + +import os +from datetime import date, datetime, time +from decimal import Decimal +from typing import Annotated, Any, Literal +from uuid import UUID, uuid4 + +import pytest +from pydantic import Field +from sqlalchemy import ( + BigInteger, + Boolean, + Column, + Date, + DateTime, + Float, + Index, + Integer, + MetaData, + Numeric, + String, + Table, + Time, + create_engine, + insert, + select, +) +from sqlalchemy.dialects.mssql import UNIQUEIDENTIFIER +from sqlalchemy.dialects.postgresql import ARRAY, INT4RANGE, JSONB +from sqlalchemy.dialects.postgresql import UUID as PostgreSQLUUID +from sqlrules_mssql import MssqlPlugin +from sqlrules_mysql import MysqlPlugin +from sqlrules_postgresql import PostgresPlugin + +from sqlrules import ( + ArrayContains, + Compiler, + FullTextMatch, + JsonContains, + JsonHasKey, + RangeContains, + RuleSchema, + notwhere, + where, +) + + +class IntegerPayloadRules(RuleSchema): + payload: int | None = Field(ge=0) + + +class StrictIntegerPayloadRules(RuleSchema): + payload: int | None = Field(strict=True) + + +class PositiveFloatPayloadRules(RuleSchema): + payload: float = Field(gt=0) + + +class NativeScalarRules(RuleSchema): + flag: bool + count: Annotated[int, Field(ge=0, multiple_of=2)] + ratio: Annotated[float, Field(gt=0)] + amount: Decimal + label: Annotated[str, Field(min_length=2, max_length=12)] + state: Literal["ready", "pending"] + day: date + created: datetime + clock: time + optional_count: int | None + required_note: str + + +class UUIDRules(RuleSchema): + token: UUID + + +@pytest.mark.parametrize( + ("backend", "provider_type", "server_version"), + [ + ("postgresql", PostgresPlugin, (16, 0)), + ("mysql", MysqlPlugin, (8, 0, 36)), + ("mssql", MssqlPlugin, (16, 0)), + ], +) +def test_live_backend_match_and_fail_sets_partition( + backend: str, + provider_type: Any, + server_version: tuple[int, ...], +) -> None: + url = os.getenv(f"SQLRULES_TEST_{backend.upper()}_URL") + if not url: + pytest.skip(f"SQLRULES_TEST_{backend.upper()}_URL is not configured") + + engine = create_engine(url) + table = Table( + f"sr_rules_{backend}_{uuid4().hex[:10]}", + MetaData(), + Column("id", Integer, primary_key=True), + Column("payload", String(128)), + ) + try: + table.create(engine) + with engine.begin() as connection: + connection.execute( + insert(table), + [ + {"id": 1, "payload": "12"}, + {"id": 2, "payload": "malformed"}, + {"id": 3, "payload": None}, + {"id": 4, "payload": "-1"}, + {"id": 5, "payload": "9223372036854775808"}, + ], + ) + provider = provider_type(server_version=server_version) + compiled = Compiler(plugins=[provider]).compile(IntegerPayloadRules, table) + matched = set(connection.execute(select(table.c.id).where(*where(compiled))).scalars()) + failed = set( + connection.execute(select(table.c.id).where(*notwhere(compiled))).scalars() + ) + assert matched == {1, 3} + assert failed == {2, 4, 5} + assert matched.isdisjoint(failed) + assert matched | failed == {1, 2, 3, 4, 5} + finally: + table.drop(engine, checkfirst=True) + engine.dispose() + + +@pytest.mark.parametrize( + ("backend", "provider_type", "server_version"), + [ + ("postgresql", PostgresPlugin, (16, 0)), + ("mysql", MysqlPlugin, (8, 0, 36)), + ("mssql", MssqlPlugin, (16, 0)), + ], +) +def test_live_strict_text_mismatch_is_complemented_by_notwhere( + backend: str, + provider_type: Any, + server_version: tuple[int, ...], +) -> None: + url = os.getenv(f"SQLRULES_TEST_{backend.upper()}_URL") + if not url: + pytest.skip(f"SQLRULES_TEST_{backend.upper()}_URL is not configured") + + engine = create_engine(url) + table = Table( + f"sr_strict_{backend}_{uuid4().hex[:10]}", + MetaData(), + Column("id", Integer, primary_key=True), + Column("payload", String(128)), + ) + try: + table.create(engine) + with engine.begin() as connection: + connection.execute( + insert(table), + [{"id": 1, "payload": "12"}, {"id": 2, "payload": None}], + ) + compiled = Compiler(plugins=[provider_type(server_version=server_version)]).compile( + StrictIntegerPayloadRules, table + ) + matched = set(connection.execute(select(table.c.id).where(*where(compiled))).scalars()) + failed = set( + connection.execute(select(table.c.id).where(*notwhere(compiled))).scalars() + ) + assert matched == {2} + assert failed == {1} + assert matched | failed == {1, 2} + finally: + table.drop(engine, checkfirst=True) + engine.dispose() + + +@pytest.mark.parametrize( + ("backend", "provider_type", "server_version"), + [ + ("postgresql", PostgresPlugin, (16, 0)), + ("mssql", MssqlPlugin, (16, 0)), + ], +) +def test_live_float_text_conversion_rejects_malformed_and_nonfinite_text( + backend: str, + provider_type: Any, + server_version: tuple[int, ...], +) -> None: + url = os.getenv(f"SQLRULES_TEST_{backend.upper()}_URL") + if not url: + pytest.skip(f"SQLRULES_TEST_{backend.upper()}_URL is not configured") + + engine = create_engine(url) + table = Table( + f"sr_float_{backend}_{uuid4().hex[:10]}", + MetaData(), + Column("id", Integer, primary_key=True), + Column("payload", String(128)), + ) + try: + table.create(engine) + with engine.begin() as connection: + connection.execute( + insert(table), + [ + {"id": 1, "payload": "2.5"}, + {"id": 2, "payload": "malformed"}, + {"id": 3, "payload": "Infinity"}, + {"id": 4, "payload": "-1"}, + {"id": 5, "payload": "1e2"}, + ], + ) + compiled = Compiler(plugins=[provider_type(server_version=server_version)]).compile( + PositiveFloatPayloadRules, table + ) + matched = set(connection.execute(select(table.c.id).where(*where(compiled))).scalars()) + failed = set( + connection.execute(select(table.c.id).where(*notwhere(compiled))).scalars() + ) + assert matched == {1, 5} + assert failed == {2, 3, 4} + assert matched.isdisjoint(failed) + assert matched | failed == {1, 2, 3, 4, 5} + finally: + table.drop(engine, checkfirst=True) + engine.dispose() + + +@pytest.mark.parametrize( + ("backend", "provider_type", "server_version", "collation"), + [ + ("postgresql", PostgresPlugin, (16, 0), "C"), + ("mysql", MysqlPlugin, (8, 0, 36), "utf8mb4_bin"), + ("mssql", MssqlPlugin, (16, 0), "Latin1_General_100_BIN2"), + ], +) +def test_live_native_scalar_constraints_and_null_partition( + backend: str, + provider_type: Any, + server_version: tuple[int, ...], + collation: str, +) -> None: + url = os.getenv(f"SQLRULES_TEST_{backend.upper()}_URL") + if not url: + pytest.skip(f"SQLRULES_TEST_{backend.upper()}_URL is not configured") + + engine = create_engine(url) + table = Table( + f"sr_native_{backend}_{uuid4().hex[:10]}", + MetaData(), + Column("id", Integer, primary_key=True), + Column("flag", Boolean), + Column("count", BigInteger), + Column("ratio", Float), + Column("amount", Numeric(12, 2)), + Column("label", String(32, collation=collation)), + Column("state", String(16, collation=collation)), + Column("day", Date), + Column("created", DateTime), + Column("clock", Time), + Column("optional_count", Integer, nullable=True), + Column("required_note", String(32), nullable=True), + ) + valid = { + "flag": True, + "count": 2, + "ratio": 2.5, + "amount": Decimal("10.25"), + "label": "Ada", + "state": "ready", + "day": date(2020, 1, 2), + "created": datetime(2020, 1, 2, 12, 30), + "clock": time(12, 30), + "optional_count": 4, + "required_note": "present", + } + rows = [ + {"id": 1, **valid}, + {"id": 2, **(valid | {"count": 3})}, + {"id": 3, **(valid | {"label": "A"})}, + {"id": 4, **(valid | {"state": "unknown"})}, + {"id": 5, **(valid | {"optional_count": None, "required_note": None})}, + ] + try: + table.create(engine) + with engine.begin() as connection: + connection.execute(insert(table), rows) + compiled = Compiler(plugins=[provider_type(server_version=server_version)]).compile( + NativeScalarRules, table + ) + matched = set(connection.execute(select(table.c.id).where(*where(compiled))).scalars()) + failed = set( + connection.execute(select(table.c.id).where(*notwhere(compiled))).scalars() + ) + assert matched == {1} + assert failed == {2, 3, 4, 5} + assert matched.isdisjoint(failed) + assert matched | failed == {1, 2, 3, 4, 5} + finally: + table.drop(engine, checkfirst=True) + engine.dispose() + + +@pytest.mark.parametrize( + ("backend", "provider_type", "server_version", "source_type"), + [ + ("postgresql", PostgresPlugin, (16, 0), PostgreSQLUUID(as_uuid=True)), + ("mssql", MssqlPlugin, (16, 0), UNIQUEIDENTIFIER(as_uuid=True)), + ], +) +def test_live_uuid_type_only_rule( + backend: str, + provider_type: Any, + server_version: tuple[int, ...], + source_type: Any, +) -> None: + url = os.getenv(f"SQLRULES_TEST_{backend.upper()}_URL") + if not url: + pytest.skip(f"SQLRULES_TEST_{backend.upper()}_URL is not configured") + + engine = create_engine(url) + table = Table( + f"sr_uuid_{backend}_{uuid4().hex[:10]}", + MetaData(), + Column("id", Integer, primary_key=True), + Column("token", source_type, nullable=True), + ) + token = uuid4() + try: + table.create(engine) + with engine.begin() as connection: + connection.execute(insert(table), [{"id": 1, "token": token}, {"id": 2, "token": None}]) + compiled = Compiler(plugins=[provider_type(server_version=server_version)]).compile( + UUIDRules, table + ) + matched = set(connection.execute(select(table.c.id).where(*where(compiled))).scalars()) + failed = set( + connection.execute(select(table.c.id).where(*notwhere(compiled))).scalars() + ) + assert matched == {1} + assert failed == {2} + assert matched | failed == {1, 2} + finally: + table.drop(engine, checkfirst=True) + engine.dispose() + + +@pytest.mark.parametrize( + ("backend", "provider_type", "server_version", "json_type"), + [ + ("postgresql", PostgresPlugin, (16, 0), JSONB), + ("mysql", MysqlPlugin, (8, 0, 36), None), + ("mssql", MssqlPlugin, (16, 0, 160), String(1024)), + ], +) +def test_live_json_markers_handle_values_null_and_malformed_documents( + backend: str, + provider_type: Any, + server_version: tuple[int, ...], + json_type: Any, +) -> None: + from sqlalchemy import JSON + + class JsonRules(RuleSchema): + meta: Annotated[ + dict[str, Any], + JsonContains({"active": True}), + JsonHasKey("active"), + ] + + url = os.getenv(f"SQLRULES_TEST_{backend.upper()}_URL") + if not url: + pytest.skip(f"SQLRULES_TEST_{backend.upper()}_URL is not configured") + + engine = create_engine(url) + selected_json_type = json_type or JSON() + table = Table( + f"sr_json_{backend}_{uuid4().hex[:10]}", + MetaData(), + Column("id", Integer, primary_key=True), + Column("meta", selected_json_type, nullable=True), + ) + if backend == "mssql": + rows = [ + {"id": 1, "meta": '{"active":true}'}, + {"id": 2, "meta": '{"active":false}'}, + {"id": 3, "meta": "{}"}, + {"id": 4, "meta": None}, + {"id": 5, "meta": "malformed"}, + ] + expected = {1, 2, 3, 4, 5} + else: + rows = [ + {"id": 1, "meta": {"active": True}}, + {"id": 2, "meta": {"active": False}}, + {"id": 3, "meta": {}}, + {"id": 4, "meta": None}, + ] + expected = {1, 2, 3, 4} + try: + table.create(engine) + with engine.begin() as connection: + connection.execute(insert(table), rows) + if backend == "mssql": + provider = provider_type( + server_version=server_version[:2], compatibility_level=server_version[2] + ) + else: + provider = provider_type(server_version=server_version) + compiled = Compiler(plugins=[provider]).compile(JsonRules, table) + matched = set(connection.execute(select(table.c.id).where(*where(compiled))).scalars()) + failed = set( + connection.execute(select(table.c.id).where(*notwhere(compiled))).scalars() + ) + assert matched == {1} + assert failed == expected - {1} + assert matched.isdisjoint(failed) + assert matched | failed == expected + finally: + table.drop(engine, checkfirst=True) + engine.dispose() + + +def test_live_postgresql_array_and_range_markers() -> None: + url = os.getenv("SQLRULES_TEST_POSTGRESQL_URL") + if not url: + pytest.skip("SQLRULES_TEST_POSTGRESQL_URL is not configured") + + from psycopg.types.range import Range + + class StructuredRules(RuleSchema): + tags: Annotated[list[str], ArrayContains(["admin"])] + span: Annotated[int, RangeContains(5)] + + engine = create_engine(url) + table = Table( + f"sr_structured_{uuid4().hex[:10]}", + MetaData(), + Column("id", Integer, primary_key=True), + Column("tags", ARRAY(String), nullable=True), + Column("span", INT4RANGE, nullable=True), + ) + rows = [ + {"id": 1, "tags": ["admin", "staff"], "span": Range(1, 10)}, + {"id": 2, "tags": ["admin"], "span": Range(10, 20)}, + {"id": 3, "tags": ["viewer"], "span": Range(1, 10)}, + {"id": 4, "tags": None, "span": None}, + ] + try: + table.create(engine) + with engine.begin() as connection: + connection.execute(insert(table), rows) + compiled = Compiler(plugins=[PostgresPlugin(server_version=(16, 0))]).compile( + StructuredRules, table + ) + matched = set(connection.execute(select(table.c.id).where(*where(compiled))).scalars()) + failed = set( + connection.execute(select(table.c.id).where(*notwhere(compiled))).scalars() + ) + assert matched == {1} + assert failed == {2, 3, 4} + assert matched | failed == {1, 2, 3, 4} + finally: + table.drop(engine, checkfirst=True) + engine.dispose() + + +def test_live_mysql_fulltext_marker_uses_an_indexed_source() -> None: + url = os.getenv("SQLRULES_TEST_MYSQL_URL") + if not url: + pytest.skip("SQLRULES_TEST_MYSQL_URL is not configured") + + class FullTextRules(RuleSchema): + body: Annotated[str, FullTextMatch("+sqlruleswidget")] + + engine = create_engine(url) + table = Table( + f"sr_fulltext_{uuid4().hex[:10]}", + MetaData(), + Column("id", Integer, primary_key=True), + Column("body", String(255)), + ) + _index = Index(f"ix_fulltext_{uuid4().hex[:8]}", table.c.body, mysql_prefix="FULLTEXT") + try: + table.create(engine) + with engine.begin() as connection: + connection.execute( + insert(table), + [ + {"id": 1, "body": "sqlruleswidget fulltext predicates"}, + {"id": 2, "body": "apples and oranges"}, + {"id": 3, "body": "query another unrelated word"}, + ], + ) + compiled = Compiler(plugins=[MysqlPlugin(server_version=(8, 0, 36))]).compile( + FullTextRules, table + ) + with engine.connect() as connection: + matched = set(connection.execute(select(table.c.id).where(*where(compiled))).scalars()) + failed = set( + connection.execute(select(table.c.id).where(*notwhere(compiled))).scalars() + ) + assert matched == {1} + assert failed == {2, 3} + assert matched | failed == {1, 2, 3} + finally: + table.drop(engine, checkfirst=True) + engine.dispose() + + +@pytest.mark.parametrize( + ("backend", "provider_type", "server_version", "collation"), + [ + ("postgresql", PostgresPlugin, (16, 0), "C"), + ("mysql", MysqlPlugin, (8, 0, 36), "utf8mb4_bin"), + ], +) +def test_live_pattern_marker_matches_supported_text( + backend: str, + provider_type: Any, + server_version: tuple[int, ...], + collation: str, +) -> None: + class PatternRules(RuleSchema): + label: Annotated[str, Field(pattern=r"^[A-Z][a-z]+$")] + + url = os.getenv(f"SQLRULES_TEST_{backend.upper()}_URL") + if not url: + pytest.skip(f"SQLRULES_TEST_{backend.upper()}_URL is not configured") + + engine = create_engine(url) + table = Table( + f"sr_pattern_{backend}_{uuid4().hex[:10]}", + MetaData(), + Column("id", Integer, primary_key=True), + Column("label", String(32, collation=collation)), + ) + try: + table.create(engine) + with engine.begin() as connection: + connection.execute( + insert(table), + [{"id": 1, "label": "Ada"}, {"id": 2, "label": "lower"}], + ) + compiled = Compiler(plugins=[provider_type(server_version=server_version)]).compile( + PatternRules, table + ) + matched = set(connection.execute(select(table.c.id).where(*where(compiled))).scalars()) + failed = set( + connection.execute(select(table.c.id).where(*notwhere(compiled))).scalars() + ) + assert matched == {1} + assert failed == {2} + assert matched | failed == {1, 2} + finally: + table.drop(engine, checkfirst=True) + engine.dispose() diff --git a/tests/test_numeric_constraints.py b/tests/test_numeric_constraints.py deleted file mode 100644 index b47c9e2..0000000 --- a/tests/test_numeric_constraints.py +++ /dev/null @@ -1,28 +0,0 @@ -from typing import Annotated - -from pydantic import BaseModel, Field -from sqlalchemy import Table - -import sqlrules -from assert_sql import assert_expr_equals, assert_rules_sql - - -class UserFilter(BaseModel): - age: Annotated[int, Field(ge=18, le=65)] - id: Annotated[int, Field(gt=0, multiple_of=2)] - - -def test_numeric_constraints_compile(users: Table) -> None: - rules = sqlrules.compile(UserFilter, users) - - assert_rules_sql( - rules, - { - "age": ["users.age >= 18", "users.age <= 65"], - "id": ["users.id > 0", "users.id % 2 = 0"], - }, - ) - assert_expr_equals(rules["age"][0], users.c.age >= 18) - assert_expr_equals(rules["age"][1], users.c.age <= 65) - assert_expr_equals(rules["id"][0], users.c.id > 0) - assert_expr_equals(rules["id"][1], users.c.id % 2 == 0) diff --git a/tests/test_plugins.py b/tests/test_plugins.py deleted file mode 100644 index e2a7163..0000000 --- a/tests/test_plugins.py +++ /dev/null @@ -1,412 +0,0 @@ -"""SQLRules 0.3 plugin system tests.""" - -from __future__ import annotations - -from typing import Annotated, Any - -import pytest -from pydantic import BaseModel, Field -from sqlalchemy import Column, Integer, MetaData, String, Table -from sqlalchemy.dialects import sqlite -from sqlalchemy.sql.elements import ColumnElement - -import sqlrules -from assert_sql import sql_literal -from sqlrules.conformance import ( - assert_builtins_preserved, - assert_plugin_api_compatible, - assert_register_idempotent_ignore, - assert_translates_operator, - run_basic_conformance, -) -from sqlrules.constraints import pattern_text -from sqlrules.ir import CompilationContext, Constraint, DiagnosticsCollector -from sqlrules.plugins import PLUGIN_API_VERSION, validate_plugin -from sqlrules.translators import TranslatorRegistry, default_registry - - -class _PatternPlugin: - name = "test-pattern" - api_version = PLUGIN_API_VERSION - - def register(self, registry: TranslatorRegistry) -> None: - registry.register_constraint( - "pattern", - lambda c, col, ctx: col.op("~")(pattern_text(c.value)[0]), - on_conflict="replace", - ) - - -class _BadVersionPlugin: - name = "bad-version" - api_version = "999" - - def register(self, registry: TranslatorRegistry) -> None: - return None - - -class _OverrideGtPlugin: - name = "override-gt" - api_version = PLUGIN_API_VERSION - - def register(self, registry: TranslatorRegistry) -> None: - # Deliberately different from builtin ``column > value``. - registry.register("gt", lambda c, col, ctx: col.op("CUSTOM_GT")(c.value)) - - -def test_plugin_api_version_constant() -> None: - assert PLUGIN_API_VERSION == "1" - assert sqlrules.PLUGIN_API_VERSION == "1" - - -def test_validate_plugin_rejects_bad_version() -> None: - with pytest.raises(sqlrules.PluginError, match="api_version"): - validate_plugin(_BadVersionPlugin()) - - -def test_validate_plugin_rejects_non_plugin() -> None: - with pytest.raises(sqlrules.PluginError, match="SQLRulesPlugin"): - validate_plugin(object()) - - -def test_compiler_plugins_register_pattern(items: Table) -> None: - class Filter(BaseModel): - name: Annotated[str, Field(pattern=r"^a")] - - compiler = sqlrules.Compiler(plugins=[_PatternPlugin()], cache=False) - rules = compiler.compile(Filter, items) - from assert_sql import sql_literal - - assert sql_literal(rules["name"][0]) == "items.name ~ '^a'" - - -def test_compiler_on_conflict_raise() -> None: - with pytest.raises(sqlrules.RegistryError): - sqlrules.Compiler(plugins=[_OverrideGtPlugin()], on_conflict="raise", cache=False) - - -def test_compiler_on_conflict_replace(items: Table) -> None: - class Filter(BaseModel): - age: Annotated[int, Field(gt=0)] - - table = Table("t", MetaData(), Column("age", Integer)) - builtin = sqlrules.Compiler(cache=False).compile(Filter, table) - from assert_sql import sql_literal - - assert sql_literal(builtin["age"][0]) == "t.age > 0" - - compiler = sqlrules.Compiler( - plugins=[_OverrideGtPlugin()], - on_conflict="replace", - cache=False, - ) - replaced = compiler.compile(Filter, table) - assert sql_literal(replaced["age"][0]) == "t.age CUSTOM_GT 0" - assert compiler.registry.lookup("gt") is not default_registry().lookup("gt") - - -def test_compiler_on_conflict_ignore() -> None: - captured: list[Any] = [] - - class Override: - name = "override-gt" - api_version = PLUGIN_API_VERSION - - def register(self, registry: TranslatorRegistry) -> None: - def alt( - c: Constraint, col: ColumnElement[Any], ctx: CompilationContext - ) -> ColumnElement[bool]: - return col > c.value - - captured.append(alt) - registry.register("gt", alt) - - compiler = sqlrules.Compiler( - plugins=[Override()], - on_conflict="ignore", - cache=False, - ) - assert captured - assert compiler.registry.lookup("gt") is not captured[0] - - -def test_invalid_on_conflict() -> None: - with pytest.raises(sqlrules.ConfigurationError, match="on_conflict"): - sqlrules.Compiler(on_conflict="explode") # type: ignore[arg-type] - - -def test_invalid_translator_not_callable() -> None: - registry = TranslatorRegistry() - with pytest.raises(sqlrules.InvalidTranslatorError): - registry.register("custom", None) # type: ignore[arg-type] - - -def test_invalid_translator_arity() -> None: - registry = TranslatorRegistry() - - def bad(a: Any, b: Any) -> Any: - return a - - with pytest.raises(sqlrules.InvalidTranslatorError): - registry.register("custom", bad) # type: ignore[arg-type] - - -def test_register_constraint_ignore() -> None: - registry = default_registry() - first = registry.lookup("gt") - registry.register_constraint( - "gt", - lambda c, col, ctx: col > c.value, - on_conflict="ignore", - ) - assert registry.lookup("gt") is first - - -def test_register_constraint_replace() -> None: - registry = default_registry() - - def alt(c: Constraint, col: ColumnElement[Any], ctx: CompilationContext) -> ColumnElement[bool]: - return col > c.value - - registry.register_constraint("gt", alt, on_conflict="replace") - assert registry.lookup("gt") is alt - - -def test_registry_copy_and_operators() -> None: - registry = default_registry() - clone = registry.copy() - assert clone.operators() == registry.operators() - assert "gt" in clone - clone.register_constraint( - "pattern", - lambda c, col, ctx: col.op("~")(pattern_text(c.value)[0]), - on_conflict="raise", - ) - assert "pattern" in clone - assert "pattern" not in registry - - -def test_plugins_do_not_mutate_external_registry() -> None: - external = default_registry() - before = external.operators() - sqlrules.Compiler(registry=external, plugins=[_PatternPlugin()], cache=False) - assert external.operators() == before - assert "pattern" not in external - - -def test_compiler_copies_registry_without_plugins() -> None: - external = default_registry() - compiler = sqlrules.Compiler(registry=external, cache=False) - compiler.registry.register_constraint( - "pattern", - lambda c, col, ctx: col.op("~")(pattern_text(c.value)[0]), - on_conflict="raise", - ) - assert "pattern" in compiler.registry - assert "pattern" not in external - - -def test_dialect_hint_on_context(items: Table) -> None: - seen: list[str | None] = [] - - class CapturePlugin: - name = "capture" - api_version = PLUGIN_API_VERSION - - def register(self, registry: TranslatorRegistry) -> None: - def translate( - constraint: Constraint, - column: ColumnElement[Any], - context: CompilationContext, - ) -> ColumnElement[bool]: - seen.append(context.dialect) - return column.op("~")(pattern_text(constraint.value)[0]) - - registry.register_constraint("pattern", translate, on_conflict="replace") - - class Filter(BaseModel): - name: Annotated[str, Field(pattern=r"^x")] - - compiler = sqlrules.Compiler( - plugins=[CapturePlugin()], - dialect="postgresql", - cache=False, - ) - compiler.compile(Filter, items) - assert seen == ["postgresql"] - - -def test_diagnostic_code_on_warn(items: Table) -> None: - class Filter(BaseModel): - name: Annotated[str, Field(pattern=r"^a")] - - compiler = sqlrules.Compiler(on_unsupported="warn", cache=False) - with pytest.warns(sqlrules.SQLRulesWarning): - compiler.compile(Filter, items) - assert compiler.diagnostics[0].code == "unsupported_constraint" - - -def test_diagnostic_code_on_ignore() -> None: - collector = DiagnosticsCollector() - registry = TranslatorRegistry() - registry.translate( - Constraint("name", "pattern", r"^A"), - Column("name", String), - CompilationContext(on_unsupported="ignore", collector=collector), - ) - assert collector.snapshot()[0].code == "unsupported_constraint" - - -def test_conformance_helpers(items: Table) -> None: - plugin = _PatternPlugin() - assert_plugin_api_compatible(plugin) - assert_builtins_preserved(plugin, on_conflict="replace") - assert_register_idempotent_ignore(plugin) - expr = assert_translates_operator(plugin, operator="pattern", table=items) - assert isinstance(expr, ColumnElement) - run_basic_conformance(plugin, operator="pattern") - - -def test_validate_plugin_rejects_empty_name() -> None: - class EmptyName: - name = "" - api_version = PLUGIN_API_VERSION - - def register(self, registry: TranslatorRegistry) -> None: - return None - - with pytest.raises(sqlrules.PluginError, match="non-empty"): - validate_plugin(EmptyName()) - - -def test_validate_plugin_rejects_class_instead_of_instance() -> None: - class MyPlugin: - name = "my" - api_version = PLUGIN_API_VERSION - - def register(self, registry: TranslatorRegistry) -> None: - return None - - with pytest.raises(sqlrules.PluginError, match="instance"): - validate_plugin(MyPlugin) - - -def test_validate_plugin_rejects_non_callable_register() -> None: - class BadRegister: - name = "bad" - api_version = PLUGIN_API_VERSION - register = "nope" # type: ignore[assignment] - - with pytest.raises(sqlrules.PluginError, match="callable register"): - validate_plugin(BadRegister()) - - -def test_register_constraint_invalid_on_conflict() -> None: - registry = TranslatorRegistry() - with pytest.raises(sqlrules.RegistryError, match="on_conflict"): - registry.register_constraint( - "x", - lambda c, col, ctx: col == c.value, - on_conflict="nope", # type: ignore[arg-type] - ) - - -def test_configuration_error_generic_option() -> None: - err = sqlrules.ConfigurationError(option="dialect", value=123) - assert "dialect" in str(err) - - -def test_conformance_conflict_raise_helper() -> None: - from sqlrules.conformance import assert_register_conflict_raise - - class DupPlugin: - name = "dup" - api_version = PLUGIN_API_VERSION - - def register(self, registry: TranslatorRegistry) -> None: - # Use raise (default) so second registration conflicts. - registry.register_constraint( - "pattern", - lambda c, col, ctx: col.op("~")(pattern_text(c.value)[0]), - ) - - assert_register_conflict_raise(DupPlugin()) - - -def test_conformance_assert_conflict_raises_on_replace_plugin() -> None: - from sqlrules.conformance import assert_register_conflict_raise - - with pytest.raises(AssertionError, match="did not raise"): - assert_register_conflict_raise(_PatternPlugin()) - - -def test_plugin_error_str() -> None: - err = sqlrules.PluginError(message="boom", plugin=None) - assert str(err) == "boom" - - -def test_validate_translator_varargs() -> None: - registry = TranslatorRegistry() - - def flexible(*args: Any) -> Any: - constraint, column, _context = args - return column == constraint.value - - registry.register("eq_custom", flexible) - assert "eq_custom" in registry - - -def test_plugin_aware_register_constraint_default(items: Table) -> None: - """Compiler adapter applies on_conflict when register_constraint omits it.""" - - class SoftPlugin: - name = "soft" - api_version = PLUGIN_API_VERSION - - def register(self, registry: TranslatorRegistry) -> None: - registry.register_constraint( - "pattern", - lambda c, col, ctx: col.op("~")(pattern_text(c.value)[0]), - ) - - # First registration ok under raise; second under ignore must not error - # and must keep the first translator identity (ignore wins on conflict). - first = sqlrules.Compiler(plugins=[SoftPlugin()], on_conflict="raise", cache=False) - first_fn = first.registry.lookup("pattern") - second = sqlrules.Compiler(plugins=[SoftPlugin()], on_conflict="ignore", cache=False) - # Separate compilers — each starts from builtins; both should register pattern. - assert first.registry.lookup("pattern") is first_fn - assert second.registry.lookup("pattern") is not None - - class Filter(BaseModel): - name: Annotated[str, Field(pattern=r"^a")] - - rules = first.compile(Filter, items) - assert sql_literal(rules["name"][0]) == "items.name ~ '^a'" - - -def test_module_compile_stays_plugin_free(items: Table) -> None: - class Filter(BaseModel): - name: Annotated[str, Field(pattern=r"^a")] - - with pytest.raises(sqlrules.UnsupportedConstraintError, match="pattern"): - sqlrules.compile(Filter, items) - - -def test_sqlite_compile_smoke_with_custom_pattern(items: Table) -> None: - class Filter(BaseModel): - name: Annotated[str, Field(pattern=r"^a")] - - class SqliteLikePlugin: - name = "sqlite-like" - api_version = PLUGIN_API_VERSION - - def register(self, registry: TranslatorRegistry) -> None: - registry.register_constraint( - "pattern", - lambda c, col, ctx: col.op("REGEXP")(pattern_text(c.value)[0]), - on_conflict="replace", - ) - - rules = sqlrules.Compiler(plugins=[SqliteLikePlugin()], cache=False).compile(Filter, items) - assert sql_literal(rules["name"][0], dialect=sqlite.dialect()) == "items.name REGEXP '^a'" diff --git a/tests/test_rule_schema.py b/tests/test_rule_schema.py new file mode 100644 index 0000000..7fb7633 --- /dev/null +++ b/tests/test_rule_schema.py @@ -0,0 +1,308 @@ +from __future__ import annotations + +from datetime import date, datetime, time +from decimal import Decimal +from enum import Enum +from typing import Annotated, Literal +from uuid import UUID + +import pytest +from annotated_types import MinLen +from fastapi import FastAPI +from fastapi.testclient import TestClient +from pydantic import ( + BaseModel, + ConfigDict, + PositiveInt, + StrictInt, + StringConstraints, + ValidationError, + field_validator, +) +from pydantic import ( + Field as PydanticField, +) +from sqlalchemy import ( + Boolean, + Column, + Date, + DateTime, + Float, + Integer, + MetaData, + Numeric, + String, + Table, + Time, +) +from sqlalchemy.dialects.postgresql import UUID as PostgreSQLUUID +from sqlrules_postgresql import PostgresPlugin + +from sqlrules import ( + Compiler, + Field, + InvalidModelError, + RuleConfig, + RuleSchema, + UnsupportedConstraintError, + from_pydantic, +) + + +class Status(str, Enum): + READY = "ready" + PENDING = "pending" + + +def test_rule_schema_keeps_normal_pydantic_model_behavior() -> None: + class UserRules(RuleSchema): + id: PositiveInt + age: int | None = PydanticField(ge=18, le=120) + name: Annotated[str, StringConstraints(min_length=2, max_length=30)] + status: Literal["ready", "pending"] + + user = UserRules.model_validate({"id": "12", "age": "21", "name": "Ada", "status": "ready"}) + assert user.id == 12 + assert user.model_dump() == { + "id": 12, + "age": 21, + "name": "Ada", + "status": "ready", + } + with pytest.raises(ValidationError): + UserRules.model_validate({"id": 0, "age": None, "name": "A", "status": "ready"}) + + app = FastAPI() + + def echo_user(payload): + return payload + + echo_user.__annotations__ = {"payload": UserRules, "return": UserRules} + app.post("/users")(echo_user) + response = TestClient(app).post( + "/users", + json={"id": "12", "age": "21", "name": "Ada", "status": "ready"}, + ) + assert response.status_code == 200 + assert response.json()["id"] == 12 + + +def test_supported_pydantic_metadata_and_strict_aliases_normalize() -> None: + class Rules(RuleSchema): + model_config = ConfigDict(strict=True) + + count: int = PydanticField(strict=False, ge=0) + label: Annotated[str, MinLen(2)] + fixed: StrictInt + + schema = Compiler().compile_model(Rules) + assert [(field.name, field.strict) for field in schema.fields] == [ + ("count", False), + ("label", True), + ("fixed", True), + ] + assert schema.fields[0].constraints[0].operator == "ge" + assert Rules.model_validate({"count": "1", "label": "ab", "fixed": 1}) + with pytest.raises(ValidationError): + Rules.model_validate({"count": "1", "label": "ab", "fixed": "1"}) + + +def test_scalar_and_domain_annotations_compile_for_postgresql() -> None: + class Scalars(RuleSchema): + flag: bool + count: int + ratio: float + amount: Decimal + title: str + day: date + created: datetime + at: time + token: UUID + status: Status + + table = Table( + "scalars", + MetaData(), + Column("flag", Boolean), + Column("count", Integer), + Column("ratio", Float), + Column("amount", Numeric), + Column("title", String), + Column("day", Date), + Column("created", DateTime), + Column("at", Time), + Column("token", PostgreSQLUUID), + Column("status", String(collation="C")), + ) + compiled = Compiler(plugins=[PostgresPlugin()]).compile(Scalars, table) + assert len(compiled.fields) == 10 + assert compiled.backend == "postgresql" + + +def test_sqlrules_column_metadata_is_separate_from_pydantic_aliases() -> None: + class Rules(RuleSchema): + display_name: str = Field(column="display", alias="displayName", min_length=2) + + value = Rules.model_validate({"displayName": "Ada"}) + assert value.display_name == "Ada" + field = Compiler().compile_model(Rules).fields[0] + assert field.column == "display" + assert field.alias == "displayName" + + +def test_single_inheritance_preserves_order_and_empty_requires_opt_in() -> None: + class Parent(RuleSchema): + first: int + + class Child(Parent): + second: str + + assert [field.name for field in Compiler().compile_model(Child).fields] == [ + "first", + "second", + ] + + class Empty(RuleSchema): + __rule_config__ = RuleConfig(allow_empty=True) + + assert Compiler().compile_model(Empty).allow_empty + + +def test_unrestricted_pydantic_model_is_rejected_by_compiler() -> None: + class Request(BaseModel): + age: int + + with pytest.raises(InvalidModelError, match="from_pydantic"): + Compiler().compile_model(Request) # type: ignore[arg-type] + + +def test_unsupported_declarations_fail_at_class_creation() -> None: + with pytest.raises(TypeError, match="min_length"): + + class InvalidConstraint(RuleSchema): + count: Annotated[int, PydanticField(min_length=1)] + + with pytest.raises(TypeError, match="metadata"): + + class InvalidMetadata(RuleSchema): + name: Annotated[str, object()] + + with pytest.raises(TypeError, match="field_validators"): + + class ValidatorRules(RuleSchema): + age: int + + @field_validator("age") + @classmethod + def positive(cls, value: int) -> int: + return abs(value) + + with pytest.raises(TypeError, match="multiple_inheritance"): + + class Left(RuleSchema): + left: int + + class Right(RuleSchema): + other: int + + class MultipleInheritance(Left, Right): + pass + + +def test_from_pydantic_drops_incompatible_features_and_reports_them() -> None: + class Request(BaseModel): + model_config = ConfigDict(populate_by_name=True) + + user_id: int = PydanticField(default=4, alias="userId", ge=1) + payload: bytes + + @field_validator("user_id") + @classmethod + def clamp(cls, value: int) -> int: + return max(value, 1) + + with pytest.warns(UserWarning, match="changed behavior"): + conversion = from_pydantic(Request, on_incompatible="warn") + assert tuple(conversion.model.model_fields) == ("user_id",) + assert conversion.model.model_fields["user_id"].alias == "userId" + assert conversion.model().user_id == 4 + assert conversion.model.model_validate({"userId": "7"}).user_id == 7 + assert any(item.reason_code == "unsupported_field_type" for item in conversion.report.entries) + assert any(item.reason_code == "python_callback_removed" for item in conversion.report.entries) + assert any(item.reason_code == "model_config_not_copied" for item in conversion.report.entries) + assert any(item.reason_code == "field_aliases_preserved" for item in conversion.report.entries) + converted_schema = Compiler().compile_model(conversion.model) + assert converted_schema.provenance == ( + f"from_pydantic:{Request.__module__}.{Request.__qualname__}" + ) + assert converted_schema.fields[0].source_location.endswith(".user_id") + + app = FastAPI() + + def echo_converted(payload): + return payload + + echo_converted.__annotations__ = { + "payload": conversion.model, + "return": conversion.model, + } + app.post("/converted")(echo_converted) + response = TestClient(app).post("/converted", json={"userId": "9"}) + assert response.status_code == 200 + assert response.json()["userId"] == 9 + + with pytest.raises(UnsupportedConstraintError, match="remove or change"): + from_pydantic(Request, on_incompatible="raise") + + +def test_from_pydantic_does_not_run_default_factories_during_conversion() -> None: + calls = 0 + + def make_default() -> int: + nonlocal calls + calls += 1 + return 3 + + class Request(BaseModel): + count: int = PydanticField(default_factory=make_default) + + converted = from_pydantic(Request, on_incompatible="drop") + assert calls == 0 + assert converted.model().count == 3 + assert calls == 1 + + +def test_from_pydantic_splits_supported_string_constraints_from_transforms() -> None: + class Request(BaseModel): + name: Annotated[ + str, + StringConstraints( + strip_whitespace=True, + min_length=2, + max_length=12, + pattern=r"^[A-Z]", + ), + ] + + conversion = from_pydantic(Request, on_incompatible="drop") + field = Compiler().compile_model(conversion.model).fields[0] + assert {item.operator for item in field.constraints} == { + "min_length", + "max_length", + "pattern", + } + assert any( + entry.reason_code == "unsupported_metadata" and "strip_whitespace" in entry.feature + for entry in conversion.report.entries + ) + + +def test_empty_conversion_requires_allow_empty() -> None: + class Request(BaseModel): + payload: bytes + + with pytest.raises(UnsupportedConstraintError, match="allow_empty"): + from_pydantic(Request, on_incompatible="drop") + conversion = from_pydantic(Request, on_incompatible="drop", allow_empty=True) + assert conversion.report.allow_empty + assert not conversion.model.model_fields diff --git a/tests/test_sqlite_execution.py b/tests/test_sqlite_execution.py new file mode 100644 index 0000000..abce4dd --- /dev/null +++ b/tests/test_sqlite_execution.py @@ -0,0 +1,169 @@ +from __future__ import annotations + +from typing import Annotated, Literal + +from pydantic import Field, StrictInt +from sqlalchemy import Column, Integer, MetaData, String, Table, create_engine, insert, select +from sqlrules_sqlite import SQLitePlugin, register_regexp, register_sqlite_functions + +from sqlrules import Compiler, RuleConfig, RuleSchema, notwhere, where + + +def test_where_and_notwhere_partition_rows_with_nulls_and_bad_values() -> None: + class Rules(RuleSchema): + id: int + payload: int | None = Field(ge=0, multiple_of=2) + name: Annotated[str, Field(min_length=2, pattern=r"^[A-Z][a-z]+$")] + status: Literal["ready", "pending"] + + table = Table( + "records", + MetaData(), + Column("id", Integer, primary_key=True), + # TEXT storage exercises guarded text-to-integer conversion. + Column("payload", String), + Column("name", String), + Column("status", String), + ) + engine = create_engine("sqlite://") + table.create(engine) + rows = [ + {"id": 1, "payload": "12", "name": "Ada", "status": "ready"}, + {"id": 2, "payload": None, "name": "Bo", "status": "pending"}, + {"id": 3, "payload": "broken", "name": "Cy", "status": "ready"}, + {"id": 4, "payload": -1, "name": "Dee", "status": "ready"}, + {"id": 5, "payload": 12, "name": "E", "status": "ready"}, + {"id": 6, "payload": 12, "name": "Fox", "status": "unknown"}, + {"id": 7, "payload": 3, "name": "Gus", "status": "ready"}, + ] + with engine.begin() as connection: + register_regexp(connection.connection.driver_connection) + connection.execute(insert(table), rows) + compiled = Compiler(plugins=[SQLitePlugin()]).compile(Rules, table) + matched = set(connection.execute(select(table.c.id).where(*where(compiled))).scalars()) + failed = set(connection.execute(select(table.c.id).where(*notwhere(compiled))).scalars()) + + assert matched == {1, 2} + assert failed == {3, 4, 5, 6, 7} + assert matched.isdisjoint(failed) + assert matched | failed == {1, 2, 3, 4, 5, 6, 7} + engine.dispose() + + +def test_strict_scalar_requires_runtime_sqlite_storage_class() -> None: + class Rules(RuleSchema): + payload: StrictInt + + table = Table( + "records", + MetaData(), + Column("id", Integer, primary_key=True), + Column("payload", String), + ) + engine = create_engine("sqlite://") + table.create(engine) + with engine.begin() as connection: + connection.execute( + insert(table), + [ + {"id": 1, "payload": 5}, + {"id": 2, "payload": "5"}, + {"id": 3, "payload": "invalid"}, + {"id": 4, "payload": None}, + ], + ) + compiled = Compiler(plugins=[SQLitePlugin()]).compile(Rules, table) + matched = set(connection.execute(select(table.c.id).where(*where(compiled))).scalars()) + failed = set(connection.execute(select(table.c.id).where(*notwhere(compiled))).scalars()) + # TEXT affinity stores both Python integers and numeric strings as text, + # so SQLite cannot prove that any row has an integer storage class. + assert matched == set() + assert failed == {1, 2, 3, 4} + engine.dispose() + + +def test_sqlite_mixed_integer_affinity_rejects_non_integer_storage() -> None: + class Rules(RuleSchema): + payload: StrictInt + + table = Table( + "records", + MetaData(), + Column("id", Integer, primary_key=True), + Column("payload", Integer), + ) + engine = create_engine("sqlite://") + table.create(engine) + with engine.begin() as connection: + connection.execute( + insert(table), + [ + {"id": 1, "payload": 5}, + {"id": 2, "payload": "not-an-integer"}, + {"id": 3, "payload": None}, + ], + ) + compiled = Compiler(plugins=[SQLitePlugin()]).compile(Rules, table) + matched = set(connection.execute(select(table.c.id).where(*where(compiled))).scalars()) + failed = set(connection.execute(select(table.c.id).where(*notwhere(compiled))).scalars()) + assert matched == {1} + assert failed == {2, 3} + engine.dispose() + + +def test_empty_rules_select_nothing_with_notwhere() -> None: + class EmptyRules(RuleSchema): + __rule_config__ = RuleConfig(allow_empty=True) + + table = Table( + "records", + MetaData(), + Column("id", Integer, primary_key=True), + ) + engine = create_engine("sqlite://") + table.create(engine) + with engine.begin() as connection: + connection.execute(insert(table), [{"id": 1}, {"id": 2}]) + compiled = Compiler(plugins=[SQLitePlugin()]).compile(EmptyRules, table) + matched = connection.execute(select(table.c.id).where(*where(compiled))).all() + failed = connection.execute(select(table.c.id).where(*notwhere(compiled))).all() + assert matched == [(1,), (2,)] + assert failed == [] + engine.dispose() + + +def test_regexp_udf_treats_mixed_storage_values_as_non_matches() -> None: + engine = create_engine("sqlite://") + with engine.connect() as connection: + register_regexp(connection.connection.driver_connection) + assert connection.exec_driver_sql("SELECT 5 REGEXP '^5$'").scalar() == 0 + assert connection.exec_driver_sql("SELECT '5' REGEXP '^5$'").scalar() == 1 + engine.dispose() + + +def test_sqlite_length_constraints_count_text_after_embedded_nul() -> None: + class Rules(RuleSchema): + value: Annotated[str, Field(min_length=2, max_length=2)] + + table = Table( + "records", + MetaData(), + Column("id", Integer, primary_key=True), + Column("value", String), + ) + engine = create_engine("sqlite://") + table.create(engine) + with engine.begin() as connection: + register_sqlite_functions(connection.connection.driver_connection) + connection.execute( + insert(table), + [ + {"id": 1, "value": "ab"}, + {"id": 2, "value": "a\x00b"}, + ], + ) + compiled = Compiler(plugins=[SQLitePlugin()]).compile(Rules, table) + matched = set(connection.execute(select(table.c.id).where(*where(compiled))).scalars()) + + assert matched == {1} + engine.dispose() diff --git a/tests/test_string_constraints.py b/tests/test_string_constraints.py deleted file mode 100644 index 7b4a6fe..0000000 --- a/tests/test_string_constraints.py +++ /dev/null @@ -1,24 +0,0 @@ -from typing import Annotated - -from pydantic import BaseModel, Field -from sqlalchemy import Table, func - -import sqlrules -from assert_sql import assert_expr_equals, assert_rules_sql - - -class UserFilter(BaseModel): - name: Annotated[str, Field(min_length=2, max_length=50)] - - -def test_string_constraints_compile(users: Table) -> None: - rules = sqlrules.compile(UserFilter, users) - - assert_rules_sql( - rules, - { - "name": ["length(users.name) >= 2", "length(users.name) <= 50"], - }, - ) - assert_expr_equals(rules["name"][0], func.length(users.c.name) >= 2) - assert_expr_equals(rules["name"][1], func.length(users.c.name) <= 50) diff --git a/tests/test_type_checks.py b/tests/test_type_checks.py deleted file mode 100644 index 77377df..0000000 --- a/tests/test_type_checks.py +++ /dev/null @@ -1,248 +0,0 @@ -"""Tests for emit_type_checks / TypeSpec IR extraction and bind-phase translation.""" - -from __future__ import annotations - -from enum import Enum -from typing import Annotated, Any, Literal - -import pytest -from pydantic import BaseModel, ConfigDict, Field, Strict -from sqlalchemy.sql.elements import ColumnElement -from sqlalchemy.sql.schema import Table - -import sqlrules -from assert_sql import assert_rules_sql -from sqlrules import Compiler, TypeSpec, type_spec -from sqlrules.cache import ModelIRCache -from sqlrules.errors import UnsupportedConstraintError -from sqlrules.ir import CompilationContext, Constraint -from sqlrules.translators import default_registry - - -class Status(str, Enum): - ACTIVE = "ACTIVE" - DISABLED = "DISABLED" - - -def test_type_spec_helper() -> None: - spec = TypeSpec(python_type=int, strict=True, allow_none=False) - assert type_spec(spec) is spec - - -def test_type_spec_rejects_non_typespec() -> None: - with pytest.raises(TypeError, match="TypeSpec"): - type_spec("int") # type: ignore[arg-type] - - -def test_default_omits_unconstrained_type_checks(users: Table) -> None: - class Filter(BaseModel): - age: int - name: str - - rules = sqlrules.compile(Filter, users) - assert rules == {} - - -def test_emit_type_checks_extracts_ir() -> None: - class Filter(BaseModel): - age: int - name: str - - model_ir = Compiler(emit_type_checks=True, cache=False).compile_model(Filter) - by_field = {field.descriptor.name: field.constraints for field in model_ir.fields} - assert len(by_field["age"]) == 1 - assert by_field["age"][0].operator == "type_check" - age_spec = type_spec(by_field["age"][0].value) - assert age_spec == TypeSpec(python_type=int, strict=False, allow_none=False) - name_spec = type_spec(by_field["name"][0].value) - assert name_spec.python_type is str - - -def test_emit_type_checks_with_other_constraints() -> None: - class Filter(BaseModel): - age: Annotated[int, Field(ge=18)] - - model_ir = Compiler(emit_type_checks=True, cache=False).compile_model(Filter) - ops = [c.operator for c in model_ir.fields[0].constraints] - assert ops == ["ge", "type_check"] - - -def test_literal_and_enum_skip_type_check() -> None: - class Filter(BaseModel): - status: Literal["ACTIVE", "DISABLED"] - kind: Status - - model_ir = Compiler(emit_type_checks=True, cache=False).compile_model(Filter) - by_field = {field.descriptor.name: field.constraints for field in model_ir.fields} - assert [c.operator for c in by_field["status"]] == ["literal"] - assert [c.operator for c in by_field["kind"]] == ["enum"] - - -def test_optional_sets_allow_none() -> None: - class Filter(BaseModel): - age: int | None = None - - model_ir = Compiler(emit_type_checks=True, cache=False).compile_model(Filter) - spec = type_spec(model_ir.fields[0].constraints[0].value) - assert spec.allow_none is True - assert spec.python_type is int - - -def test_field_strict_overrides_model_lax() -> None: - class Filter(BaseModel): - age: Annotated[int, Field(strict=True)] - other: Annotated[int, Strict()] - - model_ir = Compiler(emit_type_checks=True, cache=False).compile_model(Filter) - specs = { - field.descriptor.name: type_spec(field.constraints[0].value) for field in model_ir.fields - } - assert specs["age"].strict is True - assert specs["other"].strict is True - - -def test_model_config_strict() -> None: - class Filter(BaseModel): - model_config = ConfigDict(strict=True) - age: int - - model_ir = Compiler(emit_type_checks=True, cache=False).compile_model(Filter) - assert type_spec(model_ir.fields[0].constraints[0].value).strict is True - - -def test_field_strict_false_overrides_model_strict() -> None: - class Filter(BaseModel): - model_config = ConfigDict(strict=True) - age: int = Field(strict=False) - - model_ir = Compiler(emit_type_checks=True, cache=False).compile_model(Filter) - assert type_spec(model_ir.fields[0].constraints[0].value).strict is False - - -def test_strict_marker_not_emitted_as_operator() -> None: - class Filter(BaseModel): - age: Annotated[int, Strict(), Field(ge=1)] - - # Without type checks, Strict must not become a fake IR operator. - model_ir = Compiler(emit_type_checks=False, cache=False).compile_model(Filter) - assert [c.operator for c in model_ir.fields[0].constraints] == ["ge"] - - -def test_cache_separates_emit_flag() -> None: - cache = ModelIRCache() - - class Filter(BaseModel): - age: int - - without = Compiler(emit_type_checks=False, cache=True, model_cache=cache).compile_model(Filter) - with_checks = Compiler(emit_type_checks=True, cache=True, model_cache=cache).compile_model( - Filter - ) - assert without.fields[0].constraints == () - assert with_checks.fields[0].constraints[0].operator == "type_check" - assert ( - Compiler(emit_type_checks=False, cache=True, model_cache=cache).compile_model(Filter) - is without - ) - assert ( - Compiler(emit_type_checks=True, cache=True, model_cache=cache).compile_model(Filter) - is with_checks - ) - - -def test_module_compile_emit_type_checks_needs_translator(users: Table) -> None: - class Filter(BaseModel): - age: int - - with pytest.raises(UnsupportedConstraintError, match="type_check") as exc: - sqlrules.compile(Filter, users, emit_type_checks=True, cache=False) - assert exc.value.operator == "type_check" - - -def test_type_check_compiles_with_custom_translator(users: Table) -> None: - """Bind-phase: type_check IR translates when a plugin/custom translator exists.""" - - class Filter(BaseModel): - age: Annotated[int, Field(ge=18)] - name: str | None = None - - def type_check_translator( - constraint: Constraint, - column: ColumnElement[Any], - context: CompilationContext, - ) -> ColumnElement[bool]: - spec = type_spec(constraint.value) - label = f"{spec.python_type.__name__}:strict={spec.strict}:none={spec.allow_none}" - return column.op("TYPE_IS")(label) - - registry = default_registry().copy() - registry.register_constraint("type_check", type_check_translator, on_conflict="raise") - rules = Compiler( - registry=registry, - emit_type_checks=True, - cache=False, - ).compile(Filter, users) - - assert_rules_sql( - rules, - { - "age": [ - "users.age >= 18", - "users.age TYPE_IS 'int:strict=False:none=False'", - ], - "name": ["users.name TYPE_IS 'str:strict=False:none=True'"], - }, - ) - - -def test_containers_skip_type_check() -> None: - from sqlrules import JsonContains - - class Filter(BaseModel): - meta: Annotated[dict[str, Any], JsonContains({"a": 1})] - - model_ir = Compiler(emit_type_checks=True, cache=False).compile_model(Filter) - ops = [c.operator for c in model_ir.fields[0].constraints] - assert "type_check" not in ops - assert "json_contains" in ops - - -def test_annotated_optional_allow_none() -> None: - class Filter(BaseModel): - age: Annotated[int, Field(ge=0)] | None = None - - model_ir = Compiler(emit_type_checks=True, cache=False).compile_model(Filter) - ops = [c.operator for c in model_ir.fields[0].constraints] - assert "ge" in ops and "type_check" in ops - type_c = next(c for c in model_ir.fields[0].constraints if c.operator == "type_check") - assert type_spec(type_c.value).allow_none is True - - -def test_float_and_uuid_type_check_ir() -> None: - from decimal import Decimal - from uuid import UUID - - class Filter(BaseModel): - amount: float - price: Decimal - id: UUID - - model_ir = Compiler(emit_type_checks=True, cache=False).compile_model(Filter) - by_name = {f.descriptor.name: type_spec(f.constraints[0].value) for f in model_ir.fields} - assert by_name["amount"].python_type is float - assert by_name["price"].python_type is Decimal - assert by_name["id"].python_type is UUID - - -def test_string_constraints_strict_flag() -> None: - from pydantic import StringConstraints - - class Filter(BaseModel): - name: Annotated[str, StringConstraints(strict=True, min_length=2)] - - model_ir = Compiler(emit_type_checks=True, cache=False).compile_model(Filter) - type_c = next(c for c in model_ir.fields[0].constraints if c.operator == "type_check") - assert type_spec(type_c.value).strict is True - # Length still extracts alongside type_check. - ops = [c.operator for c in model_ir.fields[0].constraints] - assert "min_length" in ops diff --git a/tests/test_v01_features.py b/tests/test_v01_features.py deleted file mode 100644 index 27a761b..0000000 --- a/tests/test_v01_features.py +++ /dev/null @@ -1,202 +0,0 @@ -from datetime import date, datetime -from decimal import Decimal -from enum import Enum -from typing import Annotated, Literal -from uuid import UUID - -import pytest -from annotated_types import Interval -from pydantic import AfterValidator, BaseModel, Field, StringConstraints, conint, constr -from sqlalchemy import Column, Date, DateTime, Integer, MetaData, Numeric, String, Table - -import sqlrules -from assert_sql import assert_rules_sql -from sqlrules import MissingColumnError, SQLRulesWarning, UnsupportedConstraintError - - -@pytest.fixture -def people() -> Table: - return Table( - "people", - MetaData(), - Column("age", Integer), - Column("score", Numeric), - Column("name", String), - Column("active", Integer), - Column("born", Date), - Column("created", DateTime), - Column("status", String), - Column("qty", Integer), - Column("user_age", Integer), - ) - - -def test_pattern_raises_by_default(people: Table) -> None: - class Filter(BaseModel): - name: Annotated[str, Field(pattern=r"^A")] - - with pytest.raises(UnsupportedConstraintError, match="pattern"): - sqlrules.compile(Filter, people) - - -def test_pattern_warn_keeps_supported_constraints(people: Table) -> None: - class Filter(BaseModel): - name: Annotated[str, Field(min_length=2, pattern=r"^A")] - - with pytest.warns(SQLRulesWarning, match="pattern"): - rules = sqlrules.compile(Filter, people, on_unsupported="warn") - - assert_rules_sql(rules, {"name": ["length(people.name) >= 2"]}) - - -def test_pattern_ignore(people: Table) -> None: - class Filter(BaseModel): - name: Annotated[str, Field(pattern=r"^A")] - - assert sqlrules.compile(Filter, people, on_unsupported="ignore") == {} - - -def test_decimal_precision_unsupported(people: Table) -> None: - class Filter(BaseModel): - score: Annotated[Decimal, Field(max_digits=5, decimal_places=2, ge=0)] - - with pytest.raises(UnsupportedConstraintError, match="max_digits|decimal_places"): - sqlrules.compile(Filter, people) - - -def test_after_validator_unsupported(people: Table) -> None: - class Filter(BaseModel): - age: Annotated[int, AfterValidator(lambda value: value)] - - with pytest.raises(UnsupportedConstraintError, match="AfterValidator"): - sqlrules.compile(Filter, people) - - -def test_interval_and_conint(people: Table) -> None: - class Filter(BaseModel): - age: Annotated[int, Interval(gt=0, le=100)] - qty: conint(ge=18, le=65) - - rules = sqlrules.compile(Filter, people) - assert_rules_sql( - rules, - { - "age": ["people.age > 0", "people.age <= 100"], - "qty": ["people.qty >= 18", "people.qty <= 65"], - }, - ) - - -def test_constr_and_string_constraints(people: Table) -> None: - class Filter(BaseModel): - name: constr(min_length=2, max_length=10) - status: Annotated[str, StringConstraints(min_length=1)] - - rules = sqlrules.compile(Filter, people) - assert_rules_sql( - rules, - { - "name": ["length(people.name) >= 2", "length(people.name) <= 10"], - "status": ["length(people.status) >= 1"], - }, - ) - - -def test_string_constraints_with_pattern_raises(people: Table) -> None: - class Filter(BaseModel): - name: Annotated[str, StringConstraints(min_length=1, pattern=r"x")] - - with pytest.raises(UnsupportedConstraintError, match="pattern"): - sqlrules.compile(Filter, people) - - -def test_float_decimal_date_datetime_bool_literal(people: Table) -> None: - class Filter(BaseModel): - score: Annotated[float, Field(gt=0.0, lt=100.0)] - qty: Annotated[Decimal, Field(ge=Decimal("1"), le=Decimal("10"))] - born: Annotated[date, Field(ge=date(2000, 1, 1))] - created: Annotated[datetime, Field(le=datetime(2030, 1, 1))] - active: Literal[True] - - rules = sqlrules.compile(Filter, people) - assert_rules_sql( - rules, - { - "score": ["people.score > 0.0", "people.score < 100.0"], - "qty": ["people.qty >= 1", "people.qty <= 10"], - "born": ["people.born >= '2000-01-01'"], - "created": ["people.created <= '2030-01-01 00:00:00.000000'"], - "active": ["people.active IN (1)"], - }, - ) - - -def test_lt_constraint(people: Table) -> None: - class Filter(BaseModel): - age: Annotated[int, Field(lt=65)] - - rules = sqlrules.compile(Filter, people) - assert_rules_sql(rules, {"age": ["people.age < 65"]}) - - -def test_deterministic_field_order(people: Table) -> None: - class Filter(BaseModel): - qty: Annotated[int, Field(gt=0)] - age: Annotated[int, Field(ge=18)] - name: Annotated[str, Field(min_length=1)] - - rules = sqlrules.compile(Filter, people) - assert_rules_sql( - rules, - { - "qty": ["people.qty > 0"], - "age": ["people.age >= 18"], - "name": ["length(people.name) >= 1"], - }, - ) - - -def test_field_alias_resolves_column(people: Table) -> None: - class Filter(BaseModel): - years: Annotated[int, Field(ge=18, alias="user_age")] - - rules = sqlrules.compile(Filter, people) - assert_rules_sql(rules, {"years": ["people.user_age >= 18"]}) - # Keys are Python field names, never aliases. - assert "user_age" not in rules - - -def test_unsupported_container_type(people: Table) -> None: - class Filter(BaseModel): - age: tuple[int, ...] - - with pytest.raises(UnsupportedConstraintError, match="tuple"): - sqlrules.compile(Filter, people) - - -def test_unconstrained_uuid_skipped(people: Table) -> None: - class Filter(BaseModel): - age: UUID # reuse people.age column name; unconstrained → omitted - - assert sqlrules.compile(Filter, people) == {} - - -def test_enum_and_literal(people: Table) -> None: - class Status(Enum): - ACTIVE = "ACTIVE" - DISABLED = "DISABLED" - - class Filter(BaseModel): - status: Status - - rules = sqlrules.compile(Filter, people) - assert_rules_sql(rules, {"status": ["people.status IN ('ACTIVE', 'DISABLED')"]}) - - -def test_missing_column_still_raises() -> None: - class Filter(BaseModel): - age: Annotated[int, Field(ge=18)] - - table = Table("users", MetaData(), Column("id", Integer)) - with pytest.raises(MissingColumnError, match="age"): - sqlrules.compile(Filter, table) diff --git a/tests/test_v02_features.py b/tests/test_v02_features.py deleted file mode 100644 index cb46516..0000000 --- a/tests/test_v02_features.py +++ /dev/null @@ -1,412 +0,0 @@ -"""SQLRules 0.2 feature tests: two-phase compile, cache, diagnostics, types, pattern IR.""" - -from __future__ import annotations - -import warnings -from datetime import date, datetime, time, timezone -from decimal import Decimal -from enum import Enum -from typing import Annotated, Literal -from uuid import UUID - -import pytest -from pydantic import BaseModel, Field -from sqlalchemy import Column, DateTime, Integer, MetaData, Numeric, String, Table, Time - -import sqlrules -from assert_sql import assert_expr_equals, assert_rules_sql, sql_literal -from sqlrules import SQLRulesWarning, UnsupportedConstraintError -from sqlrules.cache import ModelIRCache -from sqlrules.constraints import pattern_text -from sqlrules.ir import Constraint, PatternSpec -from sqlrules.translators import default_registry - - -@pytest.fixture -def items() -> Table: - return Table( - "items", - MetaData(), - Column("id", String), - Column("name", String), - Column("score", Numeric), - Column("starts", Time), - Column("created", DateTime), - Column("age", Integer), - ) - - -def test_two_phase_equals_oneshot(items: Table) -> None: - class Filter(BaseModel): - age: Annotated[int, Field(ge=18, le=65)] - name: Annotated[str, Field(min_length=2)] - - compiler = sqlrules.Compiler(cache=False) - oneshot = compiler.compile(Filter, items) - model_ir = compiler.compile_model(Filter) - bound = compiler.bind(model_ir, items) - - expected = { - "age": ["items.age >= 18", "items.age <= 65"], - "name": ["length(items.name) >= 2"], - } - assert_rules_sql(oneshot, expected) - assert_rules_sql(bound, expected) - for field in expected: - for a, b in zip(oneshot[field], bound[field], strict=True): - assert_expr_equals(a, b) - - -def test_metadata_cache_reuses_model_ir(items: Table) -> None: - class Filter(BaseModel): - age: Annotated[int, Field(ge=18)] - - cache = ModelIRCache() - compiler = sqlrules.Compiler(cache=True, model_cache=cache) - - first = compiler.compile_model(Filter) - second = compiler.compile_model(Filter) - assert first is second - assert first.fields[0].constraints == second.fields[0].constraints - - other = Table("other", MetaData(), Column("age", Integer)) - rules_a = compiler.bind(first, items) - rules_b = compiler.bind(second, other) - assert_rules_sql(rules_a, {"age": ["items.age >= 18"]}) - assert_rules_sql(rules_b, {"age": ["other.age >= 18"]}) - - -def test_cache_disabled_builds_fresh_ir(items: Table) -> None: - class Filter(BaseModel): - age: Annotated[int, Field(ge=18)] - - cache = ModelIRCache() - compiler = sqlrules.Compiler(cache=False, model_cache=cache) - first = compiler.compile_model(Filter) - second = compiler.compile_model(Filter) - assert first is not second - assert cache.get(Filter) is None - - -def test_cache_clear(items: Table) -> None: - class Filter(BaseModel): - age: Annotated[int, Field(ge=18)] - - cache = ModelIRCache() - compiler = sqlrules.Compiler(cache=True, model_cache=cache) - compiler.compile_model(Filter) - assert cache.get(Filter) is not None - cache.clear() - assert cache.get(Filter) is None - - -def test_diagnostics_warn_records_pattern(items: Table) -> None: - class Filter(BaseModel): - name: Annotated[str, Field(min_length=2, pattern=r"^A")] - - compiler = sqlrules.Compiler(on_unsupported="warn", cache=False) - with pytest.warns(SQLRulesWarning, match="pattern"): - rules = compiler.compile(Filter, items) - - assert_rules_sql(rules, {"name": ["length(items.name) >= 2"]}) - assert len(compiler.diagnostics) == 1 - diag = compiler.diagnostics[0] - assert diag.severity == "warning" - assert diag.field == "name" - assert diag.operator == "pattern" - assert diag.value == PatternSpec(pattern=r"^A") - assert diag.code == "unsupported_constraint" - - -def test_diagnostics_ignore_records_pattern(items: Table) -> None: - class Filter(BaseModel): - name: Annotated[str, Field(pattern=r"^A")] - - compiler = sqlrules.Compiler(on_unsupported="ignore", cache=False) - assert compiler.compile(Filter, items) == {} - assert len(compiler.diagnostics) == 1 - assert compiler.diagnostics[0].severity == "info" - assert compiler.diagnostics[0].operator == "pattern" - assert compiler.diagnostics[0].code == "unsupported_constraint" - - -def test_diagnostics_cleared_on_success(items: Table) -> None: - class Bad(BaseModel): - name: Annotated[str, Field(pattern=r"^A")] - - class Good(BaseModel): - age: Annotated[int, Field(ge=18)] - - compiler = sqlrules.Compiler(on_unsupported="ignore", cache=False) - compiler.compile(Bad, items) - assert compiler.diagnostics - compiler.compile(Good, items) - assert compiler.diagnostics == () - - -def test_pattern_extracted_as_ir_operator(items: Table) -> None: - class Filter(BaseModel): - name: Annotated[str, Field(pattern=r"^A")] - - compiler = sqlrules.Compiler(cache=False) - model_ir = compiler.compile_model(Filter) - assert model_ir.fields[0].constraints == ( - Constraint("name", "pattern", PatternSpec(pattern=r"^A")), - ) - - -def test_custom_pattern_translator(items: Table) -> None: - class Filter(BaseModel): - name: Annotated[str, Field(pattern=r"^A")] - - registry = default_registry() - - def pattern_translator(constraint, column, context): # type: ignore[no-untyped-def] - return column.op("~")(pattern_text(constraint.value)[0]) - - registry.register("pattern", pattern_translator) - compiler = sqlrules.Compiler(registry=registry, cache=False) - rules = compiler.compile(Filter, items) - assert sql_literal(rules["name"][0]) == "items.name ~ '^A'" - - -def test_uuid_literal_and_unconstrained(items: Table) -> None: - uid = UUID("12345678-1234-5678-1234-567812345678") - - class Filter(BaseModel): - id: Literal[uid] # type: ignore[valid-type] - name: UUID # unconstrained → omitted - - rules = sqlrules.compile(Filter, items, cache=False) - assert list(rules) == ["id"] - assert_expr_equals(rules["id"][0], items.c.id.in_((uid,))) - - -def test_uuid_rejects_numeric_constraints(items: Table) -> None: - class Filter(BaseModel): - id: Annotated[UUID, Field(ge=0)] # type: ignore[type-var] - - with pytest.raises(UnsupportedConstraintError, match="ge"): - sqlrules.compile(Filter, items, cache=False) - - -def test_time_range_constraints(items: Table) -> None: - class Filter(BaseModel): - starts: Annotated[time, Field(ge=time(9, 0), le=time(17, 0))] - - rules = sqlrules.compile(Filter, items, cache=False) - assert_rules_sql( - rules, - { - "starts": [ - "items.starts >= '09:00:00.000000'", - "items.starts <= '17:00:00.000000'", - ], - }, - ) - - -def test_datetime_aware_no_conversion(items: Table) -> None: - aware = datetime(2030, 1, 1, tzinfo=timezone.utc) - - class Filter(BaseModel): - created: Annotated[datetime, Field(le=aware)] - - rules = sqlrules.compile(Filter, items, cache=False) - assert_expr_equals(rules["created"][0], items.c.created <= aware) - - -def test_date_range_still_works(items: Table) -> None: - table = Table("t", MetaData(), Column("born", String)) - - class Filter(BaseModel): - born: Annotated[date, Field(ge=date(2000, 1, 1))] - - # Column type is unrelated; expression still builds. - rules = sqlrules.compile(Filter, table, cache=False) - assert_rules_sql(rules, {"born": ["t.born >= '2000-01-01'"]}) - - -def test_decimal_comparisons_and_multiple_of(items: Table) -> None: - class Filter(BaseModel): - score: Annotated[Decimal, Field(ge=Decimal("0"), multiple_of=Decimal("0.5"))] - - rules = sqlrules.compile(Filter, items, cache=False) - assert_rules_sql( - rules, - {"score": ["items.score >= 0", "items.score % 0.5 = 0"]}, - ) - - -def test_decimal_precision_still_unsupported(items: Table) -> None: - class Filter(BaseModel): - score: Annotated[Decimal, Field(max_digits=5, decimal_places=2, ge=0)] - - with pytest.raises(UnsupportedConstraintError, match="max_digits|decimal_places"): - sqlrules.compile(Filter, items, cache=False) - - -def test_module_compile_cache_flag(items: Table) -> None: - class Filter(BaseModel): - age: Annotated[int, Field(ge=18)] - - rules = sqlrules.compile(Filter, items, cache=False) - assert_rules_sql(rules, {"age": ["items.age >= 18"]}) - - -def test_temporal_multiple_of_rejected(items: Table) -> None: - class DateFilter(BaseModel): - created: Annotated[datetime, Field(multiple_of=1)] # type: ignore[type-var] - - class TimeFilter(BaseModel): - starts: Annotated[time, Field(multiple_of=1)] # type: ignore[type-var] - - with pytest.raises(UnsupportedConstraintError, match="multiple_of"): - sqlrules.compile(DateFilter, items, cache=False) - with pytest.raises(UnsupportedConstraintError, match="multiple_of"): - sqlrules.compile(TimeFilter, items, cache=False) - - -def test_re_pattern_normalized_to_pattern_spec(items: Table) -> None: - import re - - class Filter(BaseModel): - name: Annotated[str, Field(pattern=re.compile(r"^A"))] - - compiler = sqlrules.Compiler(cache=False) - model_ir = compiler.compile_model(Filter) - assert model_ir.fields[0].constraints == ( - Constraint("name", "pattern", PatternSpec(pattern=r"^A")), - ) - - -def test_re_pattern_ignore_case_preserved(items: Table) -> None: - import re - - class Filter(BaseModel): - name: Annotated[str, Field(pattern=re.compile(r"^A", re.I))] - - compiler = sqlrules.Compiler(cache=False) - model_ir = compiler.compile_model(Filter) - assert model_ir.fields[0].constraints == ( - Constraint("name", "pattern", PatternSpec(pattern=r"^A", ignore_case=True)), - ) - - -def test_validation_only_flags_ignored(items: Table) -> None: - from pydantic import StringConstraints - - class StrFilter(BaseModel): - name: Annotated[str, StringConstraints(min_length=1, strip_whitespace=True)] - - class FloatFilter(BaseModel): - score: Annotated[float, Field(ge=0, allow_inf_nan=False)] - - assert_rules_sql( - sqlrules.compile(StrFilter, items, cache=False), - {"name": ["length(items.name) >= 1"]}, - ) - assert_rules_sql( - sqlrules.compile(FloatFilter, items, cache=False), - {"score": ["items.score >= 0"]}, - ) - - -def test_module_warn_attributes_outside_sqlrules(items: Table) -> None: - class Filter(BaseModel): - name: Annotated[str, Field(pattern=r"^A")] - - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - sqlrules.compile(Filter, items, on_unsupported="warn", cache=False) - - assert caught - path = caught[0].filename.replace("\\", "/") - assert "/src/sqlrules/" not in path - assert not path.endswith("/sqlrules/compiler.py") - - -def test_uuid_enum_and_length_reject(items: Table) -> None: - class IdEnum(Enum): - A = UUID("12345678-1234-5678-1234-567812345678") - B = UUID("12345678-1234-5678-1234-567812345679") - - class EnumFilter(BaseModel): - id: IdEnum - - rules = sqlrules.compile(EnumFilter, items, cache=False) - assert_expr_equals( - rules["id"][0], - items.c.id.in_( - ( - UUID("12345678-1234-5678-1234-567812345678"), - UUID("12345678-1234-5678-1234-567812345679"), - ) - ), - ) - - class LenFilter(BaseModel): - id: Annotated[UUID, Field(min_length=1)] # type: ignore[type-var] - - with pytest.raises(UnsupportedConstraintError, match="min_length"): - sqlrules.compile(LenFilter, items, cache=False) - - -def test_pattern_on_non_str_rejected(items: Table) -> None: - class Filter(BaseModel): - age: Annotated[int, Field(pattern=r"x")] # type: ignore[type-var] - - with pytest.raises(UnsupportedConstraintError, match="pattern"): - sqlrules.compile(Filter, items, cache=False) - - -def test_phase1_pattern_phase2_policy(items: Table) -> None: - class Filter(BaseModel): - name: Annotated[str, Field(pattern=r"^A")] - - model_ir = sqlrules.Compiler(cache=False).compile_model(Filter) - assert model_ir.fields[0].constraints[0].operator == "pattern" - - with pytest.raises(UnsupportedConstraintError, match="pattern"): - sqlrules.Compiler(on_unsupported="raise", cache=False).bind(model_ir, items) - - with pytest.warns(SQLRulesWarning, match="pattern"): - assert sqlrules.Compiler(on_unsupported="warn", cache=False).bind(model_ir, items) == {} - - -def test_shared_default_cache_across_compilers(items: Table) -> None: - from sqlrules.cache import default_cache - - class Filter(BaseModel): - age: Annotated[int, Field(ge=18)] - - default_cache().clear() - c1 = sqlrules.Compiler(cache=True) - c2 = sqlrules.Compiler(cache=True) - ir1 = c1.compile_model(Filter) - ir2 = c2.compile_model(Filter) - assert ir1 is ir2 - - -def test_compile_model_clears_diagnostics(items: Table) -> None: - class Bad(BaseModel): - name: Annotated[str, Field(pattern=r"^A")] - - class Good(BaseModel): - age: Annotated[int, Field(ge=18)] - - compiler = sqlrules.Compiler(on_unsupported="ignore", cache=False) - compiler.compile(Bad, items) - assert compiler.diagnostics - compiler.compile_model(Good) - assert compiler.diagnostics == () - - -def test_constrained_time_missing_column_raises() -> None: - table = Table("t", MetaData(), Column("other", Integer)) - - class Filter(BaseModel): - starts: Annotated[time, Field(ge=time(9, 0))] - - with pytest.raises(sqlrules.MissingColumnError): - sqlrules.compile(Filter, table, cache=False) diff --git a/tests/test_v04_markers.py b/tests/test_v04_markers.py deleted file mode 100644 index c1aa88e..0000000 --- a/tests/test_v04_markers.py +++ /dev/null @@ -1,162 +0,0 @@ -"""SQLRules 0.4 markers, PatternSpec, and list/dict type support.""" - -from __future__ import annotations - -from typing import Annotated, Any - -import pytest -from pydantic import BaseModel, Field -from sqlalchemy import Column, Integer, MetaData, String, Table - -import sqlrules -from assert_sql import assert_custom_op -from sqlrules import ( - ArrayContains, - ArrayOverlap, - Compiler, - FullTextMatch, - JsonContains, - JsonHasKey, - RangeContains, - RangeOverlap, - UnsupportedConstraintError, -) -from sqlrules.ir import Constraint -from sqlrules.plugins import PLUGIN_API_VERSION -from sqlrules.translators import default_registry - - -@pytest.fixture -def items() -> Table: - return Table( - "items", - MetaData(), - Column("name", String), - Column("meta", String), - Column("tags", String), - Column("span", Integer), - Column("body", String), - Column("age", Integer), - ) - - -def test_marker_extracted_into_ir(items: Table) -> None: - class Filter(BaseModel): - meta: Annotated[dict[str, Any], JsonContains({"active": True})] - tags: Annotated[list[str], ArrayContains(["admin"]), ArrayOverlap(["x"])] - span: Annotated[int, RangeContains(5), RangeOverlap([1, 10])] - body: Annotated[str, FullTextMatch("hello")] - - model_ir = Compiler(cache=False).compile_model(Filter) - by_field = {field.descriptor.name: field.constraints for field in model_ir.fields} - assert by_field["meta"] == (Constraint("meta", "json_contains", {"active": True}),) - assert by_field["tags"] == ( - Constraint("tags", "array_contains", ["admin"]), - Constraint("tags", "array_overlap", ["x"]), - ) - assert by_field["span"] == ( - Constraint("span", "range_contains", 5), - Constraint("span", "range_overlap", [1, 10]), - ) - assert by_field["body"] == (Constraint("body", "fulltext_match", "hello"),) - - -def test_marker_requires_plugin_translator(items: Table) -> None: - class Filter(BaseModel): - meta: Annotated[dict[str, Any], JsonHasKey("active")] - - with pytest.raises(UnsupportedConstraintError, match="json_has_key"): - sqlrules.compile(Filter, items) - - -def test_marker_compiles_with_custom_translator(items: Table) -> None: - class Filter(BaseModel): - meta: Annotated[dict[str, Any], JsonContains({"a": 1})] - - registry = default_registry().copy() - registry.register_constraint( - "json_contains", - lambda c, col, ctx: col.op("@>")(c.value), - on_conflict="raise", - ) - rules = Compiler(registry=registry, cache=False).compile(Filter, items) - assert_custom_op( - rules["meta"][0], - column_name="meta", - opstring="@>", - value={"a": 1}, - ) - - -def test_each_marker_compiles_with_dedicated_translator(items: Table) -> None: - class Filter(BaseModel): - meta: Annotated[dict[str, Any], JsonHasKey("k")] - tags: Annotated[list[str], ArrayContains(["a"]), ArrayOverlap(["b"])] - span: Annotated[int, RangeContains(3), RangeOverlap([0, 9])] - body: Annotated[str, FullTextMatch("q")] - - registry = default_registry().copy() - registry.register_constraint( - "json_has_key", - lambda c, col, ctx: col.op("?")(c.value), - on_conflict="raise", - ) - registry.register_constraint( - "array_contains", - lambda c, col, ctx: col.op("@>")(c.value), - on_conflict="raise", - ) - registry.register_constraint( - "array_overlap", - lambda c, col, ctx: col.op("&&")(c.value), - on_conflict="raise", - ) - registry.register_constraint( - "range_contains", - lambda c, col, ctx: col.op("@>")(c.value), - on_conflict="raise", - ) - registry.register_constraint( - "range_overlap", - lambda c, col, ctx: col.op("&&")(c.value), - on_conflict="raise", - ) - registry.register_constraint( - "fulltext_match", - lambda c, col, ctx: col.op("@@")(c.value), - on_conflict="raise", - ) - rules = Compiler(registry=registry, cache=False).compile(Filter, items) - assert_custom_op(rules["meta"][0], column_name="meta", opstring="?", value="k") - assert_custom_op(rules["tags"][0], column_name="tags", opstring="@>", value=["a"]) - assert_custom_op(rules["tags"][1], column_name="tags", opstring="&&", value=["b"]) - assert_custom_op(rules["span"][0], column_name="span", opstring="@>", value=3) - assert_custom_op(rules["span"][1], column_name="span", opstring="&&", value=[0, 9]) - assert_custom_op(rules["body"][0], column_name="body", opstring="@@", value="q") - - -def test_portable_constraint_on_list_rejected(items: Table) -> None: - class Filter(BaseModel): - tags: Annotated[list[int], Field(ge=1)] # type: ignore[type-var] - - with pytest.raises(UnsupportedConstraintError, match="list/dict"): - Compiler(cache=False).compile_model(Filter) - - -def test_pattern_on_list_rejected(items: Table) -> None: - class Filter(BaseModel): - tags: Annotated[list[str], Field(pattern=r"x")] # type: ignore[type-var] - - with pytest.raises(UnsupportedConstraintError, match="list/dict"): - Compiler(cache=False).compile_model(Filter) - - -def test_markers_exported() -> None: - assert sqlrules.JsonContains({"a": 1}).operator == "json_contains" - assert sqlrules.JsonHasKey("a").operator == "json_has_key" - assert sqlrules.ArrayContains([1]).operator == "array_contains" - assert sqlrules.ArrayOverlap([1]).operator == "array_overlap" - assert sqlrules.RangeContains(1).operator == "range_contains" - assert sqlrules.RangeOverlap([1, 2]).operator == "range_overlap" - assert sqlrules.FullTextMatch("foo").operator == "fulltext_match" - assert PLUGIN_API_VERSION == "1"