From 1010ae085daac381fcd7c67fe3e62c926801380e Mon Sep 17 00:00:00 2001 From: mattsenicksigma Date: Mon, 3 Aug 2026 08:28:36 -0400 Subject: [PATCH] [OSSIE][SIGMA] Add bidirectional Sigma Computing data model converter Adds converters/sigma, a hub-and-spoke converter between Sigma Computing data model specs and Apache Ossie, following the same structure/tooling as the dbt and NVIDIA GSF converters (uv, apache-ossie pydantic models). - A real tokenizer/parser/renderer for Sigma's formula language (ossie_sigma.sigma_formula), translating to ANSI SQL where a faithful mapping exists and always preserving the original formula verbatim in a new SIGMA dialect entry for lossless round-tripping. - Sigma <-> OSI mapping for datasets, fields, relationships (including Sigma's two column-addressing schemes), and model-level metrics, with native Sigma ids preserved via custom_extensions for stable re-export. - Controls and named/static filters are intentionally not modeled as OSI concepts (no portable equivalent) but round-trip byte-for-byte via custom_extensions; see converters/sigma/LIMITATIONS.md for this and other documented tradeoffs. - Adds SIGMA to OSIDialect/OSIVendor (python/src/ossie/models.py) and the core-spec schema/docs, plus a Sigma column in the expression_language.md cross-tool mapping tables. - Fixes two small pre-existing gaps found while validating output: core-spec/osi-schema.json was missing root-level dialects/vendors properties already present in the pydantic model, and validation/validate.py didn't skip SQL-syntax checking for the new SIGMA dialect the same way it already does for MDX/TABLEAU/MAQL. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/converter-sigma-ci.yml | 65 ++ converters/README.md | 1 + converters/sigma/LIMITATIONS.md | 332 +++++++++ converters/sigma/README.md | 176 +++++ converters/sigma/pyproject.toml | 69 ++ converters/sigma/src/ossie_sigma/__init__.py | 21 + converters/sigma/src/ossie_sigma/cli.py | 89 +++ .../sigma/src/ossie_sigma/converter_issues.py | 53 ++ .../sigma/src/ossie_sigma/expression_utils.py | 112 +++ .../sigma/src/ossie_sigma/osi_to_sigma.py | 301 ++++++++ .../sigma/src/ossie_sigma/sigma_formula.py | 652 ++++++++++++++++++ .../sigma/src/ossie_sigma/sigma_to_osi.py | 353 ++++++++++ converters/sigma/tests/__init__.py | 0 .../sigma/tests/fixtures/fixtureA_sigma.json | 100 +++ .../sigma/tests/fixtures/fixtureB_sigma.json | 84 +++ converters/sigma/tests/helpers.py | 35 + converters/sigma/tests/test_osi_to_sigma.py | 81 +++ converters/sigma/tests/test_roundtrip.py | 63 ++ converters/sigma/tests/test_sigma_formula.py | 114 +++ converters/sigma/tests/test_sigma_to_osi.py | 124 ++++ converters/sigma/uv.lock | 315 +++++++++ core-spec/expression_language.md | 96 +-- core-spec/osi-schema.json | 16 +- core-spec/spec.md | 2 + python/src/ossie/models.py | 2 + validation/validate.py | 3 +- 26 files changed, 3210 insertions(+), 49 deletions(-) create mode 100644 .github/workflows/converter-sigma-ci.yml create mode 100644 converters/sigma/LIMITATIONS.md create mode 100644 converters/sigma/README.md create mode 100644 converters/sigma/pyproject.toml create mode 100644 converters/sigma/src/ossie_sigma/__init__.py create mode 100644 converters/sigma/src/ossie_sigma/cli.py create mode 100644 converters/sigma/src/ossie_sigma/converter_issues.py create mode 100644 converters/sigma/src/ossie_sigma/expression_utils.py create mode 100644 converters/sigma/src/ossie_sigma/osi_to_sigma.py create mode 100644 converters/sigma/src/ossie_sigma/sigma_formula.py create mode 100644 converters/sigma/src/ossie_sigma/sigma_to_osi.py create mode 100644 converters/sigma/tests/__init__.py create mode 100644 converters/sigma/tests/fixtures/fixtureA_sigma.json create mode 100644 converters/sigma/tests/fixtures/fixtureB_sigma.json create mode 100644 converters/sigma/tests/helpers.py create mode 100644 converters/sigma/tests/test_osi_to_sigma.py create mode 100644 converters/sigma/tests/test_roundtrip.py create mode 100644 converters/sigma/tests/test_sigma_formula.py create mode 100644 converters/sigma/tests/test_sigma_to_osi.py create mode 100644 converters/sigma/uv.lock diff --git a/.github/workflows/converter-sigma-ci.yml b/.github/workflows/converter-sigma-ci.yml new file mode 100644 index 00000000..7d3599bf --- /dev/null +++ b/.github/workflows/converter-sigma-ci.yml @@ -0,0 +1,65 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Converters Sigma CI + +on: + push: + branches: [ "main" ] + paths: + - 'converters/sigma/**' + - 'python/**' + - '.github/workflows/converter-sigma-ci.yml' + pull_request: + branches: [ "main" ] + paths: + - 'converters/sigma/**' + - 'python/**' + - '.github/workflows/converter-sigma-ci.yml' + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11", "3.12", "3.13", "3.14"] + + steps: + - name: Checkout project + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Install uv + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + echo "${HOME}/.local/bin" >> "${GITHUB_PATH}" + + - name: Sync dependencies + working-directory: converters/sigma + run: | + uv sync + + - name: Unit Tests + working-directory: converters/sigma + run: | + uv run pytest diff --git a/converters/README.md b/converters/README.md index 5c9a4d54..d089079e 100644 --- a/converters/README.md +++ b/converters/README.md @@ -76,6 +76,7 @@ The Ossie specification currently defines extensions for the following vendors: | `OMNI` | Omni semantic model | | `WISDOM` | WisdomAI domain | | `NVIDIA_GSF` | NVIDIA Generative Semantic Fabric standalone YAML | +| `SIGMA` | Sigma Computing data model | Each vendor may define custom extensions (via the `custom_extensions` field in the Ossie spec) to carry vendor-specific metadata that does not have an equivalent in the core specification. diff --git a/converters/sigma/LIMITATIONS.md b/converters/sigma/LIMITATIONS.md new file mode 100644 index 00000000..8f9453d8 --- /dev/null +++ b/converters/sigma/LIMITATIONS.md @@ -0,0 +1,332 @@ + + +# Limitations, design tradeoffs, and testing strategy + +This document is a deliberately honest accounting of what `converters/sigma` does not +(yet) handle faithfully, why, and what a general-purpose fix would look like versus a +Sigma-specific workaround. It also documents how the converter is tested and gives an +assessment of how this contribution is likely to be received upstream. + +## 1. Controls and workbook-level filters are not modeled + +Sigma data models can contain `kind: control` elements (date-range pickers, dropdown +filters, etc.) that other parts of a workbook reference. A control has no analog in +the OSI core spec — it isn't a dataset, field, relationship, or metric; it's a +*presentation-layer* input that a downstream Sigma workbook wires to one or more +columns via `filters: [{columnId, source: {elementId}}]`. + +**What this converter does:** preserves every control element verbatim, byte-for-byte, +in a model-level `custom_extensions` entry (`vendor_name: SIGMA`, keyed under +`non_table_elements`, alongside the page it lived on). `osi-to-sigma` restores it +unchanged. A `ConverterIssue(CONTROL_ELEMENT_NOT_MODELED)` is always recorded so +callers know a control was round-tripped opaquely rather than actually converted. + +**Why not model it in OSI:** a control is fundamentally about Sigma's own UI +(what widget renders, what workbook pages it applies to) — projecting it into the +portable spec would mean adding Sigma-specific concepts to a vendor-neutral format, +which runs against the stated purpose of OSI ("the general spirit of Open Semantic +Interchange should be maintained rather than hacking Sigma-specific functionality"). +The same reasoning applies to Sigma's **named/static element filters** +(`element.filters[]`, distinct from control filters) — these are preserved verbatim in +each dataset's `custom_extensions` with a `FILTER_NOT_MODELED` issue, for the same +reason. + +**What a real fix looks like:** this is a job for a **Sigma-specific API layer above +the converter**, not the converter itself — e.g. a small adapter that, after an +`osi-to-sigma` conversion, re-applies any previously-captured controls/filters via +Sigma's own workbook/control API calls (which this converter does not call — it only +produces/consumes the data model spec document). That adapter is legitimately +Sigma-specific glue code and does not belong in a hub-and-spoke OSI converter. + +## 2. Calculated fields on tables: included, deliberately + +Sigma lets every table element define calculated columns (formulas referencing other +columns on the same element) alongside physical passthrough columns. This converter +maps **all** of them — physical and calculated alike — to `OSIField`, using the same +disambiguation OSI already has for any dataset: an `OSIField.expression` is *just an +expression*, whether it happens to be `[TABLE/COL]` (a passthrough) or +`If([Status] = "closed", 1, 0)` (a calculation). There's no reason to exclude +calculated fields from a dataset's `fields[]` — the OSI core spec draws no such +distinction, and doing so would need an invented Sigma-only concept ("calculated vs. +physical field") that has no home in the core spec. Sigma's *metrics* (element-scoped +aggregate formulas) are a different, real distinction — the core spec already has a +separate `OSIMetric` concept for exactly this, and Sigma's metrics map onto it (see +§4 for the promotion nuance). + +## 3. Stable ids: preserved-by-default, synthesized-as-fallback + +Sigma element, column, and relationship ids are load-bearing — other objects +(controls, other data models' relationships/materializations, deployment policies) +reference them by id, so an export that minted new ids for previously-existing objects +would silently break those references on re-import. + +**What this converter does:** never invents an id for anything that already has one. +Every id-bearing Sigma object's native id is preserved verbatim inside that object's +`custom_extensions` (e.g. `{"id": "colOrderId", ...}` on the corresponding `OSIField`), +and `osi-to-sigma` reuses it unchanged. Ids are synthesized — as a deterministic +`uuid5` of a fixed namespace plus the object's dataset/field path — **only** for +objects with no preserved Sigma id, i.e. ones that originate purely in an Ossie +document that was never round-tripped through Sigma. This makes `osi-to-sigma` +deterministic and idempotent (verified by `test_ids_are_deterministic_across_repeated_conversions`), +but a synthesized id is *not* a real Sigma id in the sense of being pre-registered +with Sigma's backend — the first time such a document is actually loaded into Sigma +(via `sigcli data-models spec create`), Sigma's own API, not this converter, is the +source of truth for whether that id is accepted. + +## 4. Relationship (join) resolution has a real, documented gap + +Sigma relationships address join-key columns in **two different ways** within the +same `keys[]` array: either by the owning element's own column id, or by a raw +`inode-/` reference straight to the underlying warehouse +table/column — bypassing the element's modeled column list entirely. (This second +form shows up whenever a relationship key is a physical column that was never +explicitly redefined as one of the element's own `columns[]` entries.) + +**What this converter does:** resolves both forms to a modeled Ossie field name where +possible (matching the physical column name against each element's own column +formulas), and **always preserves the raw, unresolved `sourceColumnId`/ +`targetColumnId` values in the relationship's `custom_extensions`** regardless of +whether resolution succeeded. This means round-trip fidelity (Sigma → OSI → Sigma) is +guaranteed even when the human-readable `from_columns`/`to_columns` names in the OSI +document are only best-effort. When resolution fails, a +`RELATIONSHIP_COLUMN_UNRESOLVED` issue is recorded rather than silently guessing. + +**What's genuinely unsolved:** an Ossie document authored by a *different* tool +(e.g. hand-written, or round-tripped through dbt) that gets converted to Sigma has no +such raw reference to fall back on — `osi-to-sigma` must synthesize +`sourceColumnId`/`targetColumnId` values from the field names alone, via each +element's own modeled columns. This works correctly as long as every joined field +exists as a named column, but Sigma's implicit "every warehouse column is +automatically a column, even a hidden/undefined one" behavior means there could be +edge cases (a foreign-tool document referencing a physical column that was deliberately +never redefined) this converter cannot correctly recreate without more information +than the OSI document contains. This is inherent to Sigma's addressing scheme, not +fixable purely within the converter. + +## 5. Derived ("child") elements and custom SQL elements + +A Sigma element's `source.kind` can be `warehouse-table` (a physical table) or +something else (e.g. `table`, meaning the element is layered on another element +rather than a warehouse table directly). This converter still creates an `OSIDataset` +for a derived element — with `source` set to a synthetic `element:` +marker rather than a `database.schema.table` string — and flags it with +`DERIVED_ELEMENT_NOT_MODELED`, since OSI's `OSIDataset.source` field is documented as +a physical location string and has no first-class "this dataset is derived from +another dataset" relationship concept. The parent element id is preserved in +`custom_extensions` so `osi-to-sigma` reconstructs the exact original `source` block. + +Sigma workbooks can also define **custom SQL elements** (a table backed by a +hand-written SQL query rather than a warehouse table reference or a data-model +formula). This converter has not been exercised against that element shape — Sigma's +public data-model spec endpoint was not observed producing one in the data models +inspected during development. If Sigma represents a custom-SQL element with a +`source.kind` other than the two handled here, it will currently be treated like any +other non-`warehouse-table` source (preserved as a derived element with a +`DERIVED_ELEMENT_NOT_MODELED` issue) rather than mapped to something more precise — +this is a gap to close with real fixture data from a workbook that uses one. + +## 6. Formula language coverage: real but intentionally bounded + +`ossie_sigma.sigma_formula` implements a genuine tokenizer, recursive-descent parser, +and bidirectional ANSI SQL renderer (not a regex classifier) supporting: nested +function calls at arbitrary depth, all comparison/logical/arithmetic operators, +string/number/boolean literals, and roughly 30 Sigma functions across aggregation, +conditional, string, and date categories (see the module docstring and +`core-spec/expression_language.md`'s Sigma column in the Cross-Reference Tool +Mappings tables for the full list). + +**Deliberately out of scope:** Sigma's *table calculation* functions (`RowNumber`, +`Rank`, `RunningSum`, `RunningAvg`, `Lag`, `Lead`, etc.) resolve their partition/order +context from **UI configuration** (which pivot table or chart the calculation is +attached to), not from arguments passed in the formula text. There is no way to +recover that context from the formula string alone, so these are correctly identified +as untranslatable — the original Sigma formula is preserved in the `SIGMA` dialect, +but no `ANSI_SQL` dialect entry is produced. This is not a parser limitation; it's a +structural fact about where Sigma stores that information (outside the formula). + +**Every formula, translatable or not, is never lost:** the `SIGMA` dialect entry +always carries the original text verbatim, so nothing is silently dropped — +untranslatable formulas simply don't get a second, `ANSI_SQL`-dialect representation. + +## 7. Maximizing ANSI SQL vs. vendor-specific dialects + +Per the "maximum set of expressions representable as ANSI SQL" goal, the current +implementation targets `ANSI_SQL` only (no Snowflake/Databricks/BigQuery-specific +translation) because Sigma data models are themselves warehouse-agnostic — a formula +like `Sum([Amount])` means the same thing regardless of which connection the element +points to, so there is no Sigma-side signal indicating which vendor SQL dialect would +be more useful to target. A natural follow-up (out of scope for this PR) would be: for +formulas this converter can't express in ANSI SQL, check the target connection's +warehouse type (Snowflake, BigQuery, Databricks — available on the connection, not +surfaced in the data model spec used here) and add a vendor-specific `OSIDialect` +entry using that warehouse's native date/window function syntax, which would recover +some of the table-calculation functions in §6 for warehouses with native window +function support (though the partition/order-context problem remains — Sigma still +doesn't hand that information to the formula). + +## 8. Cross-dataset ("model-level") metrics have no Sigma equivalent + +A Sigma metric (`element.metrics[]`) is always scoped to exactly one element. An OSI +`OSIMetric`, by contrast, lives at the model level and may reference multiple +datasets via relationships (e.g. a ratio metric like +`SUM(store_sales.ss_ext_sales_price) / COUNT(DISTINCT customer.c_customer_sk)`). + +**Sigma → OSI:** every Sigma metric promotes cleanly to an `OSIMetric`, re-qualified +with its owning element's name, with `element_id` preserved in `custom_extensions` +for exact placement on the way back. + +**OSI → Sigma:** for a metric with a preserved `element_id` (round-tripped from +Sigma), placement is exact. For a metric with no such extension (authored elsewhere), +the converter inspects the `ANSI_SQL` expression's column qualifiers and attaches the +metric to the single dataset it unambiguously references — but if the expression +spans more than one dataset (or none), there is **no faithful Sigma representation**, +and the metric is dropped with a `CROSS_DATASET_METRIC_DROPPED` issue rather than +silently attached to an arbitrary dataset or silently discarded without a trace. This +is exercised directly by `test_osi_to_sigma.py::test_foreign_origin_document_synthesizes_valid_spec` +against `examples/tpcds_semantic_model.yaml`, which contains exactly this kind of +metric (`customer_lifetime_value`, `store_productivity`). + +## 9. `Opaque` and `custom_extensions` usage discipline + +- `datatype: Opaque` is used **only** when a Sigma column's `format.kind` has no + portable Ossie equivalent (observed example: `variant`) — never as a default or + fallback for "we didn't bother mapping this." Most Sigma columns carry no `format` + at all in the data models this converter was developed against, and those fields + correctly get no `datatype` at all (per the core spec: "omit `datatype` when the + type is unknown or unspecified"), not `Opaque`. +- Every `custom_extensions` entry this converter writes carries only the minimum + Sigma-specific data needed for round-trip fidelity (native id, page placement, + folder/order UI grouping, raw relationship keys, unrecognized column format) — it + is never used as a dumping ground for data that has a proper OSI home (e.g. a + column's description goes in `OSIField.description`, not `custom_extensions`). +- The one intentional exception is `non_table_elements` (control/unknown-kind + elements) at the model level, and the full `folders`/`order` UI-grouping metadata + per dataset — both are genuinely presentation-layer Sigma concepts with no OSI + equivalent (see §1), so `custom_extensions` is the correct (and only) home for + them, not a workaround for something OSI should have modeled instead. + +## 10. Multiple semantic models per document + +Sigma data models are always a single model; `OSIDocument.semantic_model` is a list. +If given a multi-model Ossie document, only `semantic_model[0]` is converted, and a +`ConverterIssue` is recorded naming how many additional models were dropped, rather +than silently ignoring them or guessing which one the caller meant. + +## How this converter is tested + +- **Unit tests** (`tests/test_sigma_formula.py`) exercise the formula parser/renderer + directly: every supported function/operator in both directions, plus the + untranslatable and unparseable cases, independent of the surrounding converter. +- **Fixture-driven directional tests** (`tests/test_sigma_to_osi.py`, + `tests/test_osi_to_sigma.py`) use two hand-authored, synthetic (non-proprietary) + Sigma data model fixtures: + - `fixtureA_sigma.json` — the common path: two related tables, a composite-free + relationship, model-level metrics, nested calculated-field formulas, and a + control element. + - `fixtureB_sigma.json` — the edge cases: a composite (multi-key) relationship + mixing both column-id and `inode-`-style physical references, an unrecognized + column format (→ `Opaque`), an untranslatable table-calculation formula + (`RunningSum`), and a derived (non-warehouse-table) element. +- **Round-trip tests** (`tests/test_roundtrip.py`) assert both fixtures survive + Sigma → OSI → Sigma **byte-for-byte** (structurally, modulo key order), including + through the same YAML serialization boundary the CLI uses, and separately assert + that Sigma → OSI → Sigma → OSI preserves all portable (non-`custom_extensions`) + content on the second OSI document. +- **Real-world / foreign-origin coverage** (`test_osi_to_sigma.py::test_foreign_origin_document_synthesizes_valid_spec`) + runs the reverse converter against `examples/tpcds_semantic_model.yaml` — an Ossie + document that never touched Sigma — to verify the converter produces a valid, + useful Sigma spec (synthesized ids, ANSI-SQL-to-Sigma-formula reverse translation, + correct single-dataset metric placement, correct dropping of genuinely + cross-dataset metrics) even with no `SIGMA` custom extensions to fall back on. +- **Schema validation**: every fixture's `sigma-to-osi` output was checked against + `core-spec/osi-schema.json` and `validation/validate.py`'s SQL-syntax checker (both + of which required small, narrowly-scoped fixes as part of this PR — see below). + +What is *not* yet covered: a fixture generated from a real production Sigma data model +(only synthetic fixtures are included, deliberately, to avoid embedding any +organization's proprietary schema/business logic in an open-source repository), and +the custom-SQL-element case noted in §5. + +## Small fixes bundled with this PR (not Sigma-specific) + +While validating this converter's output against the repo's existing tooling, two +pre-existing gaps were found and fixed, since they block *any* converter from +producing schema-valid output that uses features already present in the pydantic +model: + +- `core-spec/osi-schema.json` was missing `dialects`/`vendors` as valid root-level + `OSIDocument` properties, even though `python/src/ossie/models.py`'s `OSIDocument` + has defined and exported them since before this PR. Added them. +- `validation/validate.py`'s SQL-syntax checker attempted to parse every dialect's + expression as SQL, including known non-SQL dialects — but its own + `SKIP_SQL_VALIDATION` set already excluded `MDX`/`TABLEAU`/`MAQL` for exactly this + reason. `SIGMA` was missing from that set (understandably, since it didn't exist + before this PR) and has been added alongside the new dialect. + +## Assessment: likelihood of upstream approval + +Apache Ossie uses a review-then-commit model (per `CONTRIBUTING.md`): merge requires +at least one committer +1 and no unresolved -1, and any change to `core-spec/` itself +carries a higher bar (dev@ discussion, then a `[VOTE]` thread). This PR is mostly +**not** a core-spec change — it adds a new converter under `converters/sigma/` +following the exact structure, tooling (`uv`), and conventions of the most recently +merged converter (NVIDIA GSF, PR #247) and the most actively maintained one (dbt). +The genuinely core-spec-touching pieces are narrow and precedented: + +- Adding `SIGMA` to `OSIDialect`/`OSIVendor` in `python/src/ossie/models.py` and the + corresponding `core-spec/osi-schema.json` enums — the same kind of addition every + prior converter needed (`TABLEAU`, `DATABRICKS`, `BIGQUERY`, `WISDOM`, etc. all + entered the enums this way), not a structural spec change. + It is worth confirming with the community whether such additions require the + full dev@/VOTE process or have historically been accepted as ordinary PR review — + the git history suggests the latter, but this PR does not assume that. +- The `dialects`/`vendors` schema-sync fix and the `validate.py` `SIGMA` addition are + small, mechanical, and justified independently of Sigma (see above). + +Reasonable committer concerns to expect, roughly in order of likely weight: + +1. **"Why does the relationship resolution need two addressing schemes?"** — this is + inherent to Sigma's own data model (§4), not a design choice in this converter, + but it's the single most complex piece of logic here and the one most likely to + draw close review. +2. **Formula language coverage as a moving target** — Sigma's formula function list + isn't formally published as a machine-readable grammar (unlike, say, ANSI SQL's + own grammar), so a committer may reasonably ask how coverage will be + validated/extended over time. The test suite and the module docstring's function + table are the answer, but this is worth calling out explicitly in the PR + description. +3. **Scope of `custom_extensions` for controls** — stashing whole native elements + verbatim mirrors the GSF converter's precedent (README §"Fidelity and unavoidable + losses"), but a reviewer unfamiliar with that precedent might initially read it as + "hacking around" rather than the documented, intentional choice it is; pointing + reviewers at this LIMITATIONS.md file and the GSF README directly should resolve + that quickly. +4. **No real-world fixture** — reasonable, and addressed above; a committer may ask + for one, which would need to come from a community member willing to contribute a + sanitized example (this PR intentionally does not include one, to avoid + embedding any organization's data model in the ASF repository). + +Net assessment: **likely mergeable with normal review iteration**, on the strength of +following established converter conventions closely and treating limitations as +first-class, tested, and documented rather than papered over — which is exactly what +`converters/README.md`'s own "Writing a Converter" checklist and round-trip fidelity +principles ask for. The main risk to merge speed is committer bandwidth/review +latency (an ASF-standard risk for any PR, not specific to this one), not a structural +objection to the approach. diff --git a/converters/sigma/README.md b/converters/sigma/README.md new file mode 100644 index 00000000..a9efebc7 --- /dev/null +++ b/converters/sigma/README.md @@ -0,0 +1,176 @@ + + +# apache-ossie-sigma + +Converts between [Sigma Computing](https://www.sigmacomputing.com/) Data Models (the +"code representation" spec returned by `GET /v2/dataModels/{id}/spec`, and accepted by +`POST`/`PUT` on the same resource) and the [Apache Ossie](https://github.com/apache/ossie) +format. + +Both conversion directions are supported: + +- `sigma-to-osi` — Sigma data model spec JSON → Ossie YAML +- `osi-to-sigma` — Ossie YAML → Sigma data model spec JSON + +## Requirements + +- Python 3.11+ +- [uv](https://docs.astral.sh/uv/) (recommended) or pip + +## Installation + +```bash +pip install apache-ossie-sigma +``` + +Or with uv: + +```bash +uv add apache-ossie-sigma +``` + +## CLI usage + +### Sigma → Apache Ossie + +Export a data model's spec from Sigma (e.g. with [sigcli](https://pypi.org/project/sigcli/)): + +```bash +sigcli data-models spec get --params '{"dataModelId": ""}' > data_model.json +ossie-sigma sigma-to-osi -i data_model.json -o semantic_model.yaml +``` + +### Apache Ossie → Sigma + +```bash +ossie-sigma osi-to-sigma -i semantic_model.yaml -o data_model.json +``` + +The output is a Sigma data model spec JSON document suitable for +`sigcli data-models spec create`/`update`. + +### Help + +```bash +ossie-sigma --help +ossie-sigma sigma-to-osi --help +ossie-sigma osi-to-sigma --help +``` + +## Python API + +```python +import json +from pathlib import Path + +from ossie_sigma import SigmaToOSIConverter, OSIToSigmaConverter + +spec = json.loads(Path("data_model.json").read_text()) +result = SigmaToOSIConverter().convert(spec) +for issue in result.issues: + print(f"[warning] {issue.issue_type.value}: {issue.element_name}") +Path("semantic_model.yaml").write_text(result.output.to_osi_yaml()) + +# Ossie -> Sigma +from ossie import OSIDocument +import yaml + +document = OSIDocument.model_validate(yaml.safe_load(Path("semantic_model.yaml").read_text())) +result = OSIToSigmaConverter().convert(document) +Path("data_model.json").write_text(json.dumps(result.output, indent=2)) +``` + +## Mapping overview + +| Sigma concept | Ossie concept | Notes | +|---|---|---| +| Data model (`name`, `description`) | `OSISemanticModel` | `dataModelId`, `folderId`, `documentVersion` preserved in `custom_extensions` | +| Page | *(none)* | Ossie has no page/folder-of-elements concept; page membership is preserved per-dataset in `custom_extensions` so it can be reconstructed on export | +| Element (`kind: table`) | `OSIDataset` | `source` = warehouse path joined with `.`; `connectionId` preserved in `custom_extensions` | +| Element (`kind: control`) | *not modeled* | See [Limitations](#limitations) — the entire native control element is preserved verbatim in a model-level `custom_extensions` entry so `osi-to-sigma` can restore it unchanged | +| Column (`formula`) | `OSIField.expression` | See [Expression translation](#expression-translation) | +| Element `metrics[]` | `OSIMetric` | Promoted to model level (Ossie metrics are not dataset-scoped); the formula is re-qualified with the owning dataset name | +| `relationships[]` (join keys) | `OSIRelationship` | See [Relationship resolution](#relationship-resolution) | +| Column/element/relationship native `id` | *(preserved, not surfaced)* | Stashed in `custom_extensions` (`vendor_name: SIGMA`) so re-export can reuse Sigma's own stable ids rather than minting new ones — see [Stable ids](#stable-ids) | +| Unmapped/unknown column format | `datatype: Opaque` | Only used when Sigma's column format has no portable equivalent; the original Sigma format is preserved in `custom_extensions` | + +### Expression translation + +Sigma column and metric formulas (e.g. `Sum([Orders/Amount])`, `If([Status] = "closed", 1, 0)`) +are parsed by a small recursive-descent parser (`ossie_sigma.sigma_formula`) into an AST, which is +then rendered to ANSI SQL wherever a faithful translation exists (see the module docstring for the +full function/operator coverage table). This is deliberately conservative: a formula that uses a +Sigma function or operator with no portable SQL meaning (e.g. table calculations like `RunningSum`, +which depend on UI-configured partition/order context that is not passed as a formula argument) is +**not** translated. + +Every `OSIExpression` produced by `sigma-to-osi` always carries **both**: + +1. A `SIGMA`-dialect entry with the original Sigma formula text, verbatim — this is what guarantees + lossless round-tripping regardless of how much the ANSI SQL translator understands. +2. An `ANSI_SQL`-dialect entry, present only when the formula translated successfully. + +`osi-to-sigma` prefers the `SIGMA` dialect entry when present (perfect fidelity for anything that +came from Sigma); for expressions authored by another tool (no `SIGMA` dialect entry), it falls +back to translating the `ANSI_SQL` entry back into Sigma formula syntax, using the same function +table in reverse. If neither direction is possible, the expression is preserved as an opaque +Sigma formula-language string comment plus the raw SQL, and the field is flagged in +`ConverterResult.issues` (`ConverterIssueType.EXPRESSION_NOT_TRANSLATABLE`) rather than silently +producing an invalid Sigma formula. + +### Relationship resolution + +Sigma relationships (`element.relationships[]`) join two *elements*, not two *Ossie datasets* +directly, and their `keys[].sourceColumnId`/`targetColumnId` address columns by Sigma's internal +column id — which is **not** the same id space as the modeled column's own `id` when the key +references a column that isn't explicitly redefined by the element (Sigma addresses those via an +`inode-/` reference straight to the underlying warehouse table/column, +bypassing the element's own column list entirely). `sigma_to_osi.py` resolves both addressing +schemes to a modeled column name using the element's own column formulas; when resolution succeeds, +`OSIRelationship.from_columns`/`to_columns` reference the Ossie field name. When it cannot be +resolved (the physical column has no corresponding modeled column, e.g. it was never referenced +anywhere in the element as a column), the physical column name is used verbatim and a converter +issue is recorded. **The raw, unresolved `sourceColumnId`/`targetColumnId` values are always +preserved in the relationship's `custom_extensions`,** so `osi-to-sigma` reconstructs the exact +original join regardless of whether name resolution succeeded — see [Limitations](#limitations). + +### Stable ids + +Sigma column, element, and relationship ids are load-bearing: other parts of a Sigma workbook +(controls, other data models' relationships, materializations) reference them, so an export that +mints new ids for unchanged objects would silently break those references. `sigma_to_osi.py` +therefore never invents an id for anything that already has one — it always preserves the native +Sigma id in that object's `custom_extensions` and `osi-to-sigma` reuses it verbatim. Ids are only +synthesized (as a deterministic `uuid5` of a fixed namespace plus the object's dataset/field path) +for objects that originate purely in Ossie and have never been round-tripped through Sigma before. + +## Limitations + +See [`LIMITATIONS.md`](LIMITATIONS.md) for a full accounting of what this converter does not (yet) +handle faithfully, why, and what the general-purpose OSI-native alternative would be instead of a +Sigma-specific workaround. + +## Development + +```bash +cd converters/sigma +uv sync +uv run pytest +``` diff --git a/converters/sigma/pyproject.toml b/converters/sigma/pyproject.toml new file mode 100644 index 00000000..025ed08c --- /dev/null +++ b/converters/sigma/pyproject.toml @@ -0,0 +1,69 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[dependency-groups] +dev = [ + "pytest>=8.0", +] + +[project] +name = "apache-ossie-sigma" +version = "0.2.0.dev0" +description = "Sigma Computing Data Model <> Apache Ossie converter" +authors = [{ name = "Apache Software Foundation", email = "dev@ossie.apache.org" }] +requires-python = ">=3.11" +readme = "README.md" +license = "Apache-2.0" +keywords = [ + "Apache Ossie", + "Ossie", + "Sigma", + "Sigma Computing" +] +dependencies = [ + "apache-ossie>=0.2.0.dev0", + "PyYAML>=6.0", + "sqlglot>=20.0", +] + +[project.scripts] +ossie-sigma = "ossie_sigma.cli:main" + +[project.urls] +homepage = "https://ossie.apache.org/" +repository = "https://github.com/apache/ossie/" + +[tool.hatch.build.targets.wheel] +packages = ["src/ossie_sigma"] + +[tool.pytest.ini_options] +testpaths = ["tests"] + +[tool.uv] +required-version = ">=0.9.0" +default-groups = [ + "dev" +] + +# apache-ossie is not yet published to PyPI; resolve it from the in-repo +# package for now. Remove this block once apache-ossie is published to PyPI. +[tool.uv.sources] +apache-ossie = { path = "../../python", editable = true } diff --git a/converters/sigma/src/ossie_sigma/__init__.py b/converters/sigma/src/ossie_sigma/__init__.py new file mode 100644 index 00000000..a136360b --- /dev/null +++ b/converters/sigma/src/ossie_sigma/__init__.py @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from ossie_sigma.osi_to_sigma import OSIToSigmaConverter +from ossie_sigma.sigma_to_osi import SigmaToOSIConverter + +__all__ = ["SigmaToOSIConverter", "OSIToSigmaConverter"] diff --git a/converters/sigma/src/ossie_sigma/cli.py b/converters/sigma/src/ossie_sigma/cli.py new file mode 100644 index 00000000..43f96a64 --- /dev/null +++ b/converters/sigma/src/ossie_sigma/cli.py @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""CLI entry point for the ossie-sigma converter. + +Usage: + ossie-sigma sigma-to-osi -i data_model.json -o semantic_model.yaml + ossie-sigma osi-to-sigma -i semantic_model.yaml -o data_model.json +""" + +import argparse +import json +import sys +from pathlib import Path + +import yaml + +from ossie import OSIDocument +from ossie_sigma.osi_to_sigma import OSIToSigmaConverter +from ossie_sigma.sigma_to_osi import SigmaToOSIConverter + + +def _cmd_sigma_to_osi(args: argparse.Namespace) -> None: + input_path = Path(args.input) + output_path = Path(args.output) + + spec = json.loads(input_path.read_text()) + result = SigmaToOSIConverter().convert(spec) + + for issue in result.issues: + print(f"[WARNING] {issue.issue_type.value}: {issue.element_name} — {issue.detail}", file=sys.stderr) + + output_path.write_text(result.output.to_osi_yaml()) + print(f"Written to {output_path}", file=sys.stderr) + + +def _cmd_osi_to_sigma(args: argparse.Namespace) -> None: + input_path = Path(args.input) + output_path = Path(args.output) + + raw = yaml.safe_load(input_path.read_text()) + document = OSIDocument.model_validate(raw) + result = OSIToSigmaConverter().convert(document) + + for issue in result.issues: + print(f"[WARNING] {issue.issue_type.value}: {issue.element_name} — {issue.detail}", file=sys.stderr) + + output_path.write_text(json.dumps(result.output, indent=2)) + print(f"Written to {output_path}", file=sys.stderr) + + +def main() -> None: + parser = argparse.ArgumentParser( + prog="ossie-sigma", + description="Convert between Sigma data model specs and Ossie YAML.", + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + sigma_to_osi = subparsers.add_parser("sigma-to-osi", help="Convert Sigma data model spec JSON → Ossie YAML") + sigma_to_osi.add_argument("-i", "--input", required=True, metavar="FILE", help="Path to Sigma data model spec JSON") + sigma_to_osi.add_argument("-o", "--output", required=True, metavar="FILE", help="Path for output Ossie YAML") + + osi_to_sigma = subparsers.add_parser("osi-to-sigma", help="Convert Ossie YAML → Sigma data model spec JSON") + osi_to_sigma.add_argument("-i", "--input", required=True, metavar="FILE", help="Path to Ossie YAML") + osi_to_sigma.add_argument("-o", "--output", required=True, metavar="FILE", help="Path for output Sigma data model spec JSON") + + args = parser.parse_args() + if args.command == "sigma-to-osi": + _cmd_sigma_to_osi(args) + elif args.command == "osi-to-sigma": + _cmd_osi_to_sigma(args) + + +if __name__ == "__main__": + main() diff --git a/converters/sigma/src/ossie_sigma/converter_issues.py b/converters/sigma/src/ossie_sigma/converter_issues.py new file mode 100644 index 00000000..d0b73be6 --- /dev/null +++ b/converters/sigma/src/ossie_sigma/converter_issues.py @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from dataclasses import dataclass +from enum import Enum +from typing import Generic, List, TypeVar + + +class ConverterIssueType(Enum): + """Identifies the kind of information loss or uncertainty during conversion.""" + + CONTROL_ELEMENT_NOT_MODELED = "CONTROL_ELEMENT_NOT_MODELED" + EXPRESSION_NOT_TRANSLATABLE = "EXPRESSION_NOT_TRANSLATABLE" + RELATIONSHIP_COLUMN_UNRESOLVED = "RELATIONSHIP_COLUMN_UNRESOLVED" + DERIVED_ELEMENT_NOT_MODELED = "DERIVED_ELEMENT_NOT_MODELED" + FILTER_NOT_MODELED = "FILTER_NOT_MODELED" + CROSS_DATASET_METRIC_DROPPED = "CROSS_DATASET_METRIC_DROPPED" + OPAQUE_DATATYPE = "OPAQUE_DATATYPE" + NATIVE_FRAGMENT_SYNTHESIZED = "NATIVE_FRAGMENT_SYNTHESIZED" + + +@dataclass(frozen=True) +class ConverterIssue: + """Records a single instance of information loss or uncertainty during conversion.""" + + issue_type: ConverterIssueType + element_name: str + detail: str = "" + + +T = TypeVar("T") + + +@dataclass(frozen=True) +class ConverterResult(Generic[T]): + """Return value of a converter's convert() method, pairing the output with any conversion issues.""" + + output: T + issues: List[ConverterIssue] diff --git a/converters/sigma/src/ossie_sigma/expression_utils.py b/converters/sigma/src/ossie_sigma/expression_utils.py new file mode 100644 index 00000000..fc91b0d8 --- /dev/null +++ b/converters/sigma/src/ossie_sigma/expression_utils.py @@ -0,0 +1,112 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Shared helpers for building/reading Ossie ``OSIExpression`` values from Sigma formulas.""" + +from __future__ import annotations + +from typing import Optional + +from ossie import OSIDialect, OSIDialectExpression, OSIExpression + +from ossie_sigma.sigma_formula import ( + BinOp, + ColumnRef, + FormulaNode, + FormulaParseError, + FuncCall, + UnaryOp, + parse_formula, + to_ansi_sql, +) + + +def qualify(node: FormulaNode, table_name: str) -> FormulaNode: + """Rewrite every unqualified :class:`ColumnRef` in *node* to reference *table_name*. + + Sigma metric formulas are scoped to their owning element and reference sibling + columns unqualified (e.g. ``Sum([Amount])``); Ossie metrics live at the model + level and may span datasets via relationships, so their expressions must be + fully dataset-qualified. + """ + if isinstance(node, ColumnRef): + return node if node.table is not None else ColumnRef(table_name, node.column) + if isinstance(node, UnaryOp): + return UnaryOp(node.op, qualify(node.operand, table_name)) + if isinstance(node, BinOp): + return BinOp(node.op, qualify(node.left, table_name), qualify(node.right, table_name)) + if isinstance(node, FuncCall): + return FuncCall(node.name, tuple(qualify(a, table_name) for a in node.args)) + return node + + +def build_expression(formula: str, dataset_alias: Optional[str] = None) -> OSIExpression: + """Build an :class:`OSIExpression` from a raw Sigma formula. + + Always includes a ``SIGMA``-dialect entry carrying the original formula text + verbatim (guaranteeing lossless round-tripping), plus an ``ANSI_SQL`` entry when + the formula translates cleanly. + """ + dialects = [OSIDialectExpression(dialect=OSIDialect.SIGMA, expression=formula)] + try: + node = parse_formula(formula) + sql = to_ansi_sql(node, dataset_alias=dataset_alias) + except FormulaParseError: + sql = None + if sql is not None: + dialects.append(OSIDialectExpression(dialect=OSIDialect.ANSI_SQL, expression=sql)) + return OSIExpression(dialects=dialects) + + +def sigma_dialect_text(expression: OSIExpression) -> Optional[str]: + """Return the raw Sigma formula text from *expression*, if a ``SIGMA`` dialect entry exists.""" + for dialect_expr in expression.dialects: + if dialect_expr.dialect == OSIDialect.SIGMA: + return dialect_expr.expression + return None + + +def ansi_sql_text(expression: OSIExpression) -> Optional[str]: + """Return the ``ANSI_SQL`` dialect entry from *expression*, if present.""" + for dialect_expr in expression.dialects: + if dialect_expr.dialect == OSIDialect.ANSI_SQL: + return dialect_expr.expression + return None + + +def infer_single_dataset_qualifier(sql: str, dataset_names: set) -> Optional[str]: + """Return the sole known dataset referenced by *sql*'s qualified columns, if unambiguous. + + Used to place a model-level metric with no preserved Sigma ``element_id`` (i.e. one + authored by, or round-tripped through, a non-Sigma tool) back onto a single Sigma + element — Sigma metrics are always scoped to one element, unlike Ossie metrics, + which may span datasets via relationships. + """ + import sqlglot + from sqlglot import expressions as exp + + try: + tree = sqlglot.parse_one(sql) + except Exception: # noqa: BLE001 + return None + + qualifiers = { + column.parts[0].name + for column in tree.find_all(exp.Column) + if len(column.parts) > 1 and column.parts[0].name in dataset_names + } + return qualifiers.pop() if len(qualifiers) == 1 else None diff --git a/converters/sigma/src/ossie_sigma/osi_to_sigma.py b/converters/sigma/src/ossie_sigma/osi_to_sigma.py new file mode 100644 index 00000000..f0063f54 --- /dev/null +++ b/converters/sigma/src/ossie_sigma/osi_to_sigma.py @@ -0,0 +1,301 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Apache Ossie (OSIDocument) -> Sigma data model spec (JSON).""" + +from __future__ import annotations + +import json +from typing import Any, Optional +from uuid import NAMESPACE_URL, uuid5 + +from ossie import OSICustomExtension, OSIDataset, OSIDocument, OSIField, OSIMetric, OSIRelationship, OSIVendor + +from ossie_sigma.converter_issues import ConverterIssue, ConverterIssueType, ConverterResult +from ossie_sigma.expression_utils import ansi_sql_text, infer_single_dataset_qualifier, sigma_dialect_text +from ossie_sigma.sigma_formula import sql_to_sigma_formula + +_ID_NAMESPACE = uuid5(NAMESPACE_URL, "ossie.apache.org/converters/sigma") + +_DATATYPE_TO_FORMAT = { + "String": "string", + "Integer": "integer", + "Decimal": "number", + "Float": "number", + "Boolean": "boolean", + "Date": "date", + "Time": "time", + "DateTime": "datetime", + "DateTimeTz": "datetime", +} + + +def _stable_id(*parts: str) -> str: + """Deterministic id for an object with no preserved native Sigma id. + + Only used for datasets/fields/relationships that originate purely in Ossie (no + ``SIGMA`` custom_extensions carrying a native id) — anything previously + round-tripped through Sigma keeps its real id instead, since Sigma ids are + referenced by other objects (controls, other data models) that this converter + cannot see or update. + """ + return str(uuid5(_ID_NAMESPACE, "/".join(parts))).replace("-", "") + + +def _sigma_ext(item: Any) -> Optional[dict[str, Any]]: + for ext in item.custom_extensions or []: + if ext.vendor_name == OSIVendor.SIGMA.value: + try: + return json.loads(ext.data) + except json.JSONDecodeError: + return None + return None + + +def _resolve_formula( + expression, dataset_alias: str, element_name: str, issues: list[ConverterIssue] +) -> str: + """Prefer the native Sigma formula text; otherwise best-effort translate ANSI SQL.""" + native = sigma_dialect_text(expression) + if native is not None: + return native + + sql = ansi_sql_text(expression) + if sql is not None: + translated = sql_to_sigma_formula(sql, dataset_alias=dataset_alias) + if translated is not None: + return translated + issues.append( + ConverterIssue( + ConverterIssueType.EXPRESSION_NOT_TRANSLATABLE, + element_name, + f"ANSI SQL expression {sql!r} has no Sigma formula equivalent; " + "field omitted a formula could not be produced.", + ) + ) + return "" + + +class OSIToSigmaConverter: + """Converts an :class:`OSIDocument` into a Sigma data model spec (as plain JSON).""" + + def convert(self, document: OSIDocument) -> ConverterResult[dict[str, Any]]: + issues: list[ConverterIssue] = [] + + if len(document.semantic_model) > 1: + issues.append( + ConverterIssue( + ConverterIssueType.CONTROL_ELEMENT_NOT_MODELED, + "document", + "Sigma data models are single semantic models; only semantic_model[0] " + f"was converted, {len(document.semantic_model) - 1} additional model(s) were dropped.", + ) + ) + model = document.semantic_model[0] + model_ext = _sigma_ext(model) or {} + + spec: dict[str, Any] = {"kind": "data-model", "name": model.name} + for key in ("dataModelId", "folderId", "documentVersion", "latestDocumentVersion", "schemaVersion"): + if key in model_ext: + spec[key] = model_ext[key] + + pages: dict[str, dict[str, Any]] = {} + + def _page(page_id: Optional[str], page_name: Optional[str]) -> dict[str, Any]: + key = page_id or "page-default" + if key not in pages: + pages[key] = {"id": page_id or _stable_id("page", key), "name": page_name or "Page 1", "elements": []} + return pages[key] + + dataset_names = {d.name for d in model.datasets} + dataset_element_id: dict[str, str] = {} + for dataset in model.datasets: + ext = _sigma_ext(dataset) or {} + dataset_element_id[dataset.name] = ext.get("id") or _stable_id("element", dataset.name) + + metrics_by_element: dict[str, list[OSIMetric]] = {} + for metric in model.metrics or []: + ext = _sigma_ext(metric) or {} + element_id = ext.get("element_id") + if element_id is None: + sql = ansi_sql_text(metric.expression) + owning_dataset = infer_single_dataset_qualifier(sql, dataset_names) if sql else None + element_id = dataset_element_id.get(owning_dataset) if owning_dataset else None + if element_id is None: + issues.append( + ConverterIssue( + ConverterIssueType.CROSS_DATASET_METRIC_DROPPED, + metric.name, + "Sigma metrics are scoped to a single element; this Ossie metric's " + "expression does not unambiguously reference exactly one dataset " + "(it may span datasets via a relationship, e.g. a ratio metric), so " + "it has no faithful Sigma representation and was dropped.", + ) + ) + continue + metrics_by_element.setdefault(element_id, []).append(metric) + + relationships_by_element: dict[str, list[OSIRelationship]] = {} + for rel in model.relationships or []: + ext = _sigma_ext(rel) or {} + element_id = ext.get("element_id") or dataset_element_id.get(rel.from_dataset, "") + relationships_by_element.setdefault(element_id, []).append(rel) + + for dataset in model.datasets: + element = self._build_element( + dataset, dataset_element_id, metrics_by_element, relationships_by_element, issues + ) + ext = _sigma_ext(dataset) or {} + page = _page(ext.get("page_id"), ext.get("page_name")) + page["elements"].append(element) + + for entry in model_ext.get("non_table_elements", []): + page = _page(entry.get("page_id"), entry.get("page_name")) + page["elements"].append(entry["element"]) + + spec["pages"] = list(pages.values()) or [{"id": _stable_id("page", "default"), "name": "Page 1", "elements": []}] + + return ConverterResult(output=spec, issues=issues) + + def _build_element( + self, + dataset: OSIDataset, + dataset_element_id: dict[str, str], + metrics_by_element: dict[str, list[OSIMetric]], + relationships_by_element: dict[str, list[OSIRelationship]], + issues: list[ConverterIssue], + ) -> dict[str, Any]: + ext = _sigma_ext(dataset) or {} + element_id = dataset_element_id[dataset.name] + + if "source_kind" in ext: + source: dict[str, Any] = {"kind": ext["source_kind"]} + if ext["source_kind"] == "warehouse-table": + source["path"] = dataset.source.split(".") + if "connectionId" in ext: + source["connectionId"] = ext["connectionId"] + if "source_element_id" in ext: + source["elementId"] = ext["source_element_id"] + else: + source = {"kind": "warehouse-table", "path": dataset.source.split(".")} + + field_ids: dict[str, str] = {} + columns = [] + for field in dataset.fields or []: + field_ext = _sigma_ext(field) or {} + col_id = field_ext.get("id") or _stable_id("column", dataset.name, field.name) + field_ids[field.name] = col_id + columns.append(self._build_column(dataset, field, col_id, field_ext, issues)) + + element: dict[str, Any] = { + "id": element_id, + "kind": "table", + "name": dataset.name, + "source": source, + "columns": columns, + } + if dataset.description: + element["description"] = dataset.description + if "folders" in ext: + element["folders"] = ext["folders"] + if "order" in ext: + element["order"] = ext["order"] + if "filters" in ext: + element["filters"] = ext["filters"] + + metrics = metrics_by_element.get(element_id, []) + if metrics: + element["metrics"] = [self._build_metric(m, dataset.name, issues) for m in metrics] + + relationships = relationships_by_element.get(element_id, []) + if relationships: + element["relationships"] = [ + self._build_relationship(r, dataset_element_id, field_ids) for r in relationships + ] + + return element + + def _build_column( + self, + dataset: OSIDataset, + field: OSIField, + col_id: str, + field_ext: dict[str, Any], + issues: list[ConverterIssue], + ) -> dict[str, Any]: + formula = _resolve_formula(field.expression, dataset.name, f"{dataset.name}.{field.name}", issues) + if not formula: + formula = f"[{dataset.name}/{field.name}]" + + column: dict[str, Any] = {"id": col_id, "formula": formula} + needs_name = f"/{field.name}]" not in formula and f"[{field.name}]" != formula + if field.name and (needs_name or field_ext.get("explicit_name")): + column["name"] = field.name + if field.description: + column["description"] = field.description + + if field.datatype == "Opaque" and "format" in field_ext: + column["format"] = field_ext["format"] + elif field.datatype and field.datatype in _DATATYPE_TO_FORMAT: + column["format"] = {"kind": _DATATYPE_TO_FORMAT[field.datatype]} + elif field.datatype == "Opaque": + issues.append( + ConverterIssue( + ConverterIssueType.OPAQUE_DATATYPE, + f"{dataset.name}.{field.name}", + "Field has an Opaque datatype with no preserved native Sigma format; " + "no format was emitted.", + ) + ) + return column + + def _build_metric(self, metric: OSIMetric, dataset_name: str, issues: list[ConverterIssue]) -> dict[str, Any]: + ext = _sigma_ext(metric) or {} + formula = _resolve_formula(metric.expression, dataset_name, f"{dataset_name}.{metric.name}", issues) + result = {"id": ext.get("id") or _stable_id("metric", dataset_name, metric.name), "formula": formula} + if metric.name: + result["name"] = metric.name + return result + + def _build_relationship( + self, + rel: OSIRelationship, + dataset_element_id: dict[str, str], + field_ids: dict[str, str], + ) -> dict[str, Any]: + ext = _sigma_ext(rel) or {} + target_element_id = dataset_element_id.get(rel.to, rel.to) + result: dict[str, Any] = { + "id": ext.get("id") or _stable_id("relationship", rel.name), + "name": rel.name, + "targetElementId": target_element_id, + } + if ext.get("description"): + result["description"] = ext["description"] + + raw_keys = ext.get("raw_keys") + if raw_keys is not None: + result["keys"] = raw_keys + else: + result["keys"] = [ + { + "sourceColumnId": field_ids.get(from_col, from_col), + "targetColumnId": field_ids.get(to_col, to_col), + } + for from_col, to_col in zip(rel.from_columns, rel.to_columns) + ] + return result diff --git a/converters/sigma/src/ossie_sigma/sigma_formula.py b/converters/sigma/src/ossie_sigma/sigma_formula.py new file mode 100644 index 00000000..b8850643 --- /dev/null +++ b/converters/sigma/src/ossie_sigma/sigma_formula.py @@ -0,0 +1,652 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""A parser and ANSI SQL renderer for Sigma's spreadsheet-style formula language. + +Sigma data model column and metric formulas look like ``Sum([Orders/Amount])`` or +``If([Status] = "closed", 1, 0)``. This module implements a small recursive-descent +parser that turns such a formula into an AST (:class:`FormulaNode`), and a renderer +that turns that AST into ANSI SQL where a faithful translation exists. + +Design principle (matching the rest of the Ossie converter ecosystem, e.g. the GSF +converter's SQL-dialect handling): never fail. A formula that cannot be parsed, or +that uses a function with no portable SQL equivalent, is simply not translatable — +callers fall back to carrying the original Sigma formula text verbatim (see +``ossie_sigma.sigma_to_osi``), rather than raising or emitting an approximate/lossy +translation. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Optional, Union + + +class FormulaParseError(Exception): + """Raised internally when a formula cannot be parsed; callers should catch it.""" + + +# -------------------------------------------------------------------------- +# AST +# -------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class ColumnRef: + """A ``[Column]`` or ``[Table/Column]`` reference.""" + + table: Optional[str] + column: str + + +@dataclass(frozen=True) +class Literal: + """A string, number, or boolean literal.""" + + value: Union[str, float, int, bool] + kind: str # "string" | "number" | "boolean" + + +@dataclass(frozen=True) +class FuncCall: + """A function call, e.g. ``Sum(x)`` or nested ``If(IsNull([A]), 0, Sum([B]))``.""" + + name: str + args: tuple["FormulaNode", ...] + + +@dataclass(frozen=True) +class BinOp: + """A binary operator expression, e.g. ``[A] + [B]`` or ``[A] & "x"``.""" + + op: str + left: "FormulaNode" + right: "FormulaNode" + + +@dataclass(frozen=True) +class UnaryOp: + """A unary operator expression, e.g. ``-[A]`` or ``NOT [A]``.""" + + op: str + operand: "FormulaNode" + + +FormulaNode = Union[ColumnRef, Literal, FuncCall, BinOp, UnaryOp] + + +# -------------------------------------------------------------------------- +# Tokenizer +# -------------------------------------------------------------------------- + +_TOKEN_RE = re.compile( + r""" + (?P\s+) + |(?P\[[^\[\]]+\]) + |(?P"(?:[^"]|"")*") + |(?P\d+\.\d+|\d+) + |(?P<=)|(?P>=)|(?P<>) + |(?P\()|(?P\))|(?P,) + |(?P[+\-*/&=<>^]) + |(?P[A-Za-z_][A-Za-z0-9_]*) + """, + re.VERBOSE, +) + +_KEYWORDS = {"and", "or", "not", "true", "false"} + + +@dataclass(frozen=True) +class _Token: + kind: str + text: str + + +def _tokenize(formula: str) -> list[_Token]: + pos = 0 + tokens: list[_Token] = [] + while pos < len(formula): + match = _TOKEN_RE.match(formula, pos) + if not match or match.end() == pos: + raise FormulaParseError(f"Unrecognized character at position {pos}: {formula[pos:pos + 20]!r}") + pos = match.end() + kind = match.lastgroup + text = match.group() + if kind == "ws": + continue + if kind == "ident" and text.lower() in _KEYWORDS: + kind = text.lower() + tokens.append(_Token(kind, text)) + return tokens + + +# -------------------------------------------------------------------------- +# Recursive-descent / precedence-climbing parser +# +# Precedence (low to high): or -> and -> not -> comparison -> concat (&) -> +# additive (+ -) -> multiplicative (* /) -> power (^) -> unary (- +) -> primary +# -------------------------------------------------------------------------- + + +class _Parser: + def __init__(self, tokens: list[_Token]) -> None: + self._tokens = tokens + self._pos = 0 + + def _peek(self) -> Optional[_Token]: + return self._tokens[self._pos] if self._pos < len(self._tokens) else None + + def _advance(self) -> _Token: + token = self._tokens[self._pos] + self._pos += 1 + return token + + def _expect(self, kind: str) -> _Token: + token = self._peek() + if token is None or token.kind != kind: + raise FormulaParseError(f"Expected {kind!r} at position {self._pos}, got {token!r}") + return self._advance() + + def parse(self) -> FormulaNode: + node = self._parse_or() + if self._peek() is not None: + raise FormulaParseError(f"Unexpected trailing token {self._peek()!r}") + return node + + def _parse_or(self) -> FormulaNode: + node = self._parse_and() + while (tok := self._peek()) and tok.kind == "or": + self._advance() + node = BinOp("OR", node, self._parse_and()) + return node + + def _parse_and(self) -> FormulaNode: + node = self._parse_not() + while (tok := self._peek()) and tok.kind == "and": + self._advance() + node = BinOp("AND", node, self._parse_not()) + return node + + def _parse_not(self) -> FormulaNode: + if (tok := self._peek()) and tok.kind == "not": + self._advance() + return UnaryOp("NOT", self._parse_not()) + return self._parse_comparison() + + _COMPARISON_OPS = {"=", "<>", "<", "<=", ">", ">="} + + def _parse_comparison(self) -> FormulaNode: + node = self._parse_concat() + while (tok := self._peek()) and self._op_text(tok) in self._COMPARISON_OPS: + op = self._op_text(self._advance()) + node = BinOp(op, node, self._parse_concat()) + return node + + @staticmethod + def _op_text(tok: _Token) -> Optional[str]: + if tok.kind in ("op", "le", "ge", "ne"): + return tok.text + return None + + def _parse_concat(self) -> FormulaNode: + node = self._parse_additive() + while (tok := self._peek()) and tok.kind == "op" and tok.text == "&": + self._advance() + node = BinOp("&", node, self._parse_additive()) + return node + + def _parse_additive(self) -> FormulaNode: + node = self._parse_multiplicative() + while (tok := self._peek()) and tok.kind == "op" and tok.text in ("+", "-"): + op = self._advance().text + node = BinOp(op, node, self._parse_multiplicative()) + return node + + def _parse_multiplicative(self) -> FormulaNode: + node = self._parse_power() + while (tok := self._peek()) and tok.kind == "op" and tok.text in ("*", "/"): + op = self._advance().text + node = BinOp(op, node, self._parse_power()) + return node + + def _parse_power(self) -> FormulaNode: + node = self._parse_unary() + if (tok := self._peek()) and tok.kind == "op" and tok.text == "^": + self._advance() + return BinOp("^", node, self._parse_power()) + return node + + def _parse_unary(self) -> FormulaNode: + if (tok := self._peek()) and tok.kind == "op" and tok.text in ("-", "+"): + op = self._advance().text + return UnaryOp(op, self._parse_unary()) + return self._parse_primary() + + def _parse_primary(self) -> FormulaNode: + tok = self._peek() + if tok is None: + raise FormulaParseError("Unexpected end of formula") + + if tok.kind == "column": + self._advance() + inner = tok.text[1:-1] + if "/" in inner: + table, column = inner.split("/", 1) + return ColumnRef(table, column) + return ColumnRef(None, inner) + + if tok.kind == "string": + self._advance() + return Literal(tok.text[1:-1].replace('""', '"'), "string") + + if tok.kind == "number": + self._advance() + value: Union[int, float] = float(tok.text) if "." in tok.text else int(tok.text) + return Literal(value, "number") + + if tok.kind in ("true", "false"): + self._advance() + return Literal(tok.kind == "true", "boolean") + + if tok.kind == "lparen": + self._advance() + node = self._parse_or() + self._expect("rparen") + return node + + if tok.kind == "ident": + name = self._advance().text + self._expect("lparen") + args: list[FormulaNode] = [] + if not (self._peek() and self._peek().kind == "rparen"): + args.append(self._parse_or()) + while self._peek() and self._peek().kind == "comma": + self._advance() + args.append(self._parse_or()) + self._expect("rparen") + return FuncCall(name, tuple(args)) + + raise FormulaParseError(f"Unexpected token {tok!r}") + + +def parse_formula(formula: str) -> FormulaNode: + """Parse a Sigma formula string into a :class:`FormulaNode` AST. + + Raises :class:`FormulaParseError` on any formula this parser does not understand + (e.g. functions/operators outside Sigma's grammar, or malformed input). Callers + should treat that as "not translatable" rather than a hard failure. + """ + tokens = _tokenize(formula.strip()) + if not tokens: + raise FormulaParseError("Empty formula") + return _Parser(tokens).parse() + + +def is_plain_column_ref(formula: str) -> Optional[ColumnRef]: + """Return the :class:`ColumnRef` if *formula* is exactly a single bracket reference.""" + try: + node = parse_formula(formula) + except FormulaParseError: + return None + return node if isinstance(node, ColumnRef) else None + + +# -------------------------------------------------------------------------- +# ANSI SQL rendering +# -------------------------------------------------------------------------- + + +def _quote_ident(name: str) -> str: + return '"' + name.replace('"', '""') + '"' + + +def _sql_string_literal(value: str) -> str: + return "'" + value.replace("'", "''") + "'" + + +# Functions that map 1:1 onto an ANSI SQL function/aggregate of the same arity, +# keyed by lowercase Sigma name -> ANSI SQL name. +_DIRECT_FUNCTIONS = { + "sum": "SUM", + "avg": "AVG", + "average": "AVG", + "min": "MIN", + "max": "MAX", + "count": "COUNT", + "upper": "UPPER", + "lower": "LOWER", + "trim": "TRIM", + "abs": "ABS", + "round": "ROUND", + "ceiling": "CEIL", + "floor": "FLOOR", + "sqrt": "SQRT", + "length": "CHAR_LENGTH", + "power": "POWER", + "mod": "MOD", + "coalesce": "COALESCE", + "lower_": "LOWER", +} + +_EXTRACT_PARTS = { + "year": "YEAR", + "month": "MONTH", + "day": "DAY", + "hour": "HOUR", + "minute": "MINUTE", + "second": "SECOND", + "quarter": "QUARTER", + "week": "WEEK", + "dayofweek": "DOW", +} + + +class _NotTranslatable(Exception): + pass + + +def _render_node(node: FormulaNode, dataset_alias: Optional[str]) -> str: + if isinstance(node, ColumnRef): + if node.table is not None and node.table != dataset_alias: + return f"{_quote_ident(node.table)}.{_quote_ident(node.column)}" + return _quote_ident(node.column) + + if isinstance(node, Literal): + if node.kind == "string": + return _sql_string_literal(str(node.value)) + if node.kind == "boolean": + return "TRUE" if node.value else "FALSE" + return repr(node.value) + + if isinstance(node, UnaryOp): + inner = _render_node(node.operand, dataset_alias) + if node.op == "NOT": + return f"NOT ({inner})" + return f"{node.op}({inner})" + + if isinstance(node, BinOp): + left = _render_node(node.left, dataset_alias) + right = _render_node(node.right, dataset_alias) + if node.op == "&": + return f"({left} || {right})" + if node.op == "^": + return f"POWER({left}, {right})" + return f"({left} {node.op} {right})" + + if isinstance(node, FuncCall): + return _render_call(node, dataset_alias) + + raise _NotTranslatable(f"Unknown node type: {node!r}") + + +def _args(node: FuncCall, dataset_alias: Optional[str]) -> list[str]: + return [_render_node(a, dataset_alias) for a in node.args] + + +def _render_string_slice(name: str, args: tuple[FormulaNode, ...], dataset_alias: Optional[str]) -> str: + text = _render_node(args[0], dataset_alias) + if name == "left" and len(args) == 2: + n = _render_node(args[1], dataset_alias) + return f"SUBSTRING({text} FROM 1 FOR {n})" + if name == "right" and len(args) == 2: + n = _render_node(args[1], dataset_alias) + return f"SUBSTRING({text} FROM CHAR_LENGTH({text}) - ({n}) + 1 FOR {n})" + if name in ("mid", "substring") and len(args) == 3: + start = _render_node(args[1], dataset_alias) + length = _render_node(args[2], dataset_alias) + return f"SUBSTRING({text} FROM {start} FOR {length})" + if name in ("mid", "substring") and len(args) == 2: + start = _render_node(args[1], dataset_alias) + return f"SUBSTRING({text} FROM {start})" + raise _NotTranslatable(f"Unsupported arity for {name}: {len(args)} args") + + +def _render_call(node: FuncCall, dataset_alias: Optional[str]) -> str: + name = node.name.lower() + args = node.args + + if name == "countdistinct" and len(args) == 1: + return f"COUNT(DISTINCT {_render_node(args[0], dataset_alias)})" + if name == "median" and len(args) == 1: + return f"PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY {_render_node(args[0], dataset_alias)})" + if name == "percentile" and len(args) == 2: + return ( + f"PERCENTILE_CONT({_render_node(args[1], dataset_alias)}) " + f"WITHIN GROUP (ORDER BY {_render_node(args[0], dataset_alias)})" + ) + if name in ("variance", "var") and len(args) == 1: + return f"VAR_SAMP({_render_node(args[0], dataset_alias)})" + if name in ("stddev", "standarddeviation") and len(args) == 1: + return f"STDDEV_SAMP({_render_node(args[0], dataset_alias)})" + + if name == "if" and len(args) == 3: + cond, then, otherwise = (_render_node(a, dataset_alias) for a in args) + return f"CASE WHEN {cond} THEN {then} ELSE {otherwise} END" + if name == "ifnull" and len(args) == 2: + return f"COALESCE({_render_node(args[0], dataset_alias)}, {_render_node(args[1], dataset_alias)})" + if name == "isnull" and len(args) == 1: + return f"({_render_node(args[0], dataset_alias)} IS NULL)" + if name == "isnotnull" and len(args) == 1: + return f"({_render_node(args[0], dataset_alias)} IS NOT NULL)" + + if name == "sumif" and len(args) == 2: + cond, expr = (_render_node(a, dataset_alias) for a in args) + return f"SUM(CASE WHEN {cond} THEN {expr} ELSE 0 END)" + if name == "countif" and len(args) == 1: + cond = _render_node(args[0], dataset_alias) + return f"COUNT(CASE WHEN {cond} THEN 1 END)" + if name == "countdistinctif" and len(args) == 2: + cond, expr = (_render_node(a, dataset_alias) for a in args) + return f"COUNT(DISTINCT CASE WHEN {cond} THEN {expr} END)" + if name == "averageif" and len(args) == 2: + cond, expr = (_render_node(a, dataset_alias) for a in args) + return f"AVG(CASE WHEN {cond} THEN {expr} END)" + + if name in ("left", "right", "mid", "substring") and args: + return _render_string_slice(name, args, dataset_alias) + + if name == "concat": + return " || ".join(_args(node, dataset_alias)) + + if name == "contains" and len(args) == 2: + text, needle = _args(node, dataset_alias) + return f"({text} LIKE '%' || {needle} || '%')" + if name == "startswith" and len(args) == 2: + text, needle = _args(node, dataset_alias) + return f"({text} LIKE {needle} || '%')" + if name == "endswith" and len(args) == 2: + text, needle = _args(node, dataset_alias) + return f"({text} LIKE '%' || {needle})" + if name == "replace" and len(args) == 3: + return f"REPLACE({', '.join(_args(node, dataset_alias))})" + + if name == "today" and len(args) == 0: + return "CURRENT_DATE" + if name == "now" and len(args) == 0: + return "CURRENT_TIMESTAMP" + if name in _EXTRACT_PARTS and len(args) == 1: + return f"EXTRACT({_EXTRACT_PARTS[name]} FROM {_render_node(args[0], dataset_alias)})" + + if name in _DIRECT_FUNCTIONS: + return f"{_DIRECT_FUNCTIONS[name]}({', '.join(_args(node, dataset_alias))})" + + raise _NotTranslatable(f"No ANSI SQL mapping for Sigma function {node.name!r}") + + +_REVERSE_AGG_FUNCTIONS = { + "SUM": "Sum", + "AVG": "Avg", + "MIN": "Min", + "MAX": "Max", + "UPPER": "Upper", + "LOWER": "Lower", + "TRIM": "Trim", + "ABS": "Abs", + "ROUND": "Round", + "CEIL": "Ceiling", + "FLOOR": "Floor", + "SQRT": "Sqrt", + "COALESCE": "IfNull", + "POWER": "Power", +} + +_REVERSE_EXTRACT_PARTS = {v: k.capitalize() for k, v in _EXTRACT_PARTS.items()} + + +def sql_to_sigma_formula(sql: str, dataset_alias: Optional[str] = None) -> Optional[str]: + """Best-effort reverse translation of an ANSI SQL expression into Sigma formula syntax. + + Used only for fields/metrics that did not originate in Sigma (i.e. carry no + ``SIGMA``-dialect expression to reuse verbatim). Returns ``None`` if *sql* cannot + be parsed, or uses a SQL construct with no Sigma formula-language equivalent — + callers should treat that as "not translatable", not fail the conversion. + """ + import sqlglot + from sqlglot import expressions as exp + + try: + tree = sqlglot.parse_one(sql) + except Exception: # noqa: BLE001 - sqlglot raises several internal error types + return None + + try: + return _render_sql_node(tree, dataset_alias) + except _NotTranslatable: + return None + + +def _render_sql_node(node: "object", dataset_alias: Optional[str]) -> str: # noqa: ANN001 + import sqlglot.expressions as exp + + if isinstance(node, exp.Column): + parts = [p.name for p in node.parts] + if len(parts) == 2: + table, column = parts + if table == dataset_alias: + return f"[{column}]" + return f"[{table}/{column}]" + return f"[{parts[-1]}]" + + if isinstance(node, exp.Paren): + return _render_sql_node(node.this, dataset_alias) + + if isinstance(node, exp.Literal): + if node.is_string: + return '"' + node.this.replace('"', '""') + '"' + return node.this + + if isinstance(node, exp.Boolean): + return "TRUE" if node.this else "FALSE" + + if isinstance(node, exp.Count): + inner = node.this + if isinstance(inner, exp.Distinct) and len(inner.expressions) == 1: + return f"CountDistinct({_render_sql_node(inner.expressions[0], dataset_alias)})" + if isinstance(inner, exp.Star): + raise _NotTranslatable("COUNT(*) has no unambiguous Sigma column-based equivalent") + return f"Count({_render_sql_node(inner, dataset_alias)})" + + if isinstance(node, exp.Div): + left = _render_sql_node(node.this, dataset_alias) + right = _render_sql_node(node.expression, dataset_alias) + return f"({left} / {right})" + if isinstance(node, exp.Mul): + return f"({_render_sql_node(node.this, dataset_alias)} * {_render_sql_node(node.expression, dataset_alias)})" + if isinstance(node, exp.Add): + return f"({_render_sql_node(node.this, dataset_alias)} + {_render_sql_node(node.expression, dataset_alias)})" + if isinstance(node, exp.Sub): + return f"({_render_sql_node(node.this, dataset_alias)} - {_render_sql_node(node.expression, dataset_alias)})" + if isinstance(node, exp.DPipe) or isinstance(node, exp.Concat): + parts = node.flatten() if hasattr(node, "flatten") else [node.this, node.expression] + return " & ".join(_render_sql_node(p, dataset_alias) for p in parts) + + if isinstance(node, exp.EQ): + return f"({_render_sql_node(node.this, dataset_alias)} = {_render_sql_node(node.expression, dataset_alias)})" + if isinstance(node, exp.NEQ): + return f"({_render_sql_node(node.this, dataset_alias)} <> {_render_sql_node(node.expression, dataset_alias)})" + if isinstance(node, exp.GT): + return f"({_render_sql_node(node.this, dataset_alias)} > {_render_sql_node(node.expression, dataset_alias)})" + if isinstance(node, exp.GTE): + return f"({_render_sql_node(node.this, dataset_alias)} >= {_render_sql_node(node.expression, dataset_alias)})" + if isinstance(node, exp.LT): + return f"({_render_sql_node(node.this, dataset_alias)} < {_render_sql_node(node.expression, dataset_alias)})" + if isinstance(node, exp.LTE): + return f"({_render_sql_node(node.this, dataset_alias)} <= {_render_sql_node(node.expression, dataset_alias)})" + if isinstance(node, exp.And): + return f"({_render_sql_node(node.this, dataset_alias)} AND {_render_sql_node(node.expression, dataset_alias)})" + if isinstance(node, exp.Or): + return f"({_render_sql_node(node.this, dataset_alias)} OR {_render_sql_node(node.expression, dataset_alias)})" + if isinstance(node, exp.Not): + return f"NOT ({_render_sql_node(node.this, dataset_alias)})" + + if isinstance(node, exp.Is): + inner = _render_sql_node(node.this, dataset_alias) + if isinstance(node.expression, exp.Null): + return f"IsNull({inner})" + raise _NotTranslatable("IS has no Sigma equivalent") + + if isinstance(node, exp.Case): + ifs = node.args.get("ifs", []) + default = node.args.get("default") + if len(ifs) == 1 and default is not None: + cond = _render_sql_node(ifs[0].this, dataset_alias) + then = _render_sql_node(ifs[0].args["true"], dataset_alias) + otherwise = _render_sql_node(default, dataset_alias) + return f"If({cond}, {then}, {otherwise})" + raise _NotTranslatable("Multi-branch CASE has no single Sigma If() equivalent") + + if isinstance(node, exp.Coalesce) and len(node.expressions) == 1: + return f"IfNull({_render_sql_node(node.this, dataset_alias)}, {_render_sql_node(node.expressions[0], dataset_alias)})" + + if isinstance(node, exp.CurrentDate): + return "Today()" + if isinstance(node, exp.CurrentTimestamp): + return "Now()" + + if isinstance(node, exp.Extract): + part = node.this.name.upper() if hasattr(node.this, "name") else str(node.this).upper() + sigma_part = _REVERSE_EXTRACT_PARTS.get(part) + if sigma_part is not None: + return f"{sigma_part}({_render_sql_node(node.expression, dataset_alias)})" + raise _NotTranslatable(f"Unsupported EXTRACT part: {part}") + + func_name = node.__class__.__name__.upper() + if func_name in _REVERSE_AGG_FUNCTIONS and hasattr(node, "this"): + sigma_name = _REVERSE_AGG_FUNCTIONS[func_name] + args = [node.this] + list(getattr(node, "expressions", []) or []) + rendered = [_render_sql_node(a, dataset_alias) for a in args if a is not None] + return f"{sigma_name}({', '.join(rendered)})" + + raise _NotTranslatable(f"No Sigma formula equivalent for SQL node {node.__class__.__name__}") + + +def to_ansi_sql(node: FormulaNode, dataset_alias: Optional[str] = None) -> Optional[str]: + """Render *node* as ANSI SQL, or return ``None`` if it uses a construct with no + portable SQL equivalent (e.g. a table-calculation function like ``RunningSum`` + that depends on UI-configured partition/order context Sigma does not pass as + formula arguments). + + *dataset_alias* is the name of the dataset the expression is being rendered for; + column references qualified with that same table name are rendered unqualified + (since the expression lives inside that dataset's own scope), while references + to any other table are rendered as ``"other_table"."column"``. + """ + try: + return _render_node(node, dataset_alias) + except _NotTranslatable: + return None diff --git a/converters/sigma/src/ossie_sigma/sigma_to_osi.py b/converters/sigma/src/ossie_sigma/sigma_to_osi.py new file mode 100644 index 00000000..bb931e68 --- /dev/null +++ b/converters/sigma/src/ossie_sigma/sigma_to_osi.py @@ -0,0 +1,353 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Sigma data model spec (JSON) -> Apache Ossie (OSIDocument).""" + +from __future__ import annotations + +import json +from typing import Any, Optional + +from ossie import ( + OSICustomExtension, + OSIDataset, + OSIDialect, + OSIDialectExpression, + OSIDocument, + OSIExpression, + OSIField, + OSIMetric, + OSIRelationship, + OSISemanticModel, + OSIVendor, +) + +from ossie_sigma.converter_issues import ConverterIssue, ConverterIssueType, ConverterResult +from ossie_sigma.expression_utils import build_expression, qualify +from ossie_sigma.sigma_formula import FormulaParseError, is_plain_column_ref, parse_formula, to_ansi_sql + +_MODEL_LEVEL_SPEC_KEYS = ( + "dataModelId", + "folderId", + "documentVersion", + "latestDocumentVersion", + "schemaVersion", + "kind", + "createdAt", + "createdBy", + "updatedAt", + "updatedBy", + "ownerId", + "url", +) + +# Sigma column `format.kind` -> portable Ossie datatype. Anything not listed here has +# no portable equivalent and becomes `Opaque` (with the original format preserved). +_FORMAT_TO_DATATYPE = { + "string": "String", + "text": "String", + "integer": "Integer", + "number": "Decimal", + "currency": "Decimal", + "percent": "Decimal", + "boolean": "Boolean", + "date": "Date", + "datetime": "DateTime", + "time": "Time", +} + + +def _vendor_ext(data: dict[str, Any]) -> OSICustomExtension: + return OSICustomExtension(vendor_name=OSIVendor.SIGMA.value, data=json.dumps(data, sort_keys=True)) + + +def _column_display_name(column: dict[str, Any]) -> str: + """The Ossie field name for a Sigma column: its explicit `name`, else derived from formula.""" + if column.get("name"): + return column["name"] + formula = column.get("formula", "") + ref = is_plain_column_ref(formula) + if ref is not None: + return ref.column + return column["id"] + + +def _folder_for_column(element: dict[str, Any], column_id: str) -> Optional[dict[str, Any]]: + for folder in element.get("folders") or []: + if column_id in (folder.get("items") or []): + return folder + return None + + +class _ElementIndex: + """Resolves a Sigma relationship key (model column id, or a raw + ``inode-/`` warehouse-column reference) to the Ossie + field name of a table element.""" + + def __init__(self, element: dict[str, Any]) -> None: + self.element = element + self.columns_by_id: dict[str, dict[str, Any]] = {c["id"]: c for c in element.get("columns") or []} + self.physical_by_upper: dict[str, str] = {} + for column in element.get("columns") or []: + ref = is_plain_column_ref(column.get("formula", "")) + if ref is not None: + self.physical_by_upper[ref.column.upper()] = _column_display_name(column) + + def resolve(self, column_ref_id: str) -> tuple[str, bool]: + """Return ``(ossie_field_name, resolved)``.""" + if column_ref_id in self.columns_by_id: + return _column_display_name(self.columns_by_id[column_ref_id]), True + if column_ref_id.startswith("inode-"): + physical_name = column_ref_id.rsplit("/", 1)[-1] + resolved = self.physical_by_upper.get(physical_name.upper()) + if resolved is not None: + return resolved, True + return physical_name, False + return column_ref_id, False + + +class SigmaToOSIConverter: + """Converts a Sigma data model spec (as parsed JSON) into an :class:`OSIDocument`.""" + + def convert(self, spec: dict[str, Any]) -> ConverterResult[OSIDocument]: + issues: list[ConverterIssue] = [] + + elements: list[tuple[dict[str, Any], dict[str, Any]]] = [] # (page, element) + for page in spec.get("pages") or []: + for element in page.get("elements") or []: + elements.append((page, element)) + + table_elements = [(p, e) for p, e in elements if e.get("kind") == "table"] + other_elements = [(p, e) for p, e in elements if e.get("kind") != "table"] + + element_by_id = {e["id"]: e for _, e in table_elements} + index_by_id = {e["id"]: _ElementIndex(e) for _, e in table_elements} + + datasets: list[OSIDataset] = [] + relationships: list[OSIRelationship] = [] + metrics: list[OSIMetric] = [] + + for page, element in table_elements: + dataset_name = element.get("name", element["id"]) + source = element.get("source") or {} + + if source.get("kind") == "warehouse-table": + source_str = ".".join(source.get("path") or []) + else: + source_str = f"element:{source.get('elementId', element['id'])}" + issues.append( + ConverterIssue( + ConverterIssueType.DERIVED_ELEMENT_NOT_MODELED, + dataset_name, + "Element is derived from another element rather than a warehouse table; " + "Ossie has no first-class 'derived dataset' concept, so the parent " + "reference is carried in custom_extensions only.", + ) + ) + + fields: list[OSIField] = [] + for column in element.get("columns") or []: + formula = column.get("formula", "") + field_name = _column_display_name(column) + expression = build_expression(formula, dataset_alias=dataset_name) + if not any(d.dialect == OSIDialect.ANSI_SQL for d in expression.dialects): + issues.append( + ConverterIssue( + ConverterIssueType.EXPRESSION_NOT_TRANSLATABLE, + f"{dataset_name}.{field_name}", + f"Formula {formula!r} has no ANSI SQL equivalent; preserved as SIGMA-dialect text only.", + ) + ) + + datatype = None + fmt = column.get("format") or {} + fmt_kind = fmt.get("kind") + ext_data: dict[str, Any] = {"id": column["id"]} + if column.get("name"): + ext_data["explicit_name"] = True + folder = _folder_for_column(element, column["id"]) + if folder is not None: + ext_data["folder_id"] = folder["id"] + if fmt_kind: + mapped = _FORMAT_TO_DATATYPE.get(fmt_kind) + if mapped is None: + datatype = "Opaque" + ext_data["format"] = fmt + issues.append( + ConverterIssue( + ConverterIssueType.OPAQUE_DATATYPE, + f"{dataset_name}.{field_name}", + f"Sigma column format {fmt_kind!r} has no portable Ossie datatype.", + ) + ) + else: + datatype = mapped + + fields.append( + OSIField( + name=field_name, + expression=expression, + description=column.get("description"), + datatype=datatype, + custom_extensions=[_vendor_ext(ext_data)], + ) + ) + + dataset_ext: dict[str, Any] = { + "id": element["id"], + "page_id": page.get("id"), + "page_name": page.get("name"), + } + if source.get("connectionId"): + dataset_ext["connectionId"] = source["connectionId"] + if source.get("kind"): + dataset_ext["source_kind"] = source["kind"] + if source.get("elementId"): + dataset_ext["source_element_id"] = source["elementId"] + if element.get("folders"): + dataset_ext["folders"] = element["folders"] + if element.get("order"): + dataset_ext["order"] = element["order"] + if element.get("filters"): + dataset_ext["filters"] = element["filters"] + issues.append( + ConverterIssue( + ConverterIssueType.FILTER_NOT_MODELED, + dataset_name, + "Sigma named/static element filters have no Ossie equivalent (they are a " + "presentation-layer concept, not part of the portable semantic model); " + "preserved verbatim in custom_extensions only.", + ) + ) + + datasets.append( + OSIDataset( + name=dataset_name, + source=source_str, + description=element.get("description"), + fields=fields or None, + custom_extensions=[_vendor_ext(dataset_ext)], + ) + ) + + for metric in element.get("metrics") or []: + formula = metric.get("formula", "") + metric_name = metric.get("name") or metric["id"] + try: + node = qualify(parse_formula(formula), dataset_name) + sql = to_ansi_sql(node, dataset_alias=None) + except FormulaParseError: + sql = None + + dialect_exprs = [OSIDialectExpression(dialect=OSIDialect.SIGMA, expression=formula)] + if sql is not None: + dialect_exprs.append(OSIDialectExpression(dialect=OSIDialect.ANSI_SQL, expression=sql)) + else: + issues.append( + ConverterIssue( + ConverterIssueType.EXPRESSION_NOT_TRANSLATABLE, + f"{dataset_name}.{metric_name}", + f"Metric formula {formula!r} has no ANSI SQL equivalent.", + ) + ) + + metrics.append( + OSIMetric( + name=metric_name, + expression=OSIExpression(dialects=dialect_exprs), + custom_extensions=[_vendor_ext({"id": metric["id"], "element_id": element["id"]})], + ) + ) + + for rel in element.get("relationships") or []: + target_id = rel.get("targetElementId") + target_element = element_by_id.get(target_id) + target_name = target_element.get("name", target_id) if target_element else target_id + from_index = index_by_id[element["id"]] + to_index = index_by_id.get(target_id) + + from_columns: list[str] = [] + to_columns: list[str] = [] + for key in rel.get("keys") or []: + from_col, from_resolved = from_index.resolve(key["sourceColumnId"]) + if to_index is not None: + to_col, to_resolved = to_index.resolve(key["targetColumnId"]) + else: + to_col, to_resolved = key["targetColumnId"], False + from_columns.append(from_col) + to_columns.append(to_col) + if not (from_resolved and to_resolved): + issues.append( + ConverterIssue( + ConverterIssueType.RELATIONSHIP_COLUMN_UNRESOLVED, + rel.get("name") or rel["id"], + "Could not resolve one or both join key columns to a modeled Ossie " + "field name; the raw Sigma column reference is preserved in " + "custom_extensions for exact round-trip reconstruction.", + ) + ) + + rel_ext = { + "id": rel["id"], + "element_id": element["id"], + "raw_keys": rel.get("keys"), + } + if rel.get("description"): + rel_ext["description"] = rel["description"] + + relationships.append( + OSIRelationship( + name=rel.get("name") or rel["id"], + **{"from": dataset_name}, + to=target_name, + from_columns=from_columns, + to_columns=to_columns, + custom_extensions=[_vendor_ext(rel_ext)], + ) + ) + + model_ext: dict[str, Any] = {k: spec[k] for k in _MODEL_LEVEL_SPEC_KEYS if k in spec} + if other_elements: + model_ext["non_table_elements"] = [ + {"page_id": page.get("id"), "page_name": page.get("name"), "element": element} + for page, element in other_elements + ] + for _, element in other_elements: + issues.append( + ConverterIssue( + ConverterIssueType.CONTROL_ELEMENT_NOT_MODELED, + element.get("name") or element.get("controlId") or element["id"], + f"Sigma element kind {element.get('kind')!r} (e.g. workbook controls/filters) has " + "no equivalent in the Ossie semantic model; preserved verbatim in " + "custom_extensions only. See LIMITATIONS.md.", + ) + ) + + semantic_model = OSISemanticModel( + name=spec.get("name", "sigma_data_model"), + datasets=datasets, + relationships=relationships or None, + metrics=metrics or None, + custom_extensions=[_vendor_ext(model_ext)] if model_ext else None, + ) + + document = OSIDocument( + dialects=[OSIDialect.ANSI_SQL, OSIDialect.SIGMA], + vendors=[OSIVendor.SIGMA], + semantic_model=[semantic_model], + ) + return ConverterResult(output=document, issues=issues) diff --git a/converters/sigma/tests/__init__.py b/converters/sigma/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/converters/sigma/tests/fixtures/fixtureA_sigma.json b/converters/sigma/tests/fixtures/fixtureA_sigma.json new file mode 100644 index 00000000..4b0fc9e6 --- /dev/null +++ b/converters/sigma/tests/fixtures/fixtureA_sigma.json @@ -0,0 +1,100 @@ +{ + "kind": "data-model", + "name": "Sales", + "dataModelId": "11111111-1111-1111-1111-111111111111", + "folderId": "22222222-2222-2222-2222-222222222222", + "documentVersion": 4, + "latestDocumentVersion": 4, + "schemaVersion": 3, + "pages": [ + { + "id": "pageA", + "name": "Model", + "elements": [ + { + "id": "elemOrders", + "kind": "table", + "name": "Orders", + "source": { + "connectionId": "conn-1", + "kind": "warehouse-table", + "path": ["ANALYTICS", "PUBLIC", "ORDERS"] + }, + "columns": [ + {"id": "colOrderId", "formula": "[ORDERS/ORDER_ID]", "name": "Order ID"}, + {"id": "colCustomerId", "formula": "[ORDERS/CUSTOMER_ID]", "name": "Customer ID"}, + {"id": "colStatus", "formula": "[ORDERS/STATUS]", "name": "Status"}, + {"id": "colAmount", "formula": "[ORDERS/AMOUNT]", "name": "Amount"}, + {"id": "colCreatedAt", "formula": "[ORDERS/CREATED_AT]", "name": "Created At"}, + { + "id": "colIsClosed", + "formula": "If([Status] = \"closed\", 1, 0)", + "name": "Is Closed", + "description": "1 when the order status is closed, else 0" + }, + { + "id": "colOrderYear", + "formula": "Year([Created At])", + "name": "Order Year" + }, + { + "id": "colNetAmount", + "formula": "IfNull([Amount], 0) - SumIf([Status] = \"refunded\", [Amount])", + "name": "Net Amount" + } + ], + "folders": [ + {"id": "folderIdentifiers", "name": "Identifiers", "items": ["colOrderId", "colCustomerId"]}, + {"id": "folderAttributes", "name": "Attributes", "items": ["colStatus", "colAmount", "colCreatedAt", "colIsClosed", "colOrderYear", "colNetAmount"]} + ], + "order": ["folderIdentifiers", "folderAttributes"], + "metrics": [ + {"id": "metricTotalAmount", "formula": "Sum([Amount])", "name": "Total Amount"}, + {"id": "metricOrderCount", "formula": "CountDistinct([Order ID])", "name": "Order Count"} + ], + "relationships": [ + { + "id": "relOrdersToCustomers", + "name": "Customers", + "description": "Each order belongs to one customer", + "targetElementId": "elemCustomers", + "keys": [ + {"sourceColumnId": "colCustomerId", "targetColumnId": "colCustId"} + ] + } + ] + }, + { + "id": "elemCustomers", + "kind": "table", + "name": "Customers", + "source": { + "connectionId": "conn-1", + "kind": "warehouse-table", + "path": ["ANALYTICS", "PUBLIC", "CUSTOMERS"] + }, + "columns": [ + {"id": "colCustId", "formula": "[CUSTOMERS/CUSTOMER_ID]", "name": "Customer ID"}, + {"id": "colCustName", "formula": "[CUSTOMERS/NAME]", "name": "Customer Name"}, + {"id": "colCustEmail", "formula": "Lower(Trim([CUSTOMERS/EMAIL]))", "name": "Customer Email"} + ], + "folders": [ + {"id": "folderCustAttrs", "name": "Attributes", "items": ["colCustId", "colCustName", "colCustEmail"]} + ], + "order": ["folderCustAttrs"] + }, + { + "id": "controlDateRange", + "kind": "control", + "controlId": "Order-Date", + "controlType": "date-range", + "includeToday": true, + "includeNulls": false, + "filters": [ + {"columnId": "colCreatedAt", "source": {"elementId": "elemOrders", "kind": "table"}} + ] + } + ] + } + ] +} diff --git a/converters/sigma/tests/fixtures/fixtureB_sigma.json b/converters/sigma/tests/fixtures/fixtureB_sigma.json new file mode 100644 index 00000000..2b178219 --- /dev/null +++ b/converters/sigma/tests/fixtures/fixtureB_sigma.json @@ -0,0 +1,84 @@ +{ + "kind": "data-model", + "name": "Events", + "dataModelId": "33333333-3333-3333-3333-333333333333", + "pages": [ + { + "id": "pageB", + "name": "Model", + "elements": [ + { + "id": "elemEvents", + "kind": "table", + "name": "Events", + "source": { + "connectionId": "conn-2", + "kind": "warehouse-table", + "path": ["ANALYTICS", "PUBLIC", "EVENTS"] + }, + "columns": [ + {"id": "colEventId", "formula": "[EVENTS/EVENT_ID]", "name": "Event ID"}, + {"id": "colOrgId", "formula": "[EVENTS/ORG_ID]", "name": "Org ID"}, + {"id": "colUserId", "formula": "[EVENTS/USER_ID]", "name": "User ID"}, + { + "id": "colPayload", + "formula": "[EVENTS/PAYLOAD]", + "name": "Payload", + "format": {"kind": "variant"} + }, + { + "id": "colRunningTotal", + "formula": "RunningSum([EVENTS/AMOUNT])", + "name": "Running Total" + } + ], + "relationships": [ + { + "id": "relEventsToOrgUser", + "name": "Org And User", + "targetElementId": "elemOrgUsers", + "keys": [ + { + "sourceColumnId": "inode-abc123/ORG_ID", + "targetColumnId": "inode-def456/ORGANIZATION_UUID" + }, + { + "sourceColumnId": "colUserId", + "targetColumnId": "inode-def456/USER_UUID" + } + ] + } + ] + }, + { + "id": "elemOrgUsers", + "kind": "table", + "name": "Org Users", + "source": { + "connectionId": "conn-2", + "kind": "warehouse-table", + "path": ["ANALYTICS", "PUBLIC", "ORG_USERS"] + }, + "columns": [ + {"id": "colOrgUuid", "formula": "[ORG_USERS/ORGANIZATION_UUID]"}, + {"id": "colUserUuid", "formula": "[ORG_USERS/USER_UUID]"}, + {"id": "colPlanTier", "formula": "[ORG_USERS/PLAN_TIER]", "name": "Plan Tier"} + ] + }, + { + "id": "elemActiveEvents", + "kind": "table", + "name": "Active Events", + "description": "Derived view layered on Events, not a direct warehouse table", + "source": { + "kind": "table", + "elementId": "elemEvents" + }, + "columns": [ + {"id": "colActiveEventId", "formula": "[Event ID]", "name": "Event ID"} + ] + } + ] + } + ] +} diff --git a/converters/sigma/tests/helpers.py b/converters/sigma/tests/helpers.py new file mode 100644 index 00000000..2ee80913 --- /dev/null +++ b/converters/sigma/tests/helpers.py @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json +from pathlib import Path +from typing import Any + +FIXTURES_DIR = Path(__file__).parent / "fixtures" + + +def load_fixture(name: str) -> dict[str, Any]: + return json.loads((FIXTURES_DIR / name).read_text()) + + +def normalize(obj: Any) -> Any: + """Recursively sort dict keys so structurally-equal JSON compares equal regardless of order.""" + if isinstance(obj, dict): + return {k: normalize(v) for k, v in sorted(obj.items())} + if isinstance(obj, list): + return [normalize(v) for v in obj] + return obj diff --git a/converters/sigma/tests/test_osi_to_sigma.py b/converters/sigma/tests/test_osi_to_sigma.py new file mode 100644 index 00000000..99d3bae2 --- /dev/null +++ b/converters/sigma/tests/test_osi_to_sigma.py @@ -0,0 +1,81 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from pathlib import Path + +import yaml +from ossie import OSIDocument + +from ossie_sigma.converter_issues import ConverterIssueType +from ossie_sigma.osi_to_sigma import OSIToSigmaConverter +from ossie_sigma.sigma_to_osi import SigmaToOSIConverter + +from .helpers import load_fixture, normalize + +EXAMPLES_DIR = Path(__file__).parent.parent.parent.parent / "examples" + + +def test_roundtrip_fixture_a_is_byte_identical(): + spec = load_fixture("fixtureA_sigma.json") + document = SigmaToOSIConverter().convert(spec).output + reconstructed = OSIToSigmaConverter().convert(document).output + assert normalize(reconstructed) == normalize(spec) + + +def test_roundtrip_fixture_b_is_byte_identical(): + spec = load_fixture("fixtureB_sigma.json") + document = SigmaToOSIConverter().convert(spec).output + reconstructed = OSIToSigmaConverter().convert(document).output + assert normalize(reconstructed) == normalize(spec) + + +def test_foreign_origin_document_synthesizes_valid_spec(): + """An Ossie document never touched by Sigma (no SIGMA custom_extensions) must + still convert to a structurally valid Sigma spec, with synthesized ids and + formulas best-effort translated from ANSI SQL.""" + document = OSIDocument.model_validate( + yaml.safe_load((EXAMPLES_DIR / "tpcds_semantic_model.yaml").read_text()) + ) + result = OSIToSigmaConverter().convert(document) + spec = result.output + + assert spec["kind"] == "data-model" + assert spec["pages"] + element_names = {e["name"] for p in spec["pages"] for e in p["elements"]} + assert "store_sales" in element_names + + store_sales = next(e for p in spec["pages"] for e in p["elements"] if e["name"] == "store_sales") + assert all("id" in c and "formula" in c for c in store_sales["columns"]) + # Plain passthrough columns get no explicit `name` (matches Sigma's own convention). + plain_column = next(c for c in store_sales["columns"] if c["formula"] == "[ss_sold_date_sk]") + assert "name" not in plain_column + + # Single-dataset metrics are attached to their owning element ... + assert any(m["name"] == "total_sales" for m in store_sales.get("metrics", [])) + # ... while genuinely cross-dataset metrics are dropped with a recorded issue, + # not silently discarded and not incorrectly attached to one dataset. + issue_types = {i.issue_type for i in result.issues} + assert ConverterIssueType.CROSS_DATASET_METRIC_DROPPED in issue_types + + +def test_ids_are_deterministic_across_repeated_conversions(): + document = OSIDocument.model_validate( + yaml.safe_load((EXAMPLES_DIR / "tpcds_semantic_model.yaml").read_text()) + ) + spec_1 = OSIToSigmaConverter().convert(document).output + spec_2 = OSIToSigmaConverter().convert(document).output + assert normalize(spec_1) == normalize(spec_2) diff --git a/converters/sigma/tests/test_roundtrip.py b/converters/sigma/tests/test_roundtrip.py new file mode 100644 index 00000000..c3ac5dbb --- /dev/null +++ b/converters/sigma/tests/test_roundtrip.py @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""End-to-end round trips through the same serialization boundary the CLI uses +(OSIDocument -> YAML text -> re-parsed OSIDocument), for both fixtures.""" + +import pytest +import yaml +from ossie import OSIDocument + +from ossie_sigma.osi_to_sigma import OSIToSigmaConverter +from ossie_sigma.sigma_to_osi import SigmaToOSIConverter + +from .helpers import load_fixture, normalize + + +@pytest.mark.parametrize("fixture_name", ["fixtureA_sigma.json", "fixtureB_sigma.json"]) +def test_sigma_osi_sigma_roundtrip_through_yaml_serialization(fixture_name): + spec = load_fixture(fixture_name) + + document = SigmaToOSIConverter().convert(spec).output + yaml_text = document.to_osi_yaml() + + reparsed_document = OSIDocument.model_validate(yaml.safe_load(yaml_text)) + reconstructed_spec = OSIToSigmaConverter().convert(reparsed_document).output + + assert normalize(reconstructed_spec) == normalize(spec) + + +@pytest.mark.parametrize("fixture_name", ["fixtureA_sigma.json", "fixtureB_sigma.json"]) +def test_osi_sigma_osi_roundtrip_preserves_portable_fields(fixture_name): + """Sigma -> Ossie -> Sigma -> Ossie: the second Ossie document's portable + (non-custom_extensions) content must match the first, even though the Sigma + spec in between round-trips through JSON.""" + spec = load_fixture(fixture_name) + + document_1 = SigmaToOSIConverter().convert(spec).output + spec_2 = OSIToSigmaConverter().convert(document_1).output + document_2 = SigmaToOSIConverter().convert(spec_2).output + + def portable(document): + model = document.semantic_model[0] + return { + "datasets": [(d.name, d.source, [(f.name, f.datatype) for f in d.fields or []]) for d in model.datasets], + "relationships": [(r.name, r.from_dataset, r.to, r.from_columns, r.to_columns) for r in model.relationships or []], + "metrics": [(m.name,) for m in model.metrics or []], + } + + assert portable(document_1) == portable(document_2) diff --git a/converters/sigma/tests/test_sigma_formula.py b/converters/sigma/tests/test_sigma_formula.py new file mode 100644 index 00000000..b8b1487f --- /dev/null +++ b/converters/sigma/tests/test_sigma_formula.py @@ -0,0 +1,114 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pytest + +from ossie_sigma.sigma_formula import ( + ColumnRef, + FormulaParseError, + is_plain_column_ref, + parse_formula, + sql_to_sigma_formula, + to_ansi_sql, +) + + +@pytest.mark.parametrize( + ("formula", "dataset_alias", "expected_sql"), + [ + ("[Amount]", "Orders", '"Amount"'), + ("[Orders/Amount]", "Orders", '"Amount"'), + ("[Orders/Amount]", None, '"Orders"."Amount"'), + ("Sum([Amount])", "Orders", 'SUM("Amount")'), + ("CountDistinct([Order Id])", "Orders", 'COUNT(DISTINCT "Order Id")'), + ('If([Status] = "closed", 1, 0)', "Orders", "CASE WHEN (\"Status\" = 'closed') THEN 1 ELSE 0 END"), + ("IfNull([X], 0)", "Orders", 'COALESCE("X", 0)'), + ("IsNull([X])", "Orders", '("X" IS NULL)'), + ("IsNotNull([X])", "Orders", '("X" IS NOT NULL)'), + ('[A] & " " & [B]', "T", "((\"A\" || ' ') || \"B\")"), + ("Left([Name], 3)", "T", 'SUBSTRING("Name" FROM 1 FOR 3)'), + ("Mid([Name], 2, 3)", "T", 'SUBSTRING("Name" FROM 2 FOR 3)'), + ("Year([Created At])", "T", 'EXTRACT(YEAR FROM "Created At")'), + ("Upper(Trim([Name]))", "T", 'UPPER(TRIM("Name"))'), + ("[Qty] * [Price] + 1", "T", '(("Qty" * "Price") + 1)'), + ("Median([Amount])", "T", 'PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY "Amount")'), + ('SumIf([Status] = "won", [Amount])', "T", "SUM(CASE WHEN (\"Status\" = 'won') THEN \"Amount\" ELSE 0 END)"), + ("2 ^ 3", "T", "POWER(2, 3)"), + ("-[X]", "T", '-("X")'), + ("NOT [X]", "T", 'NOT ("X")'), + ], +) +def test_translatable_formulas(formula, dataset_alias, expected_sql): + node = parse_formula(formula) + assert to_ansi_sql(node, dataset_alias=dataset_alias) == expected_sql + + +@pytest.mark.parametrize( + "formula", + [ + "RunningSum([Amount])", + "Rank([Amount])", + "SomeUnknownFunction([X])", + ], +) +def test_untranslatable_functions_return_none(formula): + node = parse_formula(formula) + assert to_ansi_sql(node) is None + + +@pytest.mark.parametrize( + "formula", + [ + "", + "[Unterminated", + "Sum([X]", + "@#$%", + ], +) +def test_unparseable_formulas_raise(formula): + with pytest.raises(FormulaParseError): + parse_formula(formula) + + +def test_is_plain_column_ref(): + assert is_plain_column_ref("[Orders/Amount]") == ColumnRef("Orders", "Amount") + assert is_plain_column_ref("[Amount]") == ColumnRef(None, "Amount") + assert is_plain_column_ref("Sum([Amount])") is None + assert is_plain_column_ref("not a formula @@@") is None + + +@pytest.mark.parametrize( + ("sql", "dataset_alias", "expected"), + [ + ('"Amount"', "Orders", "[Amount]"), + ('"Orders"."Amount"', None, "[Orders/Amount]"), + ("SUM(ss_ext_sales_price)", "store_sales", "Sum([ss_ext_sales_price])"), + ("COUNT(DISTINCT customer_id)", "customer", "CountDistinct([customer_id])"), + ("CASE WHEN status = 'won' THEN 1 ELSE 0 END", "deals", 'If((["status"] = "won"), 1, 0)'.replace('["status"]', "[status]")), + ], +) +def test_reverse_translation_basic(sql, dataset_alias, expected): + result = sql_to_sigma_formula(sql, dataset_alias=dataset_alias) + assert result == expected + + +def test_reverse_translation_gives_up_on_count_star(): + assert sql_to_sigma_formula("COUNT(*)") is None + + +def test_reverse_translation_gives_up_on_unparseable(): + assert sql_to_sigma_formula("not valid sql {{{") is None diff --git a/converters/sigma/tests/test_sigma_to_osi.py b/converters/sigma/tests/test_sigma_to_osi.py new file mode 100644 index 00000000..5b2f79a3 --- /dev/null +++ b/converters/sigma/tests/test_sigma_to_osi.py @@ -0,0 +1,124 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from ossie import OSIDialect + +from ossie_sigma.converter_issues import ConverterIssueType +from ossie_sigma.sigma_to_osi import SigmaToOSIConverter + +from .helpers import load_fixture + + +def test_basic_datasets_fields_relationships_metrics(): + spec = load_fixture("fixtureA_sigma.json") + result = SigmaToOSIConverter().convert(spec) + model = result.output.semantic_model[0] + + assert model.name == "Sales" + assert {d.name for d in model.datasets} == {"Orders", "Customers"} + + orders = next(d for d in model.datasets if d.name == "Orders") + assert orders.source == "ANALYTICS.PUBLIC.ORDERS" + field_names = {f.name for f in orders.fields} + assert {"Order ID", "Customer ID", "Status", "Amount", "Is Closed", "Order Year", "Net Amount"} <= field_names + + is_closed = next(f for f in orders.fields if f.name == "Is Closed") + dialects = {d.dialect: d.expression for d in is_closed.expression.dialects} + assert dialects[OSIDialect.SIGMA] == 'If([Status] = "closed", 1, 0)' + assert dialects[OSIDialect.ANSI_SQL] == "CASE WHEN (\"Status\" = 'closed') THEN 1 ELSE 0 END" + + assert {m.name for m in model.metrics} == {"Total Amount", "Order Count"} + assert len(model.relationships) == 1 + rel = model.relationships[0] + assert rel.from_dataset == "Orders" + assert rel.to == "Customers" + assert rel.from_columns == ["Customer ID"] + assert rel.to_columns == ["Customer ID"] + + +def test_control_element_preserved_but_not_modeled(): + spec = load_fixture("fixtureA_sigma.json") + result = SigmaToOSIConverter().convert(spec) + model = result.output.semantic_model[0] + + dataset_names = {d.name for d in model.datasets} + assert "Order-Date" not in dataset_names # controls are never modeled as datasets + + issue_types = {i.issue_type for i in result.issues} + assert ConverterIssueType.CONTROL_ELEMENT_NOT_MODELED in issue_types + + +def test_relationship_resolves_inode_style_physical_column_refs(): + spec = load_fixture("fixtureB_sigma.json") + result = SigmaToOSIConverter().convert(spec) + model = result.output.semantic_model[0] + + rel = next(r for r in model.relationships if r.name == "Org And User") + assert rel.from_columns == ["Org ID", "User ID"] + assert rel.to_columns == ["ORGANIZATION_UUID", "USER_UUID"] + + +def test_opaque_datatype_for_unrecognized_format(): + spec = load_fixture("fixtureB_sigma.json") + result = SigmaToOSIConverter().convert(spec) + model = result.output.semantic_model[0] + + events = next(d for d in model.datasets if d.name == "Events") + payload = next(f for f in events.fields if f.name == "Payload") + assert payload.datatype == "Opaque" + + issue_types = {i.issue_type for i in result.issues} + assert ConverterIssueType.OPAQUE_DATATYPE in issue_types + + +def test_untranslatable_formula_keeps_sigma_dialect_only(): + spec = load_fixture("fixtureB_sigma.json") + result = SigmaToOSIConverter().convert(spec) + model = result.output.semantic_model[0] + + events = next(d for d in model.datasets if d.name == "Events") + running_total = next(f for f in events.fields if f.name == "Running Total") + dialects = {d.dialect for d in running_total.expression.dialects} + assert dialects == {OSIDialect.SIGMA} + + issue_types = {i.issue_type for i in result.issues} + assert ConverterIssueType.EXPRESSION_NOT_TRANSLATABLE in issue_types + + +def test_derived_element_preserved_with_issue(): + spec = load_fixture("fixtureB_sigma.json") + result = SigmaToOSIConverter().convert(spec) + model = result.output.semantic_model[0] + + active_events = next(d for d in model.datasets if d.name == "Active Events") + assert active_events.source == "element:elemEvents" + + issue_types = {i.issue_type for i in result.issues} + assert ConverterIssueType.DERIVED_ELEMENT_NOT_MODELED in issue_types + + +def test_native_ids_and_page_metadata_preserved_in_custom_extensions(): + import json + + spec = load_fixture("fixtureA_sigma.json") + result = SigmaToOSIConverter().convert(spec) + model = result.output.semantic_model[0] + + orders = next(d for d in model.datasets if d.name == "Orders") + ext = json.loads(orders.custom_extensions[0].data) + assert ext["id"] == "elemOrders" + assert ext["page_id"] == "pageA" diff --git a/converters/sigma/uv.lock b/converters/sigma/uv.lock new file mode 100644 index 00000000..cac15c46 --- /dev/null +++ b/converters/sigma/uv.lock @@ -0,0 +1,315 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "annotated-types" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/56/a8120250d128bed162cd73c76d45f6ef9991f3e068f62a8ee060afa3104a/annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7", size = 15893, upload-time = "2026-07-23T20:16:13.995Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" }, +] + +[[package]] +name = "apache-ossie" +version = "0.2.0.dev0" +source = { editable = "../../python" } +dependencies = [ + { name = "pydantic" }, + { name = "pyyaml" }, +] + +[package.metadata] +requires-dist = [ + { name = "pydantic", specifier = ">=2.0" }, + { name = "pyyaml", specifier = ">=6.0" }, +] + +[[package]] +name = "apache-ossie-sigma" +version = "0.2.0.dev0" +source = { editable = "." } +dependencies = [ + { name = "apache-ossie" }, + { name = "pyyaml" }, + { name = "sqlglot" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [ + { name = "apache-ossie", editable = "../../python" }, + { name = "pyyaml", specifier = ">=6.0" }, + { name = "sqlglot", specifier = ">=20.0" }, +] + +[package.metadata.requires-dev] +dev = [{ name = "pytest", specifier = ">=8.0" }] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pydantic" +version = "2.13.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.46.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/fa/6d7708d2cfc1a832acb6aeb0cd16e801902df8a0f583bb3b4b527fde022e/pydantic_core-2.46.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594", size = 2111872, upload-time = "2026-05-06T13:40:27.596Z" }, + { url = "https://files.pythonhosted.org/packages/ae/6f/aa064a3e74b5745afbdf250594f38e7ead05e2d651bcb35994b9417a0d4d/pydantic_core-2.46.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c", size = 1948255, upload-time = "2026-05-06T13:39:12.574Z" }, + { url = "https://files.pythonhosted.org/packages/43/3a/41114a9f7569b84b4d84e7a018c57c56347dac30c0d4a872946ec4e36c46/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826", size = 1972827, upload-time = "2026-05-06T13:38:19.841Z" }, + { url = "https://files.pythonhosted.org/packages/ef/25/1ab42e8048fe551934d9884e8d64daa7e990ad386f310a15981aeb6a5b08/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04", size = 2041051, upload-time = "2026-05-06T13:38:10.447Z" }, + { url = "https://files.pythonhosted.org/packages/94/c2/1a934597ddf08da410385b3b7aae91956a5a76c635effef456074fad7e88/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e", size = 2221314, upload-time = "2026-05-06T13:40:13.089Z" }, + { url = "https://files.pythonhosted.org/packages/02/6d/9e8ad178c9c4df27ad3c8f25d1fe2a7ab0d2ba0559fad4aee5d3d1f16771/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3", size = 2285146, upload-time = "2026-05-06T13:38:59.224Z" }, + { url = "https://files.pythonhosted.org/packages/80/50/540cd3aeefc041beb111125c4bff779831a2111fc6b15a9138cda277d32c/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4", size = 2089685, upload-time = "2026-05-06T13:38:17.762Z" }, + { url = "https://files.pythonhosted.org/packages/6b/a4/b440ad35f05f6a38f89fa0f149accb3f0e02be94ca5e15f3c449a61b4bc9/pydantic_core-2.46.4-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398", size = 2115420, upload-time = "2026-05-06T13:37:58.195Z" }, + { url = "https://files.pythonhosted.org/packages/99/61/de4f55db8dfd57bfdfa9a12ec90fe1b57c4f41062f7ca86f08586b3e0ac0/pydantic_core-2.46.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3", size = 2165122, upload-time = "2026-05-06T13:37:01.167Z" }, + { url = "https://files.pythonhosted.org/packages/f7/52/7c529d7bdb2d1068bd52f51fe32572c8301f9a4febf1948f10639f1436f5/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848", size = 2182573, upload-time = "2026-05-06T13:38:45.04Z" }, + { url = "https://files.pythonhosted.org/packages/37/b3/7c40325848ba78247f2812dcf9c7274e38cd801820ca6dd9fe63bcfb0eb4/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3", size = 2317139, upload-time = "2026-05-06T13:37:15.539Z" }, + { url = "https://files.pythonhosted.org/packages/d9/37/f913f81a657c865b75da6c0dbed79876073c2a43b5bd9edbe8da785e4d49/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109", size = 2360433, upload-time = "2026-05-06T13:37:30.099Z" }, + { url = "https://files.pythonhosted.org/packages/c4/67/6acaa1be2567f9256b056d8477158cac7240813956ce86e49deae8e173b4/pydantic_core-2.46.4-cp311-cp311-win32.whl", hash = "sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda", size = 1985513, upload-time = "2026-05-06T13:38:15.669Z" }, + { url = "https://files.pythonhosted.org/packages/aa/e6/c505f83dfeda9a2e5c995cfd872949e4d05e12f7feb3dca72f633daefa94/pydantic_core-2.46.4-cp311-cp311-win_amd64.whl", hash = "sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33", size = 2071114, upload-time = "2026-05-06T13:40:35.416Z" }, + { url = "https://files.pythonhosted.org/packages/0f/da/7a263a96d965d9d0df5e8de8a475f33495451117035b09acb110288c381f/pydantic_core-2.46.4-cp311-cp311-win_arm64.whl", hash = "sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d", size = 2044298, upload-time = "2026-05-06T13:38:29.754Z" }, + { url = "https://files.pythonhosted.org/packages/ce/8c/af022f0af448d7747c5154288d46b5f2bc5f17366eaa0e23e9aa04d59f3b/pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2", size = 2106158, upload-time = "2026-05-06T13:38:57.215Z" }, + { url = "https://files.pythonhosted.org/packages/19/95/6195171e385007300f0f5574592e467c568becce2d937a0b6804f218bc49/pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f", size = 1951724, upload-time = "2026-05-06T13:37:02.697Z" }, + { url = "https://files.pythonhosted.org/packages/8e/bc/f47d1ff9cbb1620e1b5b697eef06010035735f07820180e74178226b27b3/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7", size = 1975742, upload-time = "2026-05-06T13:37:09.448Z" }, + { url = "https://files.pythonhosted.org/packages/5b/11/9b9a5b0306345664a2da6410877af6e8082481b5884b3ddd78d47c6013ce/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7", size = 2052418, upload-time = "2026-05-06T13:37:38.234Z" }, + { url = "https://files.pythonhosted.org/packages/f1/b7/a65fec226f5d78fc39f4a13c4cc0c768c22b113438f60c14adc9d2865038/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712", size = 2232274, upload-time = "2026-05-06T13:38:27.753Z" }, + { url = "https://files.pythonhosted.org/packages/68/f0/92039db98b907ef49269a8271f67db9cb78ae2fc68062ef7e4e77adb5f61/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4", size = 2309940, upload-time = "2026-05-06T13:38:05.353Z" }, + { url = "https://files.pythonhosted.org/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce", size = 2094516, upload-time = "2026-05-06T13:39:10.577Z" }, + { url = "https://files.pythonhosted.org/packages/22/37/a8aca44d40d737dde2bc05b3c6c07dff0de07ce6f82e9f3167aeaf4d5dea/pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987", size = 2136854, upload-time = "2026-05-06T13:40:22.59Z" }, + { url = "https://files.pythonhosted.org/packages/24/99/fcef1b79238c06a8cbec70819ac722ba76e02bc8ada9b0fd66eba40da01b/pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b", size = 2180306, upload-time = "2026-05-06T13:40:10.666Z" }, + { url = "https://files.pythonhosted.org/packages/ae/6c/fc44000918855b42779d007ae63b0532794739027b2f417321cddbc44f6a/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458", size = 2190044, upload-time = "2026-05-06T13:40:43.231Z" }, + { url = "https://files.pythonhosted.org/packages/6b/65/d9cadc9f1920d7a127ad2edba16c1db7916e59719285cd6c94600b0080ba/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b", size = 2329133, upload-time = "2026-05-06T13:39:57.365Z" }, + { url = "https://files.pythonhosted.org/packages/d0/cf/c873d91679f3a30bcf5e7ac280ce5573483e72295307685120d0d5ad3416/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c", size = 2374464, upload-time = "2026-05-06T13:38:06.976Z" }, + { url = "https://files.pythonhosted.org/packages/47/bd/6f2fc8188f31bf10590f1e98e7b306336161fac930a8c514cd7bd828c7dc/pydantic_core-2.46.4-cp312-cp312-win32.whl", hash = "sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894", size = 1974823, upload-time = "2026-05-06T13:40:47.985Z" }, + { url = "https://files.pythonhosted.org/packages/40/8c/985c1d41ea1107c2534abd9870e4ed5c8e7669b5c308297835c001e7a1c4/pydantic_core-2.46.4-cp312-cp312-win_amd64.whl", hash = "sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89", size = 2072919, upload-time = "2026-05-06T13:39:21.153Z" }, + { url = "https://files.pythonhosted.org/packages/c4/ba/f463d006e0c47373ca7ec5e1a261c59dc01ef4d62b2657af925fb0deee3a/pydantic_core-2.46.4-cp312-cp312-win_arm64.whl", hash = "sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a", size = 2027604, upload-time = "2026-05-06T13:39:03.753Z" }, + { url = "https://files.pythonhosted.org/packages/51/a2/5d30b469c5267a17b39dec53208222f76a8d351dfac4af661888c5aee77d/pydantic_core-2.46.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008", size = 2106306, upload-time = "2026-05-06T13:37:48.029Z" }, + { url = "https://files.pythonhosted.org/packages/c1/81/4fa520eaffa8bd7d1525e644cd6d39e7d60b1592bc5b516693c7340b50f1/pydantic_core-2.46.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4", size = 1951906, upload-time = "2026-05-06T13:37:17.012Z" }, + { url = "https://files.pythonhosted.org/packages/03/d5/fd02da45b659668b05923b17ba3a0100a0a3d5541e3bd8fcc4ecb711309e/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76", size = 1976802, upload-time = "2026-05-06T13:37:35.113Z" }, + { url = "https://files.pythonhosted.org/packages/21/f2/95727e1368be3d3ed485eaab7adbd7dda408f33f7a36e8b48e0144002b91/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3", size = 2052446, upload-time = "2026-05-06T13:37:12.313Z" }, + { url = "https://files.pythonhosted.org/packages/9c/86/5d99feea3f77c7234b8718075b23db11532773c1a0dbd9b9490215dc2eeb/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76", size = 2232757, upload-time = "2026-05-06T13:39:01.149Z" }, + { url = "https://files.pythonhosted.org/packages/d2/3a/508ac615935ef7588cf6d9e9b91309fdc2da751af865e02a9098de88258c/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4", size = 2309275, upload-time = "2026-05-06T13:37:41.406Z" }, + { url = "https://files.pythonhosted.org/packages/07/f8/41db9de19d7987d6b04715a02b3b40aea467000275d9d758ffaa31af7d50/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a", size = 2094467, upload-time = "2026-05-06T13:39:18.847Z" }, + { url = "https://files.pythonhosted.org/packages/2c/e2/f35033184cb11d0052daf4416e8e10a502ea2ac006fc4f459aee872727d1/pydantic_core-2.46.4-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262", size = 2134417, upload-time = "2026-05-06T13:40:17.944Z" }, + { url = "https://files.pythonhosted.org/packages/7e/7b/6ceeb1cc90e193862f444ebe373d8fdf613f0a82572dde03fb10734c6c71/pydantic_core-2.46.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e", size = 2179782, upload-time = "2026-05-06T13:40:32.618Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f2/c8d7773ede6af08036423a00ae0ceffce266c3c52a096c435d68c896083f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd", size = 2188782, upload-time = "2026-05-06T13:36:51.018Z" }, + { url = "https://files.pythonhosted.org/packages/59/31/0c864784e31f09f05cdd87606f08923b9c9e7f6e51dd27f20f62f975ce9f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be", size = 2328334, upload-time = "2026-05-06T13:40:37.764Z" }, + { url = "https://files.pythonhosted.org/packages/c2/eb/4f6c8a41efa30baa755590f4141abf3a8c370fab610915733e74134a7270/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d", size = 2372986, upload-time = "2026-05-06T13:39:34.152Z" }, + { url = "https://files.pythonhosted.org/packages/5b/24/b375a480d53113860c299764bfe9f349a3dc9108b3adc0d7f0d786492ebf/pydantic_core-2.46.4-cp313-cp313-win32.whl", hash = "sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb", size = 1973693, upload-time = "2026-05-06T13:37:55.072Z" }, + { url = "https://files.pythonhosted.org/packages/7e/e8/cff247591966f2d22ec8c003cd7587e27b7ba7b81ab2fb888e3ab75dc285/pydantic_core-2.46.4-cp313-cp313-win_amd64.whl", hash = "sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292", size = 2071819, upload-time = "2026-05-06T13:38:49.139Z" }, + { url = "https://files.pythonhosted.org/packages/c6/1a/f4aee670d5670e9e148e0c82c7db98d780be566c6e6a97ee8035528ca0b3/pydantic_core-2.46.4-cp313-cp313-win_arm64.whl", hash = "sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d", size = 2027411, upload-time = "2026-05-06T13:40:45.796Z" }, + { url = "https://files.pythonhosted.org/packages/8d/74/228a26ddad29c6672b805d9fd78e8d251cd04004fa7eed0e622096cd0250/pydantic_core-2.46.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb", size = 2102079, upload-time = "2026-05-06T13:38:41.019Z" }, + { url = "https://files.pythonhosted.org/packages/ad/1f/8970b150a4b4365623ae00fc88603491f763c627311ae8031e3111356d6e/pydantic_core-2.46.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462", size = 1952179, upload-time = "2026-05-06T13:36:59.812Z" }, + { url = "https://files.pythonhosted.org/packages/95/30/5211a831ae054928054b2f79731661087a2bc5c01e825c672b3a4a8f1b3e/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9", size = 1978926, upload-time = "2026-05-06T13:37:39.933Z" }, + { url = "https://files.pythonhosted.org/packages/57/e9/689668733b1eb67adeef047db3c2e8788fcf65a7fd9c9e2b46b7744fe245/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4", size = 2046785, upload-time = "2026-05-06T13:38:01.995Z" }, + { url = "https://files.pythonhosted.org/packages/60/d9/6715260422ff50a2109878fd24d948a6c3446bb2664f34ee78cd972b3acd/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914", size = 2228733, upload-time = "2026-05-06T13:40:50.371Z" }, + { url = "https://files.pythonhosted.org/packages/18/ae/fdb2f64316afca925640f8e70bb1a564b0ec2721c1389e25b8eb4bf9a299/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28", size = 2307534, upload-time = "2026-05-06T13:37:21.531Z" }, + { url = "https://files.pythonhosted.org/packages/89/1d/8eff589b45bb8190a9d12c49cfad0f176a5cbd1534908a6b5125e2886239/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b", size = 2099732, upload-time = "2026-05-06T13:39:31.942Z" }, + { url = "https://files.pythonhosted.org/packages/06/d5/ee5a3366637fee41dee51a1fc91562dcf12ddbc68fda34e6b253da2324bb/pydantic_core-2.46.4-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c", size = 2129627, upload-time = "2026-05-06T13:37:25.033Z" }, + { url = "https://files.pythonhosted.org/packages/94/33/2414be571d2c6a6c4d08be21f9292b6d3fdb08949a97b6dfe985017821db/pydantic_core-2.46.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb", size = 2179141, upload-time = "2026-05-06T13:37:14.046Z" }, + { url = "https://files.pythonhosted.org/packages/7b/79/7daa95be995be0eecc4cf75064cb33f9bbbfe3fe0158caf2f0d4a996a5c7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898", size = 2184325, upload-time = "2026-05-06T13:36:53.615Z" }, + { url = "https://files.pythonhosted.org/packages/9f/cb/d0a382f5c0de8a222dc61c65348e0ce831b1f68e0a018450d31c2cace3a5/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e", size = 2323990, upload-time = "2026-05-06T13:40:29.971Z" }, + { url = "https://files.pythonhosted.org/packages/05/db/d9ba624cc4a5aced1598e88c04fdbd8310c8a69b9d38b9a3d39ce3a61ed7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519", size = 2369978, upload-time = "2026-05-06T13:37:23.027Z" }, + { url = "https://files.pythonhosted.org/packages/f2/20/d15df15ba918c423461905802bfd2981c3af0bfa0e40d05e13edbfa48bc3/pydantic_core-2.46.4-cp314-cp314-win32.whl", hash = "sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4", size = 1966354, upload-time = "2026-05-06T13:38:03.499Z" }, + { url = "https://files.pythonhosted.org/packages/fc/b6/6b8de4c0a7d7ab3004c439c80c5c1e0a3e8d78bbae19379b01960383d9e5/pydantic_core-2.46.4-cp314-cp314-win_amd64.whl", hash = "sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac", size = 2072238, upload-time = "2026-05-06T13:39:40.807Z" }, + { url = "https://files.pythonhosted.org/packages/32/36/51eb763beec1f4cf59b1db243a7dcc39cbb41230f050a09b9d69faaf0a48/pydantic_core-2.46.4-cp314-cp314-win_arm64.whl", hash = "sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a", size = 2018251, upload-time = "2026-05-06T13:37:26.72Z" }, + { url = "https://files.pythonhosted.org/packages/e8/91/855af51d625b23aa987116a19e231d2aaef9c4a415273ddc189b79a45fee/pydantic_core-2.46.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0", size = 2099593, upload-time = "2026-05-06T13:39:47.682Z" }, + { url = "https://files.pythonhosted.org/packages/fb/1b/8784a54c65edb5f49f0a14d6977cf1b209bba85a4c77445b255c2de58ab3/pydantic_core-2.46.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d", size = 1935226, upload-time = "2026-05-06T13:40:40.428Z" }, + { url = "https://files.pythonhosted.org/packages/e8/e7/1955d28d1afc56dd4b3ad7cc0cf39df1b9852964cf16e5d13912756d6d6b/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b", size = 1974605, upload-time = "2026-05-06T13:37:32.029Z" }, + { url = "https://files.pythonhosted.org/packages/93/e2/3fedbf0ba7a22850e6e9fd78117f1c0f10f950182344d8a6c535d468fdd8/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000", size = 2030777, upload-time = "2026-05-06T13:38:55.239Z" }, + { url = "https://files.pythonhosted.org/packages/f8/61/46be275fcaaba0b4f5b9669dd852267ce1ff616592dccf7a7845588df091/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e", size = 2236641, upload-time = "2026-05-06T13:37:08.096Z" }, + { url = "https://files.pythonhosted.org/packages/60/db/12e93e46a8bac9988be3c016860f83293daea8c716c029c9ace279036f2f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd", size = 2286404, upload-time = "2026-05-06T13:40:20.221Z" }, + { url = "https://files.pythonhosted.org/packages/e2/4a/4d8b19008f38d31c53b8219cfedc2e3d5de5fe99d90076b7e767de29274f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3", size = 2109219, upload-time = "2026-05-06T13:38:12.153Z" }, + { url = "https://files.pythonhosted.org/packages/88/70/3cbc40978fefb7bb09c6708d40d4ad1a5d70fd7213c3d17f971de868ec1f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7", size = 2110594, upload-time = "2026-05-06T13:40:02.971Z" }, + { url = "https://files.pythonhosted.org/packages/9d/20/b8d36736216e29491125531685b2f9e61aa5b4b2599893f8268551da3338/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff", size = 2159542, upload-time = "2026-05-06T13:39:27.506Z" }, + { url = "https://files.pythonhosted.org/packages/1d/a2/367df868eb584dacf6bf82a389272406d7178e301c4ac82545ab98bc2dd9/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424", size = 2168146, upload-time = "2026-05-06T13:38:31.93Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b8/4460f77f7e201893f649a29ab355dddd3beee8a97bcb1a320db414f9a06e/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6", size = 2306309, upload-time = "2026-05-06T13:37:44.717Z" }, + { url = "https://files.pythonhosted.org/packages/64/c4/be2639293acd87dc8ddbcec41a73cee9b2ebf996fe6d892a1a74e88ad3f7/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565", size = 2369736, upload-time = "2026-05-06T13:37:05.645Z" }, + { url = "https://files.pythonhosted.org/packages/30/a6/9f9f380dbb301f67023bf8f707aaa75daadf84f7152d95c410fd7e81d994/pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02", size = 1955575, upload-time = "2026-05-06T13:38:51.116Z" }, + { url = "https://files.pythonhosted.org/packages/40/1f/f1eb9eb350e795d1af8586289746f5c5677d16043040d63710e22abc43c9/pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5", size = 2051624, upload-time = "2026-05-06T13:38:21.672Z" }, + { url = "https://files.pythonhosted.org/packages/f6/d2/42dd53d0a85c27606f316d3aa5d2869c4e8470a5ed6dec30e4a1abe19192/pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596", size = 2017325, upload-time = "2026-05-06T13:40:52.723Z" }, + { url = "https://files.pythonhosted.org/packages/ee/a4/73995fd4ebbb46ba0ee51e6fa049b8f02c40daebb762208feda8a6b7894d/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c", size = 2111589, upload-time = "2026-05-06T13:37:10.817Z" }, + { url = "https://files.pythonhosted.org/packages/fb/7f/f37d3a5e8bfcc2e403f5c57a730f2d815693fb42119e8ea48b3789335af1/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b", size = 1944552, upload-time = "2026-05-06T13:36:56.717Z" }, + { url = "https://files.pythonhosted.org/packages/15/3c/d7eb777b3ff43e8433a4efb39a17aa8fd98a4ee8561a24a67ef5db07b2d6/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b", size = 1982984, upload-time = "2026-05-06T13:39:06.207Z" }, + { url = "https://files.pythonhosted.org/packages/63/87/70b9f40170a81afd55ca26c9b2acb25c20d64bcfbf888fafecb3ba077d4c/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea", size = 2138417, upload-time = "2026-05-06T13:39:45.476Z" }, + { url = "https://files.pythonhosted.org/packages/9d/1d/8987ad40f65ae1432753072f214fb5c74fe47ffbd0698bb9cbbb585664f8/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7", size = 2095527, upload-time = "2026-05-06T13:39:52.283Z" }, + { url = "https://files.pythonhosted.org/packages/64/d3/84c282a7eee1d3ac4c0377546ef5a1ea436ce26840d9ac3b7ed54a377507/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df", size = 1936024, upload-time = "2026-05-06T13:40:15.671Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ca/eac61596cdeb4d7e174d3dc0bd8a6238f14f75f97a24e7b7db4c7e7340a0/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526", size = 1990696, upload-time = "2026-05-06T13:38:34.717Z" }, + { url = "https://files.pythonhosted.org/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", size = 2147590, upload-time = "2026-05-06T13:39:29.883Z" }, + { url = "https://files.pythonhosted.org/packages/11/cb/428de0385b6c8d44b716feba566abfacfbd23ee3c4439faa789a1456242f/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0", size = 2112782, upload-time = "2026-05-06T13:37:04.016Z" }, + { url = "https://files.pythonhosted.org/packages/0b/b5/6a17bdadd0fc1f170adfd05a20d37c832f52b117b4d9131da1f41bb097ce/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7", size = 1952146, upload-time = "2026-05-06T13:39:43.092Z" }, + { url = "https://files.pythonhosted.org/packages/2a/dc/03734d80e362cd43ef65428e9de77c730ce7f2f11c60d2b1e1b39f0fbf99/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2", size = 2134492, upload-time = "2026-05-06T13:36:58.124Z" }, + { url = "https://files.pythonhosted.org/packages/de/df/5e5ffc085ed07cc22d298134d3d911c63e91f6a0eb91fe646750a3209910/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9", size = 2156604, upload-time = "2026-05-06T13:37:49.88Z" }, + { url = "https://files.pythonhosted.org/packages/81/44/6e112a4253e56f5705467cbab7ab5e91ee7398ba3d56d358635958893d3e/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf", size = 2183828, upload-time = "2026-05-06T13:37:43.053Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ad/5565071e937d8e752842ac241463944c9eb14c87e2d269f2658a5bd05e98/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30", size = 2310000, upload-time = "2026-05-06T13:37:56.694Z" }, + { url = "https://files.pythonhosted.org/packages/4f/c3/66883a5cec183e7fba4d024b4cbbe61851a63750ef606b0afecc46d1f2bf/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc", size = 2361286, upload-time = "2026-05-06T13:40:05.667Z" }, + { url = "https://files.pythonhosted.org/packages/4b/2d/69abac8f838090bbecd5df894befb2c2619e7996a98ddb949db9f3b93225/pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983", size = 2193071, upload-time = "2026-05-06T13:38:08.682Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "sqlglot" +version = "30.14.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3a/cd/39a94f0f98076ee8e7c7c38fd4bba8d7845b0c629ff967057c64ef2c0989/sqlglot-30.14.0.tar.gz", hash = "sha256:df2ef5d2b8ca814313781f4ff35bf63e58f821ef517eeddbd523c19a61fa9bb9", size = 5944410, upload-time = "2026-07-27T11:23:30.698Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/ec/a729883ceda22dcd9117ce182f64d884bf494e72c4dfce00c2ad0a5978e1/sqlglot-30.14.0-py3-none-any.whl", hash = "sha256:fc768e24889d63a5e1237dea7ad305e5ffb4356a98b0bed828f89591ebcd3636", size = 719007, upload-time = "2026-07-27T11:23:28.637Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, +] diff --git a/core-spec/expression_language.md b/core-spec/expression_language.md index 42299977..8279178f 100644 --- a/core-spec/expression_language.md +++ b/core-spec/expression_language.md @@ -682,65 +682,67 @@ This section maps Ossie standard functions to their equivalents in popular BI to ### Aggregation Function Mapping -| Ossie Standard | Tableau | Looker Studio | DAX | -| :---- | :---- | :---- | :---- | -| `SUM(x)` | `SUM(x)` | `SUM(X)` | `SUM(x)` | -| `COUNT(x)` | `COUNT(x)` | `COUNT(X)` | `COUNT(x)` | -| `COUNT(DISTINCT x)` | `COUNTD(x)` | `COUNT_DISTINCT(X)` | `DISTINCTCOUNT(x)` | -| `AVG(x)` | `AVG(x)` | `AVG(X)` | `AVERAGE(x)` | -| `MIN(x)` | `MIN(x)` | `MIN(X)` | `MIN(x)` | -| `MAX(x)` | `MAX(x)` | `MAX(X)` | `MAX(x)` | -| `STDDEV(x)` | `STDEV(x)` | `STDDEV(X)` | `STDEV.S(x)` | -| `STDDEV_POP(x)` | `STDEVP(x)` | `STDDEV(X)` | `STDEV.P(x)` | -| `VARIANCE(x)` | `VAR(x)` | `VARIANCE(X)` | `VAR.S(x)` | -| `MEDIAN(x)` | `MEDIAN(x)` | `MEDIAN(X)` | `MEDIAN(x)` | -| `PERCENTILE_CONT(x, 0.75)` | `PERCENTILE(x, 0.75)` | `PERCENTILE(X, 75)` | `PERCENTILE.INC(x, 0.75)` | +| Ossie Standard | Tableau | Looker Studio | DAX | Sigma | +| :---- | :---- | :---- | :---- | :---- | +| `SUM(x)` | `SUM(x)` | `SUM(X)` | `SUM(x)` | `Sum(x)` | +| `COUNT(x)` | `COUNT(x)` | `COUNT(X)` | `COUNT(x)` | `Count(x)` | +| `COUNT(DISTINCT x)` | `COUNTD(x)` | `COUNT_DISTINCT(X)` | `DISTINCTCOUNT(x)` | `CountDistinct(x)` | +| `AVG(x)` | `AVG(x)` | `AVG(X)` | `AVERAGE(x)` | `Avg(x)` | +| `MIN(x)` | `MIN(x)` | `MIN(X)` | `MIN(x)` | `Min(x)` | +| `MAX(x)` | `MAX(x)` | `MAX(X)` | `MAX(x)` | `Max(x)` | +| `STDDEV(x)` | `STDEV(x)` | `STDDEV(X)` | `STDEV.S(x)` | `StdDev(x)` | +| `STDDEV_POP(x)` | `STDEVP(x)` | `STDDEV(X)` | `STDEV.P(x)` | N/A | +| `VARIANCE(x)` | `VAR(x)` | `VARIANCE(X)` | `VAR.S(x)` | `Variance(x)` | +| `MEDIAN(x)` | `MEDIAN(x)` | `MEDIAN(X)` | `MEDIAN(x)` | `Median(x)` | +| `PERCENTILE_CONT(x, 0.75)` | `PERCENTILE(x, 0.75)` | `PERCENTILE(X, 75)` | `PERCENTILE.INC(x, 0.75)` | `Percentile(x, 0.75)` | ### Date Function Mapping -| Ossie Standard | Tableau | Looker Studio | DAX | -| :---- | :---- | :---- | :---- | -| `YEAR(d)` | `YEAR(d)` | `YEAR(Date)` | `YEAR(d)` | -| `MONTH(d)` | `MONTH(d)` | `MONTH(Date)` | `MONTH(d)` | -| `DAY(d)` | `DAY(d)` | `DAY(Date)` | `DAY(d)` | -| `DATE_TRUNC('month', d)` | `DATETRUNC('month', d)` | `TODATE(d, "YYYYMM01", "YYYYMMDD")` | `DATE(YEAR(d), MONTH(d), 1)` | -| `DATEADD(day, n, d)` | `DATEADD('day', n, d)` | `DATE_ADD(d, n)` (days only) | `DATE(d) + n` or `DATEADD(d, n, DAY)` | -| `DATEDIFF(day, d1, d2)` | `DATEDIFF('day', d1, d2)` | `DATE_DIFF(d1, d2)` | `DATEDIFF(d1, d2, DAY)` | -| `CURRENT_DATE` | `TODAY()` | `TODAY()` | `TODAY()` | +| Ossie Standard | Tableau | Looker Studio | DAX | Sigma | +| :---- | :---- | :---- | :---- | :---- | +| `YEAR(d)` | `YEAR(d)` | `YEAR(Date)` | `YEAR(d)` | `Year(d)` | +| `MONTH(d)` | `MONTH(d)` | `MONTH(Date)` | `MONTH(d)` | `Month(d)` | +| `DAY(d)` | `DAY(d)` | `DAY(Date)` | `DAY(d)` | `Day(d)` | +| `DATE_TRUNC('month', d)` | `DATETRUNC('month', d)` | `TODATE(d, "YYYYMM01", "YYYYMMDD")` | `DATE(YEAR(d), MONTH(d), 1)` | N/A — no direct equivalent; Sigma's UI-driven date bucketing is not addressable as a formula argument | +| `DATEADD(day, n, d)` | `DATEADD('day', n, d)` | `DATE_ADD(d, n)` (days only) | `DATE(d) + n` or `DATEADD(d, n, DAY)` | `DateAdd(d, n, "day")` | +| `DATEDIFF(day, d1, d2)` | `DATEDIFF('day', d1, d2)` | `DATE_DIFF(d1, d2)` | `DATEDIFF(d1, d2, DAY)` | `DateDiff(d1, d2, "day")` | +| `CURRENT_DATE` | `TODAY()` | `TODAY()` | `TODAY()` | `Today()` | ### String Function Mapping -| Ossie Standard | Tableau | Looker Studio | DAX | -| :---- | :---- | :---- | :---- | -| `CONCAT(a, b)` | `a + b` | `CONCAT(X, Y)` | `CONCATENATE(a, b)` or `a & b` | -| `LENGTH(s)` | `LEN(s)` | `LENGTH(X)` | `LEN(s)` | -| `LOWER(s)` | `LOWER(s)` | `LOWER(X)` | `LOWER(s)` | -| `UPPER(s)` | `UPPER(s)` | `UPPER(X)` | `UPPER(s)` | -| `TRIM(s)` | `TRIM(s)` | `TRIM(X)` | `TRIM(s)` | -| `LEFT(s, n)` | `LEFT(s, n)` | `LEFT_TEXT(X, n)` | `LEFT(s, n)` | -| `RIGHT(s, n)` | `RIGHT(s, n)` | `RIGHT_TEXT(X, n)` | `RIGHT(s, n)` | -| `SUBSTRING(s, start, len)` | `MID(s, start, len)` | `SUBSTR(X, start, len)` | `MID(s, start, len)` | -| `REPLACE(s, from, to)` | `REPLACE(s, from, to)` | `REPLACE(X, Y, Z)` | `SUBSTITUTE(s, from, to)` | -| `CONTAINS(s, sub)` | `CONTAINS(s, sub)` | `CONTAINS_TEXT(X, text)` | `CONTAINSSTRING(s, sub)` | +| Ossie Standard | Tableau | Looker Studio | DAX | Sigma | +| :---- | :---- | :---- | :---- | :---- | +| `CONCAT(a, b)` | `a + b` | `CONCAT(X, Y)` | `CONCATENATE(a, b)` or `a & b` | `Concat(a, b)` or `a & b` | +| `LENGTH(s)` | `LEN(s)` | `LENGTH(X)` | `LEN(s)` | `Length(s)` | +| `LOWER(s)` | `LOWER(s)` | `LOWER(X)` | `LOWER(s)` | `Lower(s)` | +| `UPPER(s)` | `UPPER(s)` | `UPPER(X)` | `UPPER(s)` | `Upper(s)` | +| `TRIM(s)` | `TRIM(s)` | `TRIM(X)` | `TRIM(s)` | `Trim(s)` | +| `LEFT(s, n)` | `LEFT(s, n)` | `LEFT_TEXT(X, n)` | `LEFT(s, n)` | `Left(s, n)` | +| `RIGHT(s, n)` | `RIGHT(s, n)` | `RIGHT_TEXT(X, n)` | `RIGHT(s, n)` | `Right(s, n)` | +| `SUBSTRING(s, start, len)` | `MID(s, start, len)` | `SUBSTR(X, start, len)` | `MID(s, start, len)` | `Mid(s, start, len)` | +| `REPLACE(s, from, to)` | `REPLACE(s, from, to)` | `REPLACE(X, Y, Z)` | `SUBSTITUTE(s, from, to)` | `Replace(s, from, to)` | +| `CONTAINS(s, sub)` | `CONTAINS(s, sub)` | `CONTAINS_TEXT(X, text)` | `CONTAINSSTRING(s, sub)` | `Contains(s, sub)` | ### Conditional Function Mapping -| Ossie Standard | Tableau | Looker Studio | DAX | -| :---- | :---- | :---- | :---- | -| `CASE WHEN...` | `CASE WHEN...` or `IF...` | `CASE WHEN...` | `SWITCH(TRUE(), ...)` | -| `IF(cond, t, f)` | `IF cond THEN t ELSE f END` | N/A (use CASE) | `IF(cond, t, f)` | -| `COALESCE(a, b)` | `IFNULL(a, b)` or `ZN(a)` | `COALESCE(...)` | `COALESCE(a, b)` | -| `NULLIF(a, b)` | `IF a = b THEN NULL ELSE a END` | N/A | `IF(a = b, BLANK(), a)` | +| Ossie Standard | Tableau | Looker Studio | DAX | Sigma | +| :---- | :---- | :---- | :---- | :---- | +| `CASE WHEN...` | `CASE WHEN...` or `IF...` | `CASE WHEN...` | `SWITCH(TRUE(), ...)` | Only 3-argument `If(cond, t, f)`; no native multi-branch `CASE` | +| `IF(cond, t, f)` | `IF cond THEN t ELSE f END` | N/A (use CASE) | `IF(cond, t, f)` | `If(cond, t, f)` | +| `COALESCE(a, b)` | `IFNULL(a, b)` or `ZN(a)` | `COALESCE(...)` | `COALESCE(a, b)` | `IfNull(a, b)` (2-argument only) | +| `NULLIF(a, b)` | `IF a = b THEN NULL ELSE a END` | N/A | `IF(a = b, BLANK(), a)` | `If(a = b, Null(), a)` | ### Window Function Mapping -| Ossie Standard | Tableau | Looker Studio | DAX | -| :---- | :---- | :---- | :---- | -| `ROW_NUMBER() OVER(...)` | `INDEX()` | N/A | `RANKX(...)` with DENSE | -| `RANK() OVER(...)` | `RANK(expr)` | N/A | `RANKX(...)` | -| `SUM(...) OVER(PARTITION BY...)` | `{FIXED [...]: SUM(...)}` | N/A (blending only) | Context-dependent | -| `LAG(x, 1) OVER(ORDER BY...)` | `LOOKUP(x, -1)` | N/A | `CALCULATE(x, PREVIOUSDAY(...))` | -| `RUNNING_SUM(...)` | `RUNNING_SUM(SUM(...))` | N/A | `CALCULATE(SUM(...), FILTER(...))` | +| Ossie Standard | Tableau | Looker Studio | DAX | Sigma | +| :---- | :---- | :---- | :---- | :---- | +| `ROW_NUMBER() OVER(...)` | `INDEX()` | N/A | `RANKX(...)` with DENSE | `RowNumber()` — but partition/order come from UI table-calculation configuration, not formula arguments, so it has no portable expression form | +| `RANK() OVER(...)` | `RANK(expr)` | N/A | `RANKX(...)` | `Rank()` — same UI-configuration caveat as `RowNumber()` | +| `SUM(...) OVER(PARTITION BY...)` | `{FIXED [...]: SUM(...)}` | N/A (blending only) | Context-dependent | `RunningSum(...)`/`RunningAvg(...)` — same UI-configuration caveat | +| `LAG(x, 1) OVER(ORDER BY...)` | `LOOKUP(x, -1)` | N/A | `CALCULATE(x, PREVIOUSDAY(...))` | `Lag(x, 1)` — same UI-configuration caveat | +| `RUNNING_SUM(...)` | `RUNNING_SUM(SUM(...))` | N/A | `CALCULATE(SUM(...), FILTER(...))` | `RunningSum(...)` — same UI-configuration caveat | + +Sigma's table-calculation functions (`RowNumber`, `Rank`, `RunningSum`, `RunningAvg`, `Lag`, `Lead`, etc.) resolve their partition/order context from workbook UI configuration (which pivot/table the calculation is attached to) rather than from arguments passed in the formula text itself. Because that context isn't recoverable from the formula string alone, the Sigma converter (`converters/sigma/`) treats these as untranslatable to ANSI SQL and carries the original Sigma formula through in the `SIGMA` dialect only — see `converters/sigma/LIMITATIONS.md`. --- diff --git a/core-spec/osi-schema.json b/core-spec/osi-schema.json index f24e45f1..79ca055e 100644 --- a/core-spec/osi-schema.json +++ b/core-spec/osi-schema.json @@ -16,6 +16,20 @@ "items": { "$ref": "#/$defs/SemanticModel" } + }, + "dialects": { + "type": "array", + "description": "Dialects used anywhere in this document", + "items": { + "$ref": "#/$defs/Dialect" + } + }, + "vendors": { + "type": "array", + "description": "Vendors with custom_extensions present anywhere in this document", + "items": { + "$ref": "#/$defs/Vendor" + } } }, "required": ["version", "semantic_model"], @@ -23,7 +37,7 @@ "$defs": { "Dialect": { "type": "string", - "enum": ["ANSI_SQL", "SNOWFLAKE", "MDX", "TABLEAU", "DATABRICKS", "MAQL", "BIGQUERY"], + "enum": ["ANSI_SQL", "SNOWFLAKE", "MDX", "TABLEAU", "DATABRICKS", "MAQL", "BIGQUERY", "SIGMA"], "description": "Supported SQL and expression language dialects" }, "Vendor": { diff --git a/core-spec/spec.md b/core-spec/spec.md index 156cb1db..c9d1f9a3 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -58,6 +58,7 @@ Supported SQL and expression language dialects for metrics and field definitions | `DATABRICKS` | Databricks SQL | | `MAQL` | GoodData MAQL (Metric Analysis and Query Language) | | `BIGQUERY` | Google BigQuery (GoogleSQL) | +| `SIGMA` | Sigma Computing's spreadsheet-style formula language | ### Data types @@ -446,6 +447,7 @@ The following are well-known examples: | `GOODDATA` | GoodData-specific attributes | | `HONEYDEW` | Honeydew-specific attributes | | `WISDOM` | WisdomAI-specific attributes | +| `SIGMA` | Sigma Computing-specific attributes | ### Examples diff --git a/python/src/ossie/models.py b/python/src/ossie/models.py index 5406a743..e554d879 100644 --- a/python/src/ossie/models.py +++ b/python/src/ossie/models.py @@ -32,6 +32,7 @@ class OSIDialect(str, Enum): TABLEAU = "TABLEAU" DATABRICKS = "DATABRICKS" BIGQUERY = "BIGQUERY" + SIGMA = "SIGMA" class OSIDataType(str, Enum): @@ -70,6 +71,7 @@ class OSIVendor(str, Enum): GOODDATA = "GOODDATA" SEMANTIDO = "SEMANTIDO" WISDOM = "WISDOM" + SIGMA = "SIGMA" class OSIAIContextObject(BaseModel): diff --git a/validation/validate.py b/validation/validate.py index 258d34f1..4af165bf 100644 --- a/validation/validate.py +++ b/validation/validate.py @@ -69,10 +69,11 @@ "MDX": None, # Not supported by sqlglot, skip validation "TABLEAU": None, # Not supported by sqlglot, skip validation "MAQL": None, # Not supported by sqlglot, skip validation + "SIGMA": None, # Sigma's spreadsheet-style formula language, not SQL; skip validation } # Dialects that sqlglot cannot parse -SKIP_SQL_VALIDATION = {"MDX", "TABLEAU", "MAQL"} +SKIP_SQL_VALIDATION = {"MDX", "TABLEAU", "MAQL", "SIGMA"} def validate_schema(data: dict, schema: dict) -> list[str]: