From f17637f64f51568c0b6f1def80c34596d946fa1b Mon Sep 17 00:00:00 2001 From: Yufei Gu Date: Thu, 2 Jul 2026 16:56:39 -0700 Subject: [PATCH 01/13] Feat: OSI semantic-model core entity and CRUD Implements the design's Phase 0 (spec freeze) and Phase 3 (core entity), turning the 501 scaffolding from #4816 into working create/list/load/update/ drop for OSI semantic models on the JDBC backend. Core (polaris-core): - New SEMANTIC_MODEL(11, NAMESPACE) entity type and SemanticModelEntity, storing the OSI document (spec-version + JSON content) in entity properties, mirroring PolicyEntity. - ResolvedPathKey.ofSemanticModel; SEMANTIC_MODEL_MAX_{DOCUMENT,EXPRESSION}_BYTES feature configs; NoSuchSemanticModel/SemanticModelVersionMismatch exceptions wired into PolarisExceptionMapper (404/409). - Five authorizable operations registered in RbacOperationSemantics, gated on CATALOG_MANAGE_CONTENT as an interim (the dedicated SEMANTIC_MODEL_* privilege matrix and read-time enforcement modes land in the authorization phase). Extension (extensions/semantic-models): - Bundled OSI v0.1.1 JSON Schema and OsiDocumentValidator (strict schema validation with JSON-Pointer errors, document/expression size caps) via networknt json-schema-validator. - SemanticModelCatalog/Handler/Factory: optimistic concurrency on entity-version, write-time dataset.source -> TABLE_LIKE resolution (400 on unresolved), list pagination. Adapter now dispatches to the handler. - Unit tests: OsiDocumentValidatorTest and SemanticModelCatalogTest (plain JUnit + Mockito, following the extensions/auth/opa pattern). NoSQL support is deferred to a follow-up; SEMANTIC_MODEL is excluded from the NoSQL entity-type completeness check so the module keeps loading. --- build.gradle.kts | 4 + extensions/semantic-models/build.gradle.kts | 3 + .../semanticmodel/OsiDocumentValidator.java | 154 ++++++++ .../semanticmodel/SemanticModelCatalog.java | 345 ++++++++++++++++++ .../SemanticModelCatalogAdapter.java | 76 ++-- .../SemanticModelCatalogHandler.java | 141 +++++++ .../SemanticModelCatalogHandlerFactory.java | 47 +++ .../main/resources/osi/v0.1.1/osi-schema.json | 344 +++++++++++++++++ .../OsiDocumentValidatorTest.java | 96 +++++ .../SemanticModelCatalogTest.java | 269 ++++++++++++++ gradle/libs.versions.toml | 1 + .../coretypes/mapping/EntityObjMappings.java | 3 + .../mapping/TestEntityObjMappings.java | 2 + .../auth/PolarisAuthorizableOperation.java | 5 + .../core/auth/RbacOperationSemantics.java | 16 + .../core/config/FeatureConfiguration.java | 18 + .../core/entity/PolarisEntityType.java | 3 +- .../persistence/resolver/ResolvedPathKey.java | 5 + .../core/semantic/SemanticModelEntity.java | 126 +++++++ .../NoSuchSemanticModelException.java | 32 ++ ...SemanticModelVersionMismatchException.java | 35 ++ .../exception/PolarisExceptionMapper.java | 5 + 22 files changed, 1703 insertions(+), 27 deletions(-) create mode 100644 extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/OsiDocumentValidator.java create mode 100644 extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalog.java create mode 100644 extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandler.java create mode 100644 extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandlerFactory.java create mode 100644 extensions/semantic-models/src/main/resources/osi/v0.1.1/osi-schema.json create mode 100644 extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/OsiDocumentValidatorTest.java create mode 100644 extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogTest.java create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/semantic/SemanticModelEntity.java create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/semantic/exceptions/NoSuchSemanticModelException.java create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/semantic/exceptions/SemanticModelVersionMismatchException.java diff --git a/build.gradle.kts b/build.gradle.kts index 9463e128082..ac01108f262 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -87,6 +87,10 @@ tasks.named("rat").configure { // Files copied from OpenAPI Generator (ASLv2 licensed) don't have header excludes.add("server-templates/*.mustache") + // Vendored OSI JSON Schema (Apache-2.0, from open-semantic-interchange/OSI); JSON allows no + // comments so the ASF header cannot be inlined. + excludes.add("extensions/semantic-models/src/main/resources/osi/**") + // Manifest files do not allow comments excludes.add("tools/version/src/jarTest/resources/META-INF/FAKE_MANIFEST.MF") diff --git a/extensions/semantic-models/build.gradle.kts b/extensions/semantic-models/build.gradle.kts index ff03fd9da01..dad70b7b3ee 100644 --- a/extensions/semantic-models/build.gradle.kts +++ b/extensions/semantic-models/build.gradle.kts @@ -56,6 +56,9 @@ dependencies { implementation(libs.jakarta.servlet.api) implementation(libs.jakarta.ws.rs.api) + // Validates OSI documents against the bundled OSI JSON Schema (draft 2020-12). + implementation(libs.json.schema.validator) + implementation(platform(libs.micrometer.bom)) implementation("io.micrometer:micrometer-core") diff --git a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/OsiDocumentValidator.java b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/OsiDocumentValidator.java new file mode 100644 index 00000000000..12602a59cfc --- /dev/null +++ b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/OsiDocumentValidator.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.catalog.semanticmodel; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.base.Utf8; +import com.networknt.schema.JsonSchema; +import com.networknt.schema.JsonSchemaFactory; +import com.networknt.schema.SpecVersion; +import com.networknt.schema.ValidationMessage; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.iceberg.exceptions.BadRequestException; + +/** + * Validates an OSI (Open Semantic Interchange) semantic-model document against the bundled OSI JSON + * Schema and against Polaris's configured size caps. + * + *

The Polaris API represents a document as a {@code version} string plus a {@code + * semantic_model} JSON string (see the {@code SemanticModelDocument} API type). This validator + * reconstructs the canonical OSI document {@code {"version": ..., "semantic_model": ...}} and + * validates it against the frozen v0.1.1 schema (draft 2020-12), which is strict: unknown fields + * fail because the schema declares {@code additionalProperties: false}. + * + *

Version evolution is an explicit user action: documents declaring an unsupported OSI spec + * version are rejected rather than silently accepted or migrated. + */ +public final class OsiDocumentValidator { + + /** The single OSI spec version this build bundles and supports. */ + public static final String SUPPORTED_SPEC_VERSION = "0.1.1"; + + private static final String SCHEMA_RESOURCE = "/osi/v0.1.1/osi-schema.json"; + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final JsonSchema SCHEMA = loadSchema(); + + private OsiDocumentValidator() {} + + private static JsonSchema loadSchema() { + try (InputStream in = OsiDocumentValidator.class.getResourceAsStream(SCHEMA_RESOURCE)) { + if (in == null) { + throw new IllegalStateException("Bundled OSI schema not found: " + SCHEMA_RESOURCE); + } + JsonSchemaFactory factory = JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V202012); + return factory.getSchema(in); + } catch (java.io.IOException e) { + throw new UncheckedIOException("Failed to load bundled OSI schema", e); + } + } + + /** + * Validates the given OSI document. Throws {@link BadRequestException} (HTTP 400) with + * JSON-Pointer field paths on any schema or size-cap failure. + * + * @param specVersion the declared OSI spec version + * @param semanticModelJson the OSI {@code semantic_model} content, serialized as a JSON string + * @param maxDocumentBytes maximum allowed size, in UTF-8 bytes, of the semantic-model content + * @param maxExpressionBytes maximum allowed size, in UTF-8 bytes, of any single expression + * @return the parsed OSI {@code semantic_model} node, for downstream source resolution + */ + public static JsonNode validate( + String specVersion, String semanticModelJson, int maxDocumentBytes, int maxExpressionBytes) { + if (!SUPPORTED_SPEC_VERSION.equals(specVersion)) { + throw new BadRequestException( + "Unsupported OSI spec version '%s'; this server supports '%s'", + specVersion, SUPPORTED_SPEC_VERSION); + } + + if (semanticModelJson == null || semanticModelJson.isBlank()) { + throw new BadRequestException("Semantic model document must not be empty"); + } + + int documentBytes = Utf8.encodedLength(semanticModelJson); + if (documentBytes > maxDocumentBytes) { + throw new BadRequestException( + "Semantic model document of %d bytes exceeds the maximum of %d bytes", + documentBytes, maxDocumentBytes); + } + + JsonNode semanticModelNode; + try { + semanticModelNode = MAPPER.readTree(semanticModelJson); + } catch (com.fasterxml.jackson.core.JsonProcessingException e) { + throw new BadRequestException( + "Field 'semantic_model' is not valid JSON: %s", e.getOriginalMessage()); + } + + // Reconstruct the canonical OSI document {version, semantic_model} for schema validation. + ObjectNode osiDocument = MAPPER.createObjectNode(); + osiDocument.put("version", specVersion); + osiDocument.set("semantic_model", semanticModelNode); + + Set errors = SCHEMA.validate(osiDocument); + if (!errors.isEmpty()) { + String details = + errors.stream() + .map(m -> m.getInstanceLocation() + ": " + m.getMessage()) + .sorted() + .collect(Collectors.joining("; ")); + throw new BadRequestException( + "Semantic model document failed OSI schema validation: %s", details); + } + + enforceExpressionCap(semanticModelNode, maxExpressionBytes); + + return semanticModelNode; + } + + /** + * Walks the document for SQL/expression fragments (the {@code expression} string inside each + * {@code dialects[]} entry) and rejects any that exceed the per-expression cap. + */ + private static void enforceExpressionCap(JsonNode node, int maxExpressionBytes) { + if (node.isObject()) { + JsonNode dialects = node.get("dialects"); + if (dialects != null && dialects.isArray()) { + for (JsonNode dialect : dialects) { + JsonNode expr = dialect.get("expression"); + if (expr != null && expr.isTextual()) { + int bytes = Utf8.encodedLength(expr.asText()); + if (bytes > maxExpressionBytes) { + throw new BadRequestException( + "An expression of %d bytes exceeds the maximum of %d bytes", + bytes, maxExpressionBytes); + } + } + } + } + node.forEach(child -> enforceExpressionCap(child, maxExpressionBytes)); + } else if (node.isArray()) { + node.forEach(child -> enforceExpressionCap(child, maxExpressionBytes)); + } + } +} diff --git a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalog.java b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalog.java new file mode 100644 index 00000000000..274277e1805 --- /dev/null +++ b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalog.java @@ -0,0 +1,345 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.catalog.semanticmodel; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.base.Splitter; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.AlreadyExistsException; +import org.apache.iceberg.exceptions.BadRequestException; +import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.entity.CatalogEntity; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisEntity; +import org.apache.polaris.core.entity.PolarisEntitySubType; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.persistence.PolarisMetaStoreManager; +import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; +import org.apache.polaris.core.persistence.dao.entity.EntityResult; +import org.apache.polaris.core.persistence.dao.entity.ListEntitiesResult; +import org.apache.polaris.core.persistence.pagination.PageToken; +import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifestCatalogView; +import org.apache.polaris.core.persistence.resolver.ResolvedPathKey; +import org.apache.polaris.core.semantic.SemanticModelEntity; +import org.apache.polaris.core.semantic.exceptions.NoSuchSemanticModelException; +import org.apache.polaris.core.semantic.exceptions.SemanticModelVersionMismatchException; +import org.apache.polaris.service.catalog.semanticmodel.types.ListSemanticModelsResponse; +import org.apache.polaris.service.catalog.semanticmodel.types.LoadSemanticModelResponse; +import org.apache.polaris.service.catalog.semanticmodel.types.SemanticModelDocument; +import org.apache.polaris.service.catalog.semanticmodel.types.SemanticModelIdentifier; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Core create/list/load/update/drop logic for OSI semantic models. Mirrors {@link + * org.apache.polaris.service.catalog.policy.PolicyCatalog}: the OSI document body is stored inside + * the entity {@code properties} map, writes validate the document and resolve every {@code + * dataset.source} to a {@code TABLE_LIKE} entity in the current catalog, and updates use optimistic + * concurrency on the entity version. + */ +public class SemanticModelCatalog { + private static final Logger LOGGER = LoggerFactory.getLogger(SemanticModelCatalog.class); + + private final CallContext callContext; + private final PolarisResolutionManifestCatalogView resolvedEntityView; + private final CatalogEntity catalogEntity; + private final long catalogId; + private final PolarisMetaStoreManager metaStoreManager; + private final int maxDocumentBytes; + private final int maxExpressionBytes; + + public SemanticModelCatalog( + PolarisMetaStoreManager metaStoreManager, + CallContext callContext, + PolarisResolutionManifestCatalogView resolvedEntityView, + int maxDocumentBytes, + int maxExpressionBytes) { + this.callContext = callContext; + this.resolvedEntityView = resolvedEntityView; + this.catalogEntity = resolvedEntityView.getResolvedCatalogEntity(); + this.catalogId = catalogEntity.getId(); + this.metaStoreManager = metaStoreManager; + this.maxDocumentBytes = maxDocumentBytes; + this.maxExpressionBytes = maxExpressionBytes; + } + + public LoadSemanticModelResponse createSemanticModel( + SemanticModelIdentifier identifier, SemanticModelDocument document) { + Namespace namespace = toNamespace(identifier); + PolarisResolvedPathWrapper resolvedParent = + resolvedEntityView.getResolvedPath(ResolvedPathKey.ofNamespace(namespace)); + if (resolvedParent == null) { + // Illegal state because the namespace should've already been in the static resolution set. + throw new IllegalStateException( + String.format("Failed to fetch resolved parent for semantic model '%s'", identifier)); + } + List catalogPath = resolvedParent.getRawFullPath(); + + PolarisResolvedPathWrapper existing = + resolvedEntityView.getPassthroughResolvedPath( + ResolvedPathKey.ofSemanticModel(namespace, identifier.getName()), + PolarisEntitySubType.NULL_SUBTYPE); + if (existing != null && existing.getRawLeafEntity() != null) { + throw new AlreadyExistsException("Semantic model already exists: %s", identifier.getName()); + } + + // Validate against the bundled OSI schema and size caps, then resolve source tables. + validateDocumentAndSources(document); + + SemanticModelEntity entity = + new SemanticModelEntity.Builder(namespace, identifier.getName()) + .setCatalogId(catalogId) + .setParentId(resolvedParent.getRawLeafEntity().getId()) + .setSpecVersion(document.getVersion()) + .setContent(document.getSemanticModel()) + .setId( + metaStoreManager.generateNewEntityId(callContext.getPolarisCallContext()).getId()) + .setCreateTimestamp(System.currentTimeMillis()) + .build(); + + EntityResult res = + metaStoreManager.createEntityIfNotExists( + callContext.getPolarisCallContext(), PolarisEntity.toCoreList(catalogPath), entity); + if (!res.isSuccess()) { + switch (res.getReturnStatus()) { + case ENTITY_ALREADY_EXISTS -> + throw new AlreadyExistsException( + "Semantic model already exists: %s", identifier.getName()); + default -> + throw new IllegalStateException( + String.format( + "Unknown error status for identifier %s: %s with extraInfo: %s", + identifier, res.getReturnStatus(), res.getExtraInformation())); + } + } + + SemanticModelEntity result = SemanticModelEntity.of(res.getEntity()); + LOGGER.debug("Created semantic model entity {} with identifier {}", result, identifier); + return toLoadResponse(result); + } + + public ListSemanticModelsResponse listSemanticModels(Namespace namespace, PageToken pageToken) { + PolarisResolvedPathWrapper resolvedEntities = + resolvedEntityView.getResolvedPath(ResolvedPathKey.ofNamespace(namespace)); + if (resolvedEntities == null) { + throw new IllegalStateException( + String.format("Failed to fetch resolved namespace '%s'", namespace)); + } + List catalogPath = resolvedEntities.getRawFullPath(); + + ListEntitiesResult result = + metaStoreManager.listEntities( + callContext.getPolarisCallContext(), + PolarisEntity.toCoreList(catalogPath), + PolarisEntityType.SEMANTIC_MODEL, + PolarisEntitySubType.NULL_SUBTYPE, + pageToken); + if (!result.isSuccess()) { + throw new IllegalStateException("Failed to list semantic models in namespace: " + namespace); + } + + // LinkedHashSet keeps the page order stable in the serialized response. + Set identifiers = + result.getEntities().stream() + .map( + record -> + SemanticModelIdentifier.builder() + .setNamespace(List.of(namespace.levels())) + .setName(record.getName()) + .build()) + .collect(Collectors.toCollection(LinkedHashSet::new)); + return ListSemanticModelsResponse.builder() + .setIdentifiers(identifiers) + .setNextPageToken(result.getPage().encodedResponseToken()) + .build(); + } + + public LoadSemanticModelResponse loadSemanticModel(SemanticModelIdentifier identifier) { + return toLoadResponse(resolveModelOrThrow(identifier)); + } + + public LoadSemanticModelResponse updateSemanticModel( + SemanticModelIdentifier identifier, SemanticModelDocument document, String expectedVersion) { + PolarisResolvedPathWrapper resolvedPath = resolveModelPathOrThrow(identifier); + SemanticModelEntity current = SemanticModelEntity.of(resolvedPath.getRawLeafEntity()); + + String currentVersion = Integer.toString(current.getEntityVersion()); + if (!currentVersion.equals(expectedVersion)) { + throw new SemanticModelVersionMismatchException( + String.format( + "Semantic model version mismatch. Given version is %s, current version is %s", + expectedVersion, currentVersion)); + } + + validateDocumentAndSources(document); + + SemanticModelEntity newEntity = + new SemanticModelEntity.Builder(current) + .setSpecVersion(document.getVersion()) + .setContent(document.getSemanticModel()) + .build(); + + List catalogPath = resolvedPath.getRawParentPath(); + SemanticModelEntity updated = + Optional.ofNullable( + metaStoreManager + .updateEntityPropertiesIfNotChanged( + callContext.getPolarisCallContext(), + PolarisEntity.toCoreList(catalogPath), + newEntity) + .getEntity()) + .map(SemanticModelEntity::of) + .orElse(null); + if (updated == null) { + // Lost the optimistic-concurrency race with a concurrent writer. + throw new SemanticModelVersionMismatchException( + String.format( + "Semantic model %s was modified concurrently; retry after reloading", + identifier.getName())); + } + + return toLoadResponse(updated); + } + + public void dropSemanticModel(SemanticModelIdentifier identifier) { + PolarisResolvedPathWrapper resolvedPath = resolveModelPathOrThrow(identifier); + List catalogPath = resolvedPath.getRawParentPath(); + PolarisBaseEntity entity = resolvedPath.getRawLeafEntity(); + + var result = + metaStoreManager.dropEntityIfExists( + callContext.getPolarisCallContext(), + PolarisEntity.toCoreList(catalogPath), + entity, + java.util.Map.of(), + false); + if (!result.isSuccess()) { + throw new IllegalStateException( + String.format( + "Failed to drop semantic model %s error status: %s with extraInfo: %s", + identifier, result.getReturnStatus(), result.getExtraInformation())); + } + } + + // ---- helpers ---- + + private PolarisResolvedPathWrapper resolveModelPathOrThrow(SemanticModelIdentifier identifier) { + Namespace namespace = toNamespace(identifier); + PolarisResolvedPathWrapper resolved = + resolvedEntityView.getPassthroughResolvedPath( + ResolvedPathKey.ofSemanticModel(namespace, identifier.getName()), + PolarisEntitySubType.NULL_SUBTYPE); + if (resolved == null || resolved.getRawLeafEntity() == null) { + throw new NoSuchSemanticModelException( + String.format("Semantic model does not exist: %s", identifier.getName())); + } + return resolved; + } + + private SemanticModelEntity resolveModelOrThrow(SemanticModelIdentifier identifier) { + return SemanticModelEntity.of(resolveModelPathOrThrow(identifier).getRawLeafEntity()); + } + + /** + * Validates the document against the bundled OSI schema and size caps, then resolves every {@code + * dataset.source} to a {@code TABLE_LIKE} entity. Shared by create and update. + */ + private void validateDocumentAndSources(SemanticModelDocument document) { + JsonNode semanticModel = + OsiDocumentValidator.validate( + document.getVersion(), + document.getSemanticModel(), + maxDocumentBytes, + maxExpressionBytes); + resolveSourcesOrThrow(semanticModel); + } + + /** + * Resolves every {@code dataset.source} in the OSI document to a {@code TABLE_LIKE} entity in the + * current catalog. Fails with 400 and a JSON-Pointer to the offending dataset if any source does + * not resolve. Column-level checks are deferred (F5). + */ + private void resolveSourcesOrThrow(JsonNode semanticModel) { + if (!semanticModel.isArray()) { + return; + } + for (int m = 0; m < semanticModel.size(); m++) { + JsonNode datasets = semanticModel.get(m).get("datasets"); + if (datasets == null || !datasets.isArray()) { + continue; + } + for (int d = 0; d < datasets.size(); d++) { + JsonNode source = datasets.get(d).get("source"); + if (source != null && source.isTextual()) { + String pointer = String.format("/semantic_model/%d/datasets/%d/source", m, d); + resolveSourceOrThrow(source.asText(), pointer); + } + } + } + } + + private void resolveSourceOrThrow(String source, String pointer) { + TableIdentifier tableIdentifier = parseSource(source, pointer); + PolarisResolvedPathWrapper resolved = + resolvedEntityView.getPassthroughResolvedPath( + ResolvedPathKey.ofTableLike(tableIdentifier), PolarisEntitySubType.ANY_SUBTYPE); + if (resolved == null + || resolved.getRawLeafEntity() == null + || resolved.getRawLeafEntity().getType() != PolarisEntityType.TABLE_LIKE) { + throw new BadRequestException( + "Semantic model source '%s' at %s does not resolve to a table or view in catalog '%s'", + source, pointer, catalogEntity.getName()); + } + } + + private TableIdentifier parseSource(String source, String pointer) { + List parts = Splitter.on('.').splitToList(source); + if (parts.size() < 2 || parts.stream().anyMatch(String::isEmpty)) { + throw new BadRequestException( + "Semantic model source '%s' at %s must be a namespace-qualified identifier " + + "'.'", + source, pointer); + } + String name = parts.get(parts.size() - 1); + String[] levels = parts.subList(0, parts.size() - 1).toArray(new String[0]); + return TableIdentifier.of(Namespace.of(levels), name); + } + + private static Namespace toNamespace(SemanticModelIdentifier identifier) { + return Namespace.of(identifier.getNamespace().toArray(new String[0])); + } + + private static LoadSemanticModelResponse toLoadResponse(SemanticModelEntity entity) { + SemanticModelDocument document = + SemanticModelDocument.builder() + .setVersion(entity.getSpecVersion()) + .setSemanticModel(entity.getContent()) + .build(); + return LoadSemanticModelResponse.builder() + .setDocument(document) + .setEntityVersion(Integer.toString(entity.getEntityVersion())) + .build(); + } +} diff --git a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogAdapter.java b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogAdapter.java index a569a120370..38fbf099d74 100644 --- a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogAdapter.java +++ b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogAdapter.java @@ -22,39 +22,57 @@ import jakarta.inject.Inject; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.SecurityContext; +import java.util.List; +import org.apache.iceberg.catalog.Namespace; +import org.apache.polaris.core.auth.PolarisPrincipal; import org.apache.polaris.core.config.FeatureConfiguration; import org.apache.polaris.core.config.RealmConfig; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.rest.NamespaceUtils; +import org.apache.polaris.service.catalog.CatalogPrefixParser; import org.apache.polaris.service.catalog.common.CatalogAdapter; import org.apache.polaris.service.catalog.semanticmodel.api.PolarisCatalogSemanticModelApiService; import org.apache.polaris.service.catalog.semanticmodel.types.CreateSemanticModelRequest; +import org.apache.polaris.service.catalog.semanticmodel.types.SemanticModelIdentifier; import org.apache.polaris.service.catalog.semanticmodel.types.UpdateSemanticModelRequest; /** - * Stub adapter for the OSI semantic-model API. The endpoints are wired and gated by {@link - * FeatureConfiguration#ENABLE_SEMANTIC_MODELS}, but every operation returns {@code 501 Not - * Implemented}. Persistence, validation, authorization, and source-link resolution land in - * subsequent phases. + * Adapter for the OSI semantic-model API. The endpoints are gated by {@link + * FeatureConfiguration#ENABLE_SEMANTIC_MODELS} and dispatch to {@link SemanticModelCatalogHandler} + * for authorization, validation, source resolution, and persistence. */ @RequestScoped public class SemanticModelCatalogAdapter implements PolarisCatalogSemanticModelApiService, CatalogAdapter { private final RealmConfig realmConfig; + private final CatalogPrefixParser prefixParser; + private final SemanticModelCatalogHandlerFactory handlerFactory; @Inject - public SemanticModelCatalogAdapter(CallContext callContext) { + public SemanticModelCatalogAdapter( + CallContext callContext, + CatalogPrefixParser prefixParser, + SemanticModelCatalogHandlerFactory handlerFactory) { this.realmConfig = callContext.getRealmConfig(); + this.prefixParser = prefixParser; + this.handlerFactory = handlerFactory; } - private void ensureEnabled() { + private SemanticModelCatalogHandler newHandler(SecurityContext securityContext, String prefix) { FeatureConfiguration.enforceFeatureEnabledOrThrow( realmConfig, FeatureConfiguration.ENABLE_SEMANTIC_MODELS); + PolarisPrincipal principal = validatePrincipal(securityContext); + String catalogName = prefixParser.prefixToCatalogName(prefix); + return handlerFactory.createHandler(catalogName, principal); } - private static Response notImplemented() { - return Response.status(Response.Status.NOT_IMPLEMENTED).build(); + private static SemanticModelIdentifier identifier(Namespace namespace, String name) { + return SemanticModelIdentifier.builder() + .setNamespace(List.of(namespace.levels())) + .setName(name) + .build(); } @Override @@ -64,10 +82,10 @@ public Response createSemanticModel( CreateSemanticModelRequest createSemanticModelRequest, RealmContext realmContext, SecurityContext securityContext) { - ensureEnabled(); - // TODO: authorize the principal, validate the OSI document against the bundled OSI JSON Schema, - // then persist a new semantic-model entity and return the stored document. - return notImplemented(); + Namespace ns = + NamespaceUtils.splitNamespace(namespace, NamespaceUtils.DEFAULT_NAMESPACE_SEPARATOR); + SemanticModelCatalogHandler handler = newHandler(securityContext, prefix); + return Response.ok(handler.createSemanticModel(ns, createSemanticModelRequest)).build(); } @Override @@ -78,10 +96,10 @@ public Response listSemanticModels( Integer pageSize, RealmContext realmContext, SecurityContext securityContext) { - ensureEnabled(); - // TODO: authorize the principal, then page through the namespace's semantic-model entities and - // return their identifiers. - return notImplemented(); + Namespace ns = + NamespaceUtils.splitNamespace(namespace, NamespaceUtils.DEFAULT_NAMESPACE_SEPARATOR); + SemanticModelCatalogHandler handler = newHandler(securityContext, prefix); + return Response.ok(handler.listSemanticModels(ns, pageToken, pageSize)).build(); } @Override @@ -91,9 +109,10 @@ public Response loadSemanticModel( String semanticModelName, RealmContext realmContext, SecurityContext securityContext) { - ensureEnabled(); - // TODO: authorize the principal, then read and return the stored OSI document. - return notImplemented(); + Namespace ns = + NamespaceUtils.splitNamespace(namespace, NamespaceUtils.DEFAULT_NAMESPACE_SEPARATOR); + SemanticModelCatalogHandler handler = newHandler(securityContext, prefix); + return Response.ok(handler.loadSemanticModel(identifier(ns, semanticModelName))).build(); } @Override @@ -104,10 +123,13 @@ public Response updateSemanticModel( UpdateSemanticModelRequest updateSemanticModelRequest, RealmContext realmContext, SecurityContext securityContext) { - ensureEnabled(); - // TODO: authorize the principal, validate the OSI document, then replace the stored document - // and return it. - return notImplemented(); + Namespace ns = + NamespaceUtils.splitNamespace(namespace, NamespaceUtils.DEFAULT_NAMESPACE_SEPARATOR); + SemanticModelCatalogHandler handler = newHandler(securityContext, prefix); + return Response.ok( + handler.updateSemanticModel( + identifier(ns, semanticModelName), updateSemanticModelRequest)) + .build(); } @Override @@ -117,8 +139,10 @@ public Response dropSemanticModel( String semanticModelName, RealmContext realmContext, SecurityContext securityContext) { - ensureEnabled(); - // TODO: authorize the principal, then delete the semantic-model entity. - return notImplemented(); + Namespace ns = + NamespaceUtils.splitNamespace(namespace, NamespaceUtils.DEFAULT_NAMESPACE_SEPARATOR); + SemanticModelCatalogHandler handler = newHandler(securityContext, prefix); + handler.dropSemanticModel(identifier(ns, semanticModelName)); + return Response.noContent().build(); } } diff --git a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandler.java b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandler.java new file mode 100644 index 00000000000..8bf75a1e1af --- /dev/null +++ b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandler.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.catalog.semanticmodel; + +import java.util.List; +import org.apache.iceberg.catalog.Namespace; +import org.apache.polaris.core.auth.PolarisAuthorizableOperation; +import org.apache.polaris.core.catalog.PolarisCatalogHelpers; +import org.apache.polaris.core.config.FeatureConfiguration; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; +import org.apache.polaris.core.persistence.pagination.PageToken; +import org.apache.polaris.core.persistence.resolver.ResolvedPathKey; +import org.apache.polaris.core.persistence.resolver.ResolverPath; +import org.apache.polaris.core.semantic.exceptions.NoSuchSemanticModelException; +import org.apache.polaris.immutables.PolarisImmutable; +import org.apache.polaris.service.catalog.common.CatalogHandler; +import org.apache.polaris.service.catalog.semanticmodel.types.CreateSemanticModelRequest; +import org.apache.polaris.service.catalog.semanticmodel.types.ListSemanticModelsResponse; +import org.apache.polaris.service.catalog.semanticmodel.types.LoadSemanticModelResponse; +import org.apache.polaris.service.catalog.semanticmodel.types.SemanticModelIdentifier; +import org.apache.polaris.service.catalog.semanticmodel.types.UpdateSemanticModelRequest; + +/** + * Authorizes and delegates OSI semantic-model operations to {@link SemanticModelCatalog}. Mirrors + * {@link org.apache.polaris.service.catalog.policy.PolicyCatalogHandler}. + * + *

Authorization is intentionally minimal in this phase: operations are gated by the coarse + * {@code CATALOG_MANAGE_CONTENT} privilege (see {@code RbacOperationSemantics}). The dedicated + * {@code SEMANTIC_MODEL_*} privilege matrix, the write-time source-access check, and the + * independent/propagated read-time enforcement modes land in the authorization phase. + */ +@PolarisImmutable +@SuppressWarnings("immutables:incompat") +public abstract class SemanticModelCatalogHandler extends CatalogHandler { + + private SemanticModelCatalog semanticModelCatalog; + + @Override + protected void initializeCatalog() { + this.semanticModelCatalog = + new SemanticModelCatalog( + metaStoreManager(), + callContext(), + this.resolutionManifest, + realmConfig().getConfig(FeatureConfiguration.SEMANTIC_MODEL_MAX_DOCUMENT_BYTES), + realmConfig().getConfig(FeatureConfiguration.SEMANTIC_MODEL_MAX_EXPRESSION_BYTES)); + } + + public LoadSemanticModelResponse createSemanticModel( + Namespace namespace, CreateSemanticModelRequest request) { + PolarisAuthorizableOperation op = PolarisAuthorizableOperation.CREATE_SEMANTIC_MODEL; + authorizeBasicNamespaceOperationOrThrow(op, namespace); + + SemanticModelIdentifier identifier = + SemanticModelIdentifier.builder() + .setNamespace(List.of(namespace.levels())) + .setName(request.getName()) + .build(); + return semanticModelCatalog.createSemanticModel(identifier, request.getDocument()); + } + + public ListSemanticModelsResponse listSemanticModels( + Namespace namespace, String pageToken, Integer pageSize) { + PolarisAuthorizableOperation op = PolarisAuthorizableOperation.LIST_SEMANTIC_MODEL; + authorizeBasicNamespaceOperationOrThrow(op, namespace); + + PageToken pageRequest = PageToken.build(pageToken, pageSize, this::shouldDecodeToken); + return semanticModelCatalog.listSemanticModels(namespace, pageRequest); + } + + public LoadSemanticModelResponse loadSemanticModel(SemanticModelIdentifier identifier) { + PolarisAuthorizableOperation op = PolarisAuthorizableOperation.LOAD_SEMANTIC_MODEL; + authorizeBasicSemanticModelOperationOrThrow(op, identifier); + return semanticModelCatalog.loadSemanticModel(identifier); + } + + public LoadSemanticModelResponse updateSemanticModel( + SemanticModelIdentifier identifier, UpdateSemanticModelRequest request) { + PolarisAuthorizableOperation op = PolarisAuthorizableOperation.UPDATE_SEMANTIC_MODEL; + authorizeBasicSemanticModelOperationOrThrow(op, identifier); + return semanticModelCatalog.updateSemanticModel( + identifier, request.getDocument(), request.getEntityVersion()); + } + + public void dropSemanticModel(SemanticModelIdentifier identifier) { + PolarisAuthorizableOperation op = PolarisAuthorizableOperation.DROP_SEMANTIC_MODEL; + authorizeBasicSemanticModelOperationOrThrow(op, identifier); + semanticModelCatalog.dropSemanticModel(identifier); + } + + private boolean shouldDecodeToken() { + return realmConfig().getConfig(FeatureConfiguration.LIST_PAGINATION_ENABLED); + } + + private void authorizeBasicSemanticModelOperationOrThrow( + PolarisAuthorizableOperation op, SemanticModelIdentifier identifier) { + Namespace namespace = Namespace.of(identifier.getNamespace().toArray(new String[0])); + resolutionManifest = newResolutionManifest(); + resolutionManifest.addPassthroughPath( + new ResolverPath( + PolarisCatalogHelpers.identifierToList(namespace, identifier.getName()), + PolarisEntityType.SEMANTIC_MODEL, + true /* optional */)); + resolutionManifest.resolveAll(); + + PolarisResolvedPathWrapper target = + resolutionManifest.getResolvedPath( + ResolvedPathKey.ofSemanticModel(namespace, identifier.getName()), true); + if (target == null) { + throw new NoSuchSemanticModelException( + String.format("Semantic model does not exist: %s", identifier.getName())); + } + + authorizer() + .authorizeOrThrow( + polarisPrincipal(), + resolutionManifest.getAllActivatedCatalogRoleAndPrincipalRoles(), + op, + target, + null /* secondary */); + + initializeCatalog(); + } +} diff --git a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandlerFactory.java b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandlerFactory.java new file mode 100644 index 00000000000..16a9b9e5517 --- /dev/null +++ b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandlerFactory.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.catalog.semanticmodel; + +import jakarta.enterprise.context.RequestScoped; +import jakarta.inject.Inject; +import org.apache.polaris.core.auth.PolarisAuthorizer; +import org.apache.polaris.core.auth.PolarisPrincipal; +import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.persistence.PolarisMetaStoreManager; +import org.apache.polaris.core.persistence.resolver.ResolutionManifestFactory; + +@RequestScoped +public class SemanticModelCatalogHandlerFactory { + + @Inject CallContext callContext; + @Inject ResolutionManifestFactory resolutionManifestFactory; + @Inject PolarisMetaStoreManager metaStoreManager; + @Inject PolarisAuthorizer authorizer; + + public SemanticModelCatalogHandler createHandler(String catalogName, PolarisPrincipal principal) { + return ImmutableSemanticModelCatalogHandler.builder() + .catalogName(catalogName) + .polarisPrincipal(principal) + .callContext(callContext) + .resolutionManifestFactory(resolutionManifestFactory) + .metaStoreManager(metaStoreManager) + .authorizer(authorizer) + .build(); + } +} diff --git a/extensions/semantic-models/src/main/resources/osi/v0.1.1/osi-schema.json b/extensions/semantic-models/src/main/resources/osi/v0.1.1/osi-schema.json new file mode 100644 index 00000000000..a4a6f92af51 --- /dev/null +++ b/extensions/semantic-models/src/main/resources/osi/v0.1.1/osi-schema.json @@ -0,0 +1,344 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/open-semantic-interchange/OSI/core-spec/osi-schema.json", + "title": "OSI Core Metadata Specification", + "description": "JSON Schema for validating OSI (Open Semantic Interoperability) semantic model definitions", + "type": "object", + "properties": { + "version": { + "type": "string", + "const": "0.1.1", + "description": "OSI specification version" + }, + "dialects": { + "type": "array", + "description": "Supported expression language dialects (enumeration definition)", + "items": { + "$ref": "#/$defs/Dialect" + } + }, + "vendors": { + "type": "array", + "description": "Supported vendors for custom extensions (enumeration definition)", + "items": { + "$ref": "#/$defs/Vendor" + } + }, + "semantic_model": { + "type": "array", + "description": "Collection of semantic model definitions", + "items": { + "$ref": "#/$defs/SemanticModel" + } + } + }, + "required": ["version", "semantic_model"], + "additionalProperties": false, + "$defs": { + "Dialect": { + "type": "string", + "enum": ["ANSI_SQL", "SNOWFLAKE", "MDX", "TABLEAU", "DATABRICKS", "MAQL"], + "description": "Supported SQL and expression language dialects" + }, + "Vendor": { + "type": "string", + "enum": ["COMMON", "SNOWFLAKE", "SALESFORCE", "DBT", "DATABRICKS", "GOODDATA"], + "description": "Supported vendors for custom extensions" + }, + "AIContext": { + "description": "Additional context for AI tools", + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": { + "instructions": { + "type": "string", + "description": "Instructions for AI on how to use this entity" + }, + "synonyms": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Alternative names and terms" + }, + "examples": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Sample questions or use cases" + } + }, + "additionalProperties": true + } + ] + }, + "CustomExtension": { + "type": "object", + "description": "Vendor-specific attributes for extensibility", + "properties": { + "vendor_name": { + "$ref": "#/$defs/Vendor" + }, + "data": { + "type": "string", + "description": "JSON string containing vendor-specific data" + } + }, + "required": ["vendor_name", "data"], + "additionalProperties": false + }, + "DialectExpression": { + "type": "object", + "description": "Expression in a specific dialect", + "properties": { + "dialect": { + "$ref": "#/$defs/Dialect" + }, + "expression": { + "type": "string", + "description": "SQL or dialect-specific expression" + } + }, + "required": ["dialect", "expression"], + "additionalProperties": false + }, + "Expression": { + "type": "object", + "description": "Expression definition with multi-dialect support", + "properties": { + "dialects": { + "type": "array", + "items": { + "$ref": "#/$defs/DialectExpression" + }, + "minItems": 1 + } + }, + "required": ["dialects"], + "additionalProperties": false + }, + "Dimension": { + "type": "object", + "description": "Dimension metadata", + "properties": { + "is_time": { + "type": "boolean", + "description": "Indicates if this is a time-based dimension for temporal filtering" + } + }, + "additionalProperties": false + }, + "Field": { + "type": "object", + "description": "Row-level attribute for grouping, filtering, and metric expressions", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier for the field within the dataset" + }, + "expression": { + "$ref": "#/$defs/Expression" + }, + "dimension": { + "$ref": "#/$defs/Dimension" + }, + "label": { + "type": "string", + "description": "Label for categorization" + }, + "description": { + "type": "string", + "description": "Human-readable description" + }, + "ai_context": { + "$ref": "#/$defs/AIContext" + }, + "custom_extensions": { + "type": "array", + "items": { + "$ref": "#/$defs/CustomExtension" + } + } + }, + "required": ["name", "expression"], + "additionalProperties": false + }, + "Dataset": { + "type": "object", + "description": "Logical dataset representing a business entity (fact or dimension table)", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier for the dataset" + }, + "source": { + "type": "string", + "description": "Reference to underlying physical table/view (database.schema.table) or query" + }, + "primary_key": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Primary key columns (single or composite)" + }, + "unique_keys": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "description": "Array of unique key definitions (each can be single or composite)" + }, + "description": { + "type": "string", + "description": "Human-readable description" + }, + "ai_context": { + "$ref": "#/$defs/AIContext" + }, + "fields": { + "type": "array", + "items": { + "$ref": "#/$defs/Field" + } + }, + "custom_extensions": { + "type": "array", + "items": { + "$ref": "#/$defs/CustomExtension" + } + } + }, + "required": ["name", "source"], + "additionalProperties": false + }, + "Relationship": { + "type": "object", + "description": "Foreign key relationship between datasets", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier for the relationship" + }, + "from": { + "type": "string", + "description": "Dataset on the many side of the relationship" + }, + "to": { + "type": "string", + "description": "Dataset on the one side of the relationship" + }, + "from_columns": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "description": "Foreign key columns in the 'from' dataset" + }, + "to_columns": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "description": "Primary/unique key columns in the 'to' dataset" + }, + "ai_context": { + "$ref": "#/$defs/AIContext" + }, + "custom_extensions": { + "type": "array", + "items": { + "$ref": "#/$defs/CustomExtension" + } + } + }, + "required": ["name", "from", "to", "from_columns", "to_columns"], + "additionalProperties": false + }, + "Metric": { + "type": "object", + "description": "Quantitative measure defined on business data", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier for the metric" + }, + "expression": { + "$ref": "#/$defs/Expression" + }, + "description": { + "type": "string", + "description": "Human-readable description of what the metric measures" + }, + "ai_context": { + "$ref": "#/$defs/AIContext" + }, + "custom_extensions": { + "type": "array", + "items": { + "$ref": "#/$defs/CustomExtension" + } + } + }, + "required": ["name", "expression"], + "additionalProperties": false + }, + "SemanticModel": { + "type": "object", + "description": "Top-level container representing a complete semantic model", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier for the semantic model" + }, + "description": { + "type": "string", + "description": "Human-readable description" + }, + "ai_context": { + "$ref": "#/$defs/AIContext" + }, + "datasets": { + "type": "array", + "items": { + "$ref": "#/$defs/Dataset" + }, + "minItems": 1, + "description": "Collection of logical datasets" + }, + "relationships": { + "type": "array", + "items": { + "$ref": "#/$defs/Relationship" + }, + "description": "Defines how datasets are connected" + }, + "metrics": { + "type": "array", + "items": { + "$ref": "#/$defs/Metric" + }, + "description": "Quantifiable measures spanning datasets" + }, + "custom_extensions": { + "type": "array", + "items": { + "$ref": "#/$defs/CustomExtension" + } + } + }, + "required": ["name", "datasets"], + "additionalProperties": false + } + } +} diff --git a/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/OsiDocumentValidatorTest.java b/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/OsiDocumentValidatorTest.java new file mode 100644 index 00000000000..2f8adc8c014 --- /dev/null +++ b/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/OsiDocumentValidatorTest.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.catalog.semanticmodel; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import org.apache.iceberg.exceptions.BadRequestException; +import org.junit.jupiter.api.Test; + +class OsiDocumentValidatorTest { + + private static final int MAX_DOC = 1024 * 1024; + private static final int MAX_EXPR = 16 * 1024; + + private static final String VALID_MODEL = + "[{\"name\":\"m\",\"datasets\":[{\"name\":\"d\",\"source\":\"sales.store_sales\"}]}]"; + + @Test + void acceptsValidDocument() { + var node = OsiDocumentValidator.validate("0.1.1", VALID_MODEL, MAX_DOC, MAX_EXPR); + assertThat(node.isArray()).isTrue(); + assertThat(node.get(0).get("name").asText()).isEqualTo("m"); + } + + @Test + void rejectsUnsupportedSpecVersion() { + assertThatThrownBy(() -> OsiDocumentValidator.validate("0.2.0", VALID_MODEL, MAX_DOC, MAX_EXPR)) + .isInstanceOf(BadRequestException.class) + .hasMessageContaining("Unsupported OSI spec version"); + } + + @Test + void rejectsUnknownField() { + String withUnknown = + "[{\"name\":\"m\",\"bogus\":1,\"datasets\":[{\"name\":\"d\",\"source\":\"a.b\"}]}]"; + assertThatThrownBy(() -> OsiDocumentValidator.validate("0.1.1", withUnknown, MAX_DOC, MAX_EXPR)) + .isInstanceOf(BadRequestException.class) + .hasMessageContaining("OSI schema validation"); + } + + @Test + void rejectsMissingRequiredField() { + // dataset missing the required 'source' + String missingSource = "[{\"name\":\"m\",\"datasets\":[{\"name\":\"d\"}]}]"; + assertThatThrownBy( + () -> OsiDocumentValidator.validate("0.1.1", missingSource, MAX_DOC, MAX_EXPR)) + .isInstanceOf(BadRequestException.class) + .hasMessageContaining("OSI schema validation"); + } + + @Test + void rejectsInvalidJson() { + assertThatThrownBy(() -> OsiDocumentValidator.validate("0.1.1", "{not json", MAX_DOC, MAX_EXPR)) + .isInstanceOf(BadRequestException.class) + .hasMessageContaining("not valid JSON"); + } + + @Test + void rejectsOversizeDocument() { + assertThatThrownBy(() -> OsiDocumentValidator.validate("0.1.1", VALID_MODEL, 10, MAX_EXPR)) + .isInstanceOf(BadRequestException.class) + .hasMessageContaining("exceeds the maximum"); + } + + @Test + void rejectsOversizeExpression() { + String bigExpr = "SUM(" + "x".repeat(100) + ")"; + String model = + "[{\"name\":\"m\",\"datasets\":[{\"name\":\"d\",\"source\":\"a.b\"}]," + + "\"metrics\":[{\"name\":\"total\",\"expression\":{\"dialects\":" + + "[{\"dialect\":\"ANSI_SQL\",\"expression\":\"" + + bigExpr + + "\"}]}}]}]"; + // Schema-valid, but the expression exceeds a tiny per-expression cap. + assertThatThrownBy(() -> OsiDocumentValidator.validate("0.1.1", model, MAX_DOC, 10)) + .isInstanceOf(BadRequestException.class) + .hasMessageContaining("expression"); + } +} diff --git a/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogTest.java b/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogTest.java new file mode 100644 index 00000000000..5ee4a994243 --- /dev/null +++ b/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogTest.java @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.catalog.semanticmodel; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.Arrays; +import java.util.List; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.AlreadyExistsException; +import org.apache.iceberg.exceptions.BadRequestException; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.entity.CatalogEntity; +import org.apache.polaris.core.entity.EntityNameLookupRecord; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisEntity; +import org.apache.polaris.core.entity.PolarisEntitySubType; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.persistence.PolarisMetaStoreManager; +import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; +import org.apache.polaris.core.persistence.ResolvedPolarisEntity; +import org.apache.polaris.core.persistence.dao.entity.DropEntityResult; +import org.apache.polaris.core.persistence.dao.entity.EntityResult; +import org.apache.polaris.core.persistence.dao.entity.GenerateEntityIdResult; +import org.apache.polaris.core.persistence.dao.entity.ListEntitiesResult; +import org.apache.polaris.core.persistence.pagination.Page; +import org.apache.polaris.core.persistence.pagination.PageToken; +import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifestCatalogView; +import org.apache.polaris.core.persistence.resolver.ResolvedPathKey; +import org.apache.polaris.core.semantic.SemanticModelEntity; +import org.apache.polaris.core.semantic.exceptions.NoSuchSemanticModelException; +import org.apache.polaris.core.semantic.exceptions.SemanticModelVersionMismatchException; +import org.apache.polaris.service.catalog.semanticmodel.types.LoadSemanticModelResponse; +import org.apache.polaris.service.catalog.semanticmodel.types.SemanticModelDocument; +import org.apache.polaris.service.catalog.semanticmodel.types.SemanticModelIdentifier; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.invocation.InvocationOnMock; + +/** + * Unit tests for {@link SemanticModelCatalog}. Following the {@code extensions/auth/opa} pattern, + * these are plain JUnit + Mockito tests that drive the catalog directly against a mocked resolution + * view and metastore manager — no Quarkus bootstrap. Full-stack coverage is left to the + * integration-test layer. + */ +class SemanticModelCatalogTest { + + private static final long CATALOG_ID = 1L; + private static final Namespace NS = Namespace.of("sales"); + private static final String MODEL = "m"; + private static final SemanticModelIdentifier IDENTIFIER = + SemanticModelIdentifier.builder().setNamespace(List.of("sales")).setName(MODEL).build(); + private static final String VALID_MODEL_JSON = + "[{\"name\":\"m\",\"datasets\":[{\"name\":\"d\",\"source\":\"sales.store_sales\"}]}]"; + private static final int MAX_DOC = 1024 * 1024; + private static final int MAX_EXPR = 16 * 1024; + + private PolarisResolutionManifestCatalogView view; + private PolarisMetaStoreManager metaStoreManager; + private SemanticModelCatalog catalog; + + private PolarisEntity catalogEntity; + private PolarisEntity namespaceEntity; + + @BeforeEach + void setUp() { + view = mock(PolarisResolutionManifestCatalogView.class); + metaStoreManager = mock(PolarisMetaStoreManager.class); + CallContext callContext = mock(CallContext.class); + PolarisCallContext polarisCallContext = mock(PolarisCallContext.class); + when(callContext.getPolarisCallContext()).thenReturn(polarisCallContext); + + catalogEntity = entity(PolarisEntityType.CATALOG, CATALOG_ID, "cat", 0L); + namespaceEntity = entity(PolarisEntityType.NAMESPACE, 2L, "sales", CATALOG_ID); + when(view.getResolvedCatalogEntity()).thenReturn(new CatalogEntity(catalogEntity)); + when(view.getResolvedPath(ResolvedPathKey.ofNamespace(NS))) + .thenReturn(path(catalogEntity, namespaceEntity)); + + catalog = new SemanticModelCatalog(metaStoreManager, callContext, view, MAX_DOC, MAX_EXPR); + } + + private SemanticModelDocument doc(String semanticModelJson) { + return SemanticModelDocument.builder() + .setVersion("0.1.1") + .setSemanticModel(semanticModelJson) + .build(); + } + + private void stubResolvableSource() { + PolarisEntity table = + new PolarisEntity.Builder() + .setType(PolarisEntityType.TABLE_LIKE) + .setSubType(PolarisEntitySubType.ICEBERG_TABLE) + .setId(5L) + .setCatalogId(CATALOG_ID) + .setParentId(2L) + .setName("store_sales") + .build(); + when(view.getPassthroughResolvedPath( + eq(ResolvedPathKey.ofTableLike(TableIdentifier.of(NS, "store_sales"))), + eq(PolarisEntitySubType.ANY_SUBTYPE))) + .thenReturn(path(catalogEntity, namespaceEntity, table)); + } + + private void stubExistingModel(int entityVersion) { + SemanticModelEntity stored = + new SemanticModelEntity.Builder(NS, MODEL) + .setSpecVersion("0.1.1") + .setContent(VALID_MODEL_JSON) + .setId(10L) + .setCatalogId(CATALOG_ID) + .setParentId(2L) + .setEntityVersion(entityVersion) + .build(); + when(view.getPassthroughResolvedPath( + eq(ResolvedPathKey.ofSemanticModel(NS, MODEL)), eq(PolarisEntitySubType.NULL_SUBTYPE))) + .thenReturn(path(catalogEntity, namespaceEntity, stored)); + } + + @Test + void createResolvesSourcesAndPersists() { + stubResolvableSource(); + when(metaStoreManager.generateNewEntityId(any())).thenReturn(new GenerateEntityIdResult(10L)); + // Echo the entity back as the persisted result. + when(metaStoreManager.createEntityIfNotExists(any(), any(), any())) + .thenAnswer(SemanticModelCatalogTest::echoPersistedEntity); + + LoadSemanticModelResponse response = + catalog.createSemanticModel(IDENTIFIER, doc(VALID_MODEL_JSON)); + + assertThat(response.getDocument().getSemanticModel()).isEqualTo(VALID_MODEL_JSON); + assertThat(response.getDocument().getVersion()).isEqualTo("0.1.1"); + assertThat(response.getEntityVersion()).isEqualTo("1"); + } + + @Test + void createRejectsUnresolvedSource() { + // No stub for the source table -> passthrough resolution returns null. + assertThatThrownBy(() -> catalog.createSemanticModel(IDENTIFIER, doc(VALID_MODEL_JSON))) + .isInstanceOf(BadRequestException.class) + .hasMessageContaining("/semantic_model/0/datasets/0/source") + .hasMessageContaining("store_sales"); + } + + @Test + void createRejectsExistingModel() { + stubExistingModel(1); + assertThatThrownBy(() -> catalog.createSemanticModel(IDENTIFIER, doc(VALID_MODEL_JSON))) + .isInstanceOf(AlreadyExistsException.class); + } + + @Test + void loadReturnsStoredDocument() { + stubExistingModel(4); + LoadSemanticModelResponse response = catalog.loadSemanticModel(IDENTIFIER); + assertThat(response.getDocument().getSemanticModel()).isEqualTo(VALID_MODEL_JSON); + assertThat(response.getEntityVersion()).isEqualTo("4"); + } + + @Test + void loadRejectsMissingModel() { + assertThatThrownBy(() -> catalog.loadSemanticModel(IDENTIFIER)) + .isInstanceOf(NoSuchSemanticModelException.class); + } + + @Test + void updateRejectsVersionMismatch() { + stubExistingModel(3); + assertThatThrownBy(() -> catalog.updateSemanticModel(IDENTIFIER, doc(VALID_MODEL_JSON), "1")) + .isInstanceOf(SemanticModelVersionMismatchException.class); + } + + @Test + void updatePersistsWhenVersionMatches() { + stubExistingModel(3); + stubResolvableSource(); + when(metaStoreManager.updateEntityPropertiesIfNotChanged(any(), any(), any())) + .thenAnswer(SemanticModelCatalogTest::echoPersistedEntity); + + LoadSemanticModelResponse response = + catalog.updateSemanticModel(IDENTIFIER, doc(VALID_MODEL_JSON), "3"); + assertThat(response.getDocument().getSemanticModel()).isEqualTo(VALID_MODEL_JSON); + } + + @Test + void dropRemovesModel() { + stubExistingModel(1); + when(metaStoreManager.dropEntityIfExists(any(), any(), any(), any(), eq(false))) + .thenReturn(new DropEntityResult()); + assertThatCode(() -> catalog.dropSemanticModel(IDENTIFIER)).doesNotThrowAnyException(); + } + + @Test + void dropRejectsMissingModel() { + assertThatThrownBy(() -> catalog.dropSemanticModel(IDENTIFIER)) + .isInstanceOf(NoSuchSemanticModelException.class); + } + + @Test + void listReturnsIdentifiers() { + EntityNameLookupRecord record = + new EntityNameLookupRecord( + entity(PolarisEntityType.SEMANTIC_MODEL, 10L, MODEL, CATALOG_ID)); + when(metaStoreManager.listEntities( + any(), + any(), + eq(PolarisEntityType.SEMANTIC_MODEL), + eq(PolarisEntitySubType.NULL_SUBTYPE), + any())) + .thenReturn(new ListEntitiesResult(Page.fromItems(List.of(record)))); + + var response = catalog.listSemanticModels(NS, PageToken.readEverything()); + assertThat(response.getIdentifiers()) + .singleElement() + .satisfies( + id -> { + assertThat(id.getName()).isEqualTo(MODEL); + assertThat(id.getNamespace()).containsExactly("sales"); + }); + } + + // ---- helpers ---- + + /** Mockito answer that echoes the entity passed to a persist call back as the stored result. */ + private static EntityResult echoPersistedEntity(InvocationOnMock invocation) { + return new EntityResult((PolarisBaseEntity) invocation.getArgument(2)); + } + + private static PolarisEntity entity( + PolarisEntityType type, long id, String name, long catalogId) { + return new PolarisEntity.Builder() + .setType(type) + .setId(id) + .setCatalogId(catalogId) + .setName(name) + .build(); + } + + private static PolarisResolvedPathWrapper path(PolarisEntity... entities) { + return new PolarisResolvedPathWrapper( + Arrays.stream(entities) + .map(e -> new ResolvedPolarisEntity(e, List.of(), List.of())) + .toList()); + } +} diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 2e9b8983249..96490e2fbdc 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -82,6 +82,7 @@ jakarta-servlet-api = { module = "jakarta.servlet:jakarta.servlet-api", version jakarta-validation-api = { module = "jakarta.validation:jakarta.validation-api", version = "3.1.1" } jakarta-ws-rs-api = { module = "jakarta.ws.rs:jakarta.ws.rs-api", version = "4.0.0" } javax-servlet-api = { module = "javax.servlet:javax.servlet-api", version = "4.0.1" } +json-schema-validator = { module = "com.networknt:json-schema-validator", version = "1.5.9" } jspecify = { module = "org.jspecify:jspecify", version = "1.0.0" } jcstress-core = { module = "org.openjdk.jcstress:jcstress-core", version = "0.16" } jmh-core = { module = "org.openjdk.jmh:jmh-core", version.ref = "jmh" } diff --git a/persistence/nosql/persistence/metastore-types/src/main/java/org/apache/polaris/persistence/nosql/coretypes/mapping/EntityObjMappings.java b/persistence/nosql/persistence/metastore-types/src/main/java/org/apache/polaris/persistence/nosql/coretypes/mapping/EntityObjMappings.java index 1349b50ebb9..c4f8104b940 100644 --- a/persistence/nosql/persistence/metastore-types/src/main/java/org/apache/polaris/persistence/nosql/coretypes/mapping/EntityObjMappings.java +++ b/persistence/nosql/persistence/metastore-types/src/main/java/org/apache/polaris/persistence/nosql/coretypes/mapping/EntityObjMappings.java @@ -90,6 +90,9 @@ public final class EntityObjMappings { static { var missingEntityTypes = new HashSet<>(Set.of(PolarisEntityType.values())); missingEntityTypes.remove(PolarisEntityType.NULL_TYPE); + // SEMANTIC_MODEL is not yet supported by the NoSQL backend; its mapping lands in a follow-up. + // Exclude it here so the NoSQL module still loads while the type exists in the shared enum. + missingEntityTypes.remove(PolarisEntityType.SEMANTIC_MODEL); missingEntityTypes.removeAll(BY_ENTITY_TYPE.keySet()); checkState( missingEntityTypes.isEmpty(), diff --git a/persistence/nosql/persistence/metastore-types/src/test/java/org/apache/polaris/persistence/nosql/coretypes/mapping/TestEntityObjMappings.java b/persistence/nosql/persistence/metastore-types/src/test/java/org/apache/polaris/persistence/nosql/coretypes/mapping/TestEntityObjMappings.java index 7687c7ec2fc..50a84e4da55 100644 --- a/persistence/nosql/persistence/metastore-types/src/test/java/org/apache/polaris/persistence/nosql/coretypes/mapping/TestEntityObjMappings.java +++ b/persistence/nosql/persistence/metastore-types/src/test/java/org/apache/polaris/persistence/nosql/coretypes/mapping/TestEntityObjMappings.java @@ -119,6 +119,8 @@ public void entityObjMapping(PolarisEntityType entityType, PolarisEntitySubType static Stream entityObjMapping() { return Arrays.stream(PolarisEntityType.values()) .filter(t -> t != PolarisEntityType.NULL_TYPE) + // SEMANTIC_MODEL is intentionally not mapped by the NoSQL backend yet (follow-up). + .filter(t -> t != PolarisEntityType.SEMANTIC_MODEL) .flatMap( t -> { var subTypes = diff --git a/polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizableOperation.java b/polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizableOperation.java index c016749459b..2fc3b6157da 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizableOperation.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizableOperation.java @@ -115,6 +115,11 @@ public enum PolarisAuthorizableOperation { GET_APPLICABLE_POLICIES_ON_TABLE, ADD_POLICY_GRANT_TO_CATALOG_ROLE, REVOKE_POLICY_GRANT_FROM_CATALOG_ROLE, + CREATE_SEMANTIC_MODEL, + LOAD_SEMANTIC_MODEL, + UPDATE_SEMANTIC_MODEL, + DROP_SEMANTIC_MODEL, + LIST_SEMANTIC_MODEL, ASSIGN_TABLE_UUID, UPGRADE_TABLE_FORMAT_VERSION, ADD_TABLE_SCHEMA, diff --git a/polaris-core/src/main/java/org/apache/polaris/core/auth/RbacOperationSemantics.java b/polaris-core/src/main/java/org/apache/polaris/core/auth/RbacOperationSemantics.java index f63999959dd..20fa79f6525 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/auth/RbacOperationSemantics.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/auth/RbacOperationSemantics.java @@ -41,6 +41,7 @@ import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.CREATE_POLICY; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.CREATE_PRINCIPAL; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.CREATE_PRINCIPAL_ROLE; +import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.CREATE_SEMANTIC_MODEL; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.CREATE_TABLE_DIRECT; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.CREATE_TABLE_DIRECT_WITH_WRITE_DELEGATION; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.CREATE_TABLE_STAGED; @@ -55,6 +56,7 @@ import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.DETACH_POLICY_FROM_TABLE; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.DROP_NAMESPACE; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.DROP_POLICY; +import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.DROP_SEMANTIC_MODEL; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.DROP_TABLE_WITHOUT_PURGE; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.DROP_TABLE_WITH_PURGE; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.DROP_VIEW; @@ -76,10 +78,12 @@ import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.LIST_PRINCIPALS; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.LIST_PRINCIPAL_ROLES; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.LIST_PRINCIPAL_ROLES_ASSIGNED; +import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.LIST_SEMANTIC_MODEL; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.LIST_TABLES; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.LIST_VIEWS; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.LOAD_NAMESPACE_METADATA; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.LOAD_POLICY; +import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.LOAD_SEMANTIC_MODEL; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.LOAD_TABLE; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.LOAD_TABLE_WITH_READ_DELEGATION; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.LOAD_TABLE_WITH_WRITE_DELEGATION; @@ -126,6 +130,7 @@ import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.UPDATE_POLICY; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.UPDATE_PRINCIPAL; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.UPDATE_PRINCIPAL_ROLE; +import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.UPDATE_SEMANTIC_MODEL; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.UPDATE_TABLE; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.UPDATE_TABLE_FOR_STAGED_CREATE; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.UPGRADE_TABLE_FORMAT_VERSION; @@ -135,6 +140,7 @@ import static org.apache.polaris.core.entity.PolarisPrivilege.CATALOG_DETACH_POLICY; import static org.apache.polaris.core.entity.PolarisPrivilege.CATALOG_DROP; import static org.apache.polaris.core.entity.PolarisPrivilege.CATALOG_LIST; +import static org.apache.polaris.core.entity.PolarisPrivilege.CATALOG_MANAGE_CONTENT; import static org.apache.polaris.core.entity.PolarisPrivilege.CATALOG_MANAGE_GRANTS_ON_SECURABLE; import static org.apache.polaris.core.entity.PolarisPrivilege.CATALOG_READ_PROPERTIES; import static org.apache.polaris.core.entity.PolarisPrivilege.CATALOG_ROLE_CREATE; @@ -407,6 +413,16 @@ private static void register( register(UPDATE_POLICY, POLICY_WRITE); register(LIST_POLICY, POLICY_LIST); + // Semantic-model operations. + // Interim (Phase 3): gated behind the coarse CATALOG_MANAGE_CONTENT privilege, matching the + // design's "coarser-grained access reuses existing higher-level privileges" posture. The + // dedicated SEMANTIC_MODEL_* privilege matrix and read-time enforcement modes land in Phase 2. + register(CREATE_SEMANTIC_MODEL, CATALOG_MANAGE_CONTENT); + register(LOAD_SEMANTIC_MODEL, CATALOG_MANAGE_CONTENT); + register(UPDATE_SEMANTIC_MODEL, CATALOG_MANAGE_CONTENT); + register(DROP_SEMANTIC_MODEL, CATALOG_MANAGE_CONTENT); + register(LIST_SEMANTIC_MODEL, CATALOG_MANAGE_CONTENT); + // Policy attachment operations (use CATALOG rooting) register( ATTACH_POLICY_TO_CATALOG, diff --git a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java index 87aecac3026..cdb1812df42 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java @@ -517,6 +517,24 @@ public static void enforceFeatureEnabledOrThrow( .defaultValue(false) // beta feature, keep it off by default .buildFeatureConfiguration(); + public static final FeatureConfiguration SEMANTIC_MODEL_MAX_DOCUMENT_BYTES = + PolarisConfiguration.builder() + .key("SEMANTIC_MODEL_MAX_DOCUMENT_BYTES") + .description( + "The maximum size, in bytes, of a stored OSI semantic-model document. Writes whose " + + "document exceeds this cap are rejected with 400.") + .defaultValue(1024 * 1024) // 1 MB + .buildFeatureConfiguration(); + + public static final FeatureConfiguration SEMANTIC_MODEL_MAX_EXPRESSION_BYTES = + PolarisConfiguration.builder() + .key("SEMANTIC_MODEL_MAX_EXPRESSION_BYTES") + .description( + "The maximum size, in bytes, of a single expression fragment inside an OSI " + + "semantic-model document. Guards against metadata-as-blob abuse.") + .defaultValue(16 * 1024) // 16 KB + .buildFeatureConfiguration(); + public static final FeatureConfiguration> SUPPORTED_CATALOG_CONNECTION_TYPES = PolarisConfiguration.>builder() .key("SUPPORTED_CATALOG_CONNECTION_TYPES") diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntityType.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntityType.java index 13aa1f465db..0896f314163 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntityType.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntityType.java @@ -34,7 +34,8 @@ public enum PolarisEntityType { TABLE_LIKE(7, NAMESPACE, false, false), TASK(8, ROOT, false, false), FILE(9, TABLE_LIKE, false, false), - POLICY(10, NAMESPACE, false, false); + POLICY(10, NAMESPACE, false, false), + SEMANTIC_MODEL(11, NAMESPACE, false, false); // to efficiently map a code to its corresponding entity type, use a reverse array which // is initialized below diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/ResolvedPathKey.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/ResolvedPathKey.java index d1acfc2ca45..934cd5bdbc9 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/ResolvedPathKey.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/ResolvedPathKey.java @@ -54,6 +54,11 @@ public static ResolvedPathKey ofPolicy(Namespace namespace, String name) { PolarisCatalogHelpers.identifierToList(namespace, name), PolarisEntityType.POLICY); } + public static ResolvedPathKey ofSemanticModel(Namespace namespace, String name) { + return new ResolvedPathKey( + PolarisCatalogHelpers.identifierToList(namespace, name), PolarisEntityType.SEMANTIC_MODEL); + } + public static ResolvedPathKey ofCatalogRole(String roleName) { return new ResolvedPathKey(List.of(roleName), PolarisEntityType.CATALOG_ROLE); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/semantic/SemanticModelEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/semantic/SemanticModelEntity.java new file mode 100644 index 00000000000..6ad2201641a --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/semantic/SemanticModelEntity.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.semantic; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.common.base.Preconditions; +import org.apache.iceberg.catalog.Namespace; +import org.apache.polaris.core.entity.NamespaceEntity; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisEntity; +import org.apache.polaris.core.entity.PolarisEntitySubType; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.entity.PolarisEntityUtils; +import org.jspecify.annotations.Nullable; + +/** + * A Polaris entity that stores an Open Semantic Interchange (OSI) semantic-model document. + * + *

Following the precedent set by {@link org.apache.polaris.core.policy.PolicyEntity}, the full + * document body is stored inside the entity {@code properties} map: the declared OSI spec version + * under {@link #SPEC_VERSION_KEY} and the OSI document (as a JSON string) under {@link + * #CONTENT_KEY}. + */ +public class SemanticModelEntity extends PolarisEntity { + + /** The declared OSI spec version of the stored document (e.g. {@code 0.1.1}). */ + public static final String SPEC_VERSION_KEY = "semantic-model.spec-version"; + + /** The OSI document serialized as a JSON string. */ + public static final String CONTENT_KEY = "semantic-model.content"; + + public SemanticModelEntity(PolarisBaseEntity sourceEntity) { + super(sourceEntity); + Preconditions.checkState( + getType() == PolarisEntityType.SEMANTIC_MODEL, "Invalid entity type: %s", getType()); + Preconditions.checkState( + getSubType() == PolarisEntitySubType.NULL_SUBTYPE, + "Invalid entity sub type: %s", + getSubType()); + } + + public static @Nullable SemanticModelEntity of(@Nullable PolarisBaseEntity sourceEntity) { + if (sourceEntity != null) { + return new SemanticModelEntity(sourceEntity); + } + return null; + } + + /** The declared OSI spec version of the stored document. */ + @JsonIgnore + public String getSpecVersion() { + return getPropertiesAsMap().get(SPEC_VERSION_KEY); + } + + /** The OSI document body, serialized as a JSON string. */ + @JsonIgnore + public String getContent() { + return getPropertiesAsMap().get(CONTENT_KEY); + } + + @JsonIgnore + public @Nullable Namespace getParentNamespace() { + String parentNamespace = getInternalPropertiesAsMap().get(NamespaceEntity.PARENT_NAMESPACE_KEY); + if (parentNamespace != null) { + return PolarisEntityUtils.decodeNamespace(parentNamespace); + } + return null; + } + + public static class Builder extends PolarisEntity.BaseBuilder { + public Builder(Namespace namespace, String modelName) { + super(); + setType(PolarisEntityType.SEMANTIC_MODEL); + setParentNamespace(namespace); + setName(modelName); + } + + public Builder(SemanticModelEntity original) { + super(original); + } + + @Override + public SemanticModelEntity build() { + Preconditions.checkArgument( + properties.containsKey(SPEC_VERSION_KEY), + "Semantic model spec version must be specified"); + Preconditions.checkArgument( + properties.containsKey(CONTENT_KEY), "Semantic model content must be specified"); + return new SemanticModelEntity(buildBase()); + } + + public Builder setParentNamespace(Namespace namespace) { + if (namespace != null && !namespace.isEmpty()) { + internalProperties.put( + NamespaceEntity.PARENT_NAMESPACE_KEY, PolarisEntityUtils.encodeNamespace(namespace)); + } + return this; + } + + public Builder setSpecVersion(String specVersion) { + properties.put(SPEC_VERSION_KEY, specVersion); + return this; + } + + public Builder setContent(String content) { + properties.put(CONTENT_KEY, content); + return this; + } + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/semantic/exceptions/NoSuchSemanticModelException.java b/polaris-core/src/main/java/org/apache/polaris/core/semantic/exceptions/NoSuchSemanticModelException.java new file mode 100644 index 00000000000..ecae524d764 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/semantic/exceptions/NoSuchSemanticModelException.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.semantic.exceptions; + +import org.apache.polaris.core.exceptions.PolarisException; + +/** Thrown when a semantic model cannot be found. Maps to HTTP 404. */ +public class NoSuchSemanticModelException extends PolarisException { + public NoSuchSemanticModelException(String message) { + super(message); + } + + public NoSuchSemanticModelException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/semantic/exceptions/SemanticModelVersionMismatchException.java b/polaris-core/src/main/java/org/apache/polaris/core/semantic/exceptions/SemanticModelVersionMismatchException.java new file mode 100644 index 00000000000..b7804392e22 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/semantic/exceptions/SemanticModelVersionMismatchException.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.semantic.exceptions; + +import org.apache.polaris.core.exceptions.PolarisException; + +/** + * Thrown when a semantic-model update supplies an {@code entity-version} that does not match the + * version currently stored in the catalog (optimistic-concurrency conflict). Maps to HTTP 409. + */ +public class SemanticModelVersionMismatchException extends PolarisException { + public SemanticModelVersionMismatchException(String message) { + super(message); + } + + public SemanticModelVersionMismatchException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/runtime/service/src/main/java/org/apache/polaris/service/exception/PolarisExceptionMapper.java b/runtime/service/src/main/java/org/apache/polaris/service/exception/PolarisExceptionMapper.java index 1e9926bfd7c..443982ba035 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/exception/PolarisExceptionMapper.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/exception/PolarisExceptionMapper.java @@ -33,6 +33,8 @@ import org.apache.polaris.core.policy.exceptions.PolicyInUseException; import org.apache.polaris.core.policy.exceptions.PolicyVersionMismatchException; import org.apache.polaris.core.policy.validator.InvalidPolicyException; +import org.apache.polaris.core.semantic.exceptions.NoSuchSemanticModelException; +import org.apache.polaris.core.semantic.exceptions.SemanticModelVersionMismatchException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.event.Level; @@ -58,6 +60,9 @@ private Response.Status getStatus(PolarisException exception) { case PolicyMappingAlreadyExistsException policyMappingAlreadyExistsException -> Response.Status.CONFLICT; case PolicyInUseException policyInUseException -> Response.Status.BAD_REQUEST; + case NoSuchSemanticModelException noSuchSemanticModelException -> Response.Status.NOT_FOUND; + case SemanticModelVersionMismatchException semanticModelVersionMismatchException -> + Response.Status.CONFLICT; default -> Response.Status.INTERNAL_SERVER_ERROR; }; } From 701274004c297b93c6cee6db53a6b5c9fe9bc965 Mon Sep 17 00:00:00 2001 From: Yufei Gu Date: Thu, 2 Jul 2026 23:22:15 -0700 Subject: [PATCH 02/13] Fix CI: update entity-type test and LICENSE for semantic models - PolarisEntityTypeTest.fromCode: code 11 now maps to SEMANTIC_MODEL (was asserted as null); shift the unknown-code case to 12. - runtime/server/distribution/LICENSE: add Apache-2.0 mentions for the new json-schema-validator dependency and its transitive com.ethlo.time:itu, required by :polaris-server:generateLicenseReport. --- .../core/entity/PolarisEntityTypeTest.java | 3 ++- runtime/server/distribution/LICENSE | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/polaris-core/src/test/java/org/apache/polaris/core/entity/PolarisEntityTypeTest.java b/polaris-core/src/test/java/org/apache/polaris/core/entity/PolarisEntityTypeTest.java index c2b9cd6ea9c..11c090b61b0 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/entity/PolarisEntityTypeTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/entity/PolarisEntityTypeTest.java @@ -40,7 +40,8 @@ static Stream entityTypes() { Arguments.of(8, PolarisEntityType.TASK), Arguments.of(9, PolarisEntityType.FILE), Arguments.of(10, PolarisEntityType.POLICY), - Arguments.of(11, null)); + Arguments.of(11, PolarisEntityType.SEMANTIC_MODEL), + Arguments.of(12, null)); } @ParameterizedTest diff --git a/runtime/server/distribution/LICENSE b/runtime/server/distribution/LICENSE index 014efbe619b..33d9d4891f4 100644 --- a/runtime/server/distribution/LICENSE +++ b/runtime/server/distribution/LICENSE @@ -516,6 +516,15 @@ License: MIT License -------------------------------------------------------------------------------- +This product bundles ethlo Internet Time Utility. + +* Maven group:artifact IDs: com.ethlo.time:itu + +Project URL: https://github.com/ethlo/itu +License: Apache License 2.0 - https://www.apache.org/licenses/LICENSE-2.0.txt + +-------------------------------------------------------------------------------- + This product bundles FasterXML Java Classmate. * Maven group:artifact IDs: com.fasterxml:classmate @@ -917,6 +926,15 @@ License: MIT License -------------------------------------------------------------------------------- +This product bundles networknt JSON Schema Validator. + +* Maven group:artifact IDs: com.networknt:json-schema-validator + +Project URL: https://github.com/networknt/json-schema-validator +License: Apache License 2.0 - https://www.apache.org/licenses/LICENSE-2.0.txt + +-------------------------------------------------------------------------------- + This product bundles Nimbus Jose JWT. * Maven group:artifact IDs: com.nimbusds:nimbus-jose-jwt From 6afcd44844f7b65a0865cd8568ba057b81f777e2 Mon Sep 17 00:00:00 2001 From: Yufei Gu Date: Thu, 2 Jul 2026 23:46:24 -0700 Subject: [PATCH 03/13] Defer semantic-model document validation behind an interface Ship only the SemanticDocumentValidator contract in this PR; the concrete OSI schema-validation implementation lands in a follow-up. - Add SemanticDocumentValidator interface (validate(SemanticModelDocument)). - Remove OsiDocumentValidator and its test, the bundled osi-schema.json resource and its RAT exclude, the networknt json-schema-validator dependency, the two LICENSE mentions (json-schema-validator, com.ethlo.time:itu), and the SEMANTIC_MODEL_MAX_*_BYTES feature configs. - SemanticModelCatalog now parses the document body and resolves dataset.source references only (schema/size validation deferred); drop the size-cap constructor params. Malformed JSON still fails with 400. --- build.gradle.kts | 4 - extensions/semantic-models/build.gradle.kts | 3 - .../semanticmodel/OsiDocumentValidator.java | 154 -------- .../SemanticDocumentValidator.java | 39 ++ .../semanticmodel/SemanticModelCatalog.java | 42 ++- .../SemanticModelCatalogHandler.java | 7 +- .../main/resources/osi/v0.1.1/osi-schema.json | 344 ------------------ .../OsiDocumentValidatorTest.java | 96 ----- .../SemanticModelCatalogTest.java | 4 +- gradle/libs.versions.toml | 1 - .../core/config/FeatureConfiguration.java | 18 - runtime/server/distribution/LICENSE | 18 - 12 files changed, 64 insertions(+), 666 deletions(-) delete mode 100644 extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/OsiDocumentValidator.java create mode 100644 extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticDocumentValidator.java delete mode 100644 extensions/semantic-models/src/main/resources/osi/v0.1.1/osi-schema.json delete mode 100644 extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/OsiDocumentValidatorTest.java diff --git a/build.gradle.kts b/build.gradle.kts index ac01108f262..9463e128082 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -87,10 +87,6 @@ tasks.named("rat").configure { // Files copied from OpenAPI Generator (ASLv2 licensed) don't have header excludes.add("server-templates/*.mustache") - // Vendored OSI JSON Schema (Apache-2.0, from open-semantic-interchange/OSI); JSON allows no - // comments so the ASF header cannot be inlined. - excludes.add("extensions/semantic-models/src/main/resources/osi/**") - // Manifest files do not allow comments excludes.add("tools/version/src/jarTest/resources/META-INF/FAKE_MANIFEST.MF") diff --git a/extensions/semantic-models/build.gradle.kts b/extensions/semantic-models/build.gradle.kts index dad70b7b3ee..ff03fd9da01 100644 --- a/extensions/semantic-models/build.gradle.kts +++ b/extensions/semantic-models/build.gradle.kts @@ -56,9 +56,6 @@ dependencies { implementation(libs.jakarta.servlet.api) implementation(libs.jakarta.ws.rs.api) - // Validates OSI documents against the bundled OSI JSON Schema (draft 2020-12). - implementation(libs.json.schema.validator) - implementation(platform(libs.micrometer.bom)) implementation("io.micrometer:micrometer-core") diff --git a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/OsiDocumentValidator.java b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/OsiDocumentValidator.java deleted file mode 100644 index 12602a59cfc..00000000000 --- a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/OsiDocumentValidator.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.catalog.semanticmodel; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.base.Utf8; -import com.networknt.schema.JsonSchema; -import com.networknt.schema.JsonSchemaFactory; -import com.networknt.schema.SpecVersion; -import com.networknt.schema.ValidationMessage; -import java.io.InputStream; -import java.io.UncheckedIOException; -import java.util.Set; -import java.util.stream.Collectors; -import org.apache.iceberg.exceptions.BadRequestException; - -/** - * Validates an OSI (Open Semantic Interchange) semantic-model document against the bundled OSI JSON - * Schema and against Polaris's configured size caps. - * - *

The Polaris API represents a document as a {@code version} string plus a {@code - * semantic_model} JSON string (see the {@code SemanticModelDocument} API type). This validator - * reconstructs the canonical OSI document {@code {"version": ..., "semantic_model": ...}} and - * validates it against the frozen v0.1.1 schema (draft 2020-12), which is strict: unknown fields - * fail because the schema declares {@code additionalProperties: false}. - * - *

Version evolution is an explicit user action: documents declaring an unsupported OSI spec - * version are rejected rather than silently accepted or migrated. - */ -public final class OsiDocumentValidator { - - /** The single OSI spec version this build bundles and supports. */ - public static final String SUPPORTED_SPEC_VERSION = "0.1.1"; - - private static final String SCHEMA_RESOURCE = "/osi/v0.1.1/osi-schema.json"; - private static final ObjectMapper MAPPER = new ObjectMapper(); - private static final JsonSchema SCHEMA = loadSchema(); - - private OsiDocumentValidator() {} - - private static JsonSchema loadSchema() { - try (InputStream in = OsiDocumentValidator.class.getResourceAsStream(SCHEMA_RESOURCE)) { - if (in == null) { - throw new IllegalStateException("Bundled OSI schema not found: " + SCHEMA_RESOURCE); - } - JsonSchemaFactory factory = JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V202012); - return factory.getSchema(in); - } catch (java.io.IOException e) { - throw new UncheckedIOException("Failed to load bundled OSI schema", e); - } - } - - /** - * Validates the given OSI document. Throws {@link BadRequestException} (HTTP 400) with - * JSON-Pointer field paths on any schema or size-cap failure. - * - * @param specVersion the declared OSI spec version - * @param semanticModelJson the OSI {@code semantic_model} content, serialized as a JSON string - * @param maxDocumentBytes maximum allowed size, in UTF-8 bytes, of the semantic-model content - * @param maxExpressionBytes maximum allowed size, in UTF-8 bytes, of any single expression - * @return the parsed OSI {@code semantic_model} node, for downstream source resolution - */ - public static JsonNode validate( - String specVersion, String semanticModelJson, int maxDocumentBytes, int maxExpressionBytes) { - if (!SUPPORTED_SPEC_VERSION.equals(specVersion)) { - throw new BadRequestException( - "Unsupported OSI spec version '%s'; this server supports '%s'", - specVersion, SUPPORTED_SPEC_VERSION); - } - - if (semanticModelJson == null || semanticModelJson.isBlank()) { - throw new BadRequestException("Semantic model document must not be empty"); - } - - int documentBytes = Utf8.encodedLength(semanticModelJson); - if (documentBytes > maxDocumentBytes) { - throw new BadRequestException( - "Semantic model document of %d bytes exceeds the maximum of %d bytes", - documentBytes, maxDocumentBytes); - } - - JsonNode semanticModelNode; - try { - semanticModelNode = MAPPER.readTree(semanticModelJson); - } catch (com.fasterxml.jackson.core.JsonProcessingException e) { - throw new BadRequestException( - "Field 'semantic_model' is not valid JSON: %s", e.getOriginalMessage()); - } - - // Reconstruct the canonical OSI document {version, semantic_model} for schema validation. - ObjectNode osiDocument = MAPPER.createObjectNode(); - osiDocument.put("version", specVersion); - osiDocument.set("semantic_model", semanticModelNode); - - Set errors = SCHEMA.validate(osiDocument); - if (!errors.isEmpty()) { - String details = - errors.stream() - .map(m -> m.getInstanceLocation() + ": " + m.getMessage()) - .sorted() - .collect(Collectors.joining("; ")); - throw new BadRequestException( - "Semantic model document failed OSI schema validation: %s", details); - } - - enforceExpressionCap(semanticModelNode, maxExpressionBytes); - - return semanticModelNode; - } - - /** - * Walks the document for SQL/expression fragments (the {@code expression} string inside each - * {@code dialects[]} entry) and rejects any that exceed the per-expression cap. - */ - private static void enforceExpressionCap(JsonNode node, int maxExpressionBytes) { - if (node.isObject()) { - JsonNode dialects = node.get("dialects"); - if (dialects != null && dialects.isArray()) { - for (JsonNode dialect : dialects) { - JsonNode expr = dialect.get("expression"); - if (expr != null && expr.isTextual()) { - int bytes = Utf8.encodedLength(expr.asText()); - if (bytes > maxExpressionBytes) { - throw new BadRequestException( - "An expression of %d bytes exceeds the maximum of %d bytes", - bytes, maxExpressionBytes); - } - } - } - } - node.forEach(child -> enforceExpressionCap(child, maxExpressionBytes)); - } else if (node.isArray()) { - node.forEach(child -> enforceExpressionCap(child, maxExpressionBytes)); - } - } -} diff --git a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticDocumentValidator.java b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticDocumentValidator.java new file mode 100644 index 00000000000..9308212f6d0 --- /dev/null +++ b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticDocumentValidator.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.catalog.semanticmodel; + +import org.apache.polaris.service.catalog.semanticmodel.types.SemanticModelDocument; + +/** + * Contract for validating an OSI (Open Semantic Interchange) semantic-model document at write time. + * + *

Only the interface ships in this phase; a concrete implementation (schema validation against + * the bundled OSI JSON Schema, size caps, etc.) lands in a follow-up. Implementations are expected + * to signal an invalid document by throwing an exception that maps to HTTP 400 (e.g. {@link + * org.apache.iceberg.exceptions.BadRequestException}) with field-level detail. + */ +public interface SemanticDocumentValidator { + + /** + * Validates the given OSI document, throwing on failure. + * + * @param document the document to validate (its {@code version} and {@code semantic_model} body) + */ + void validate(SemanticModelDocument document); +} diff --git a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalog.java b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalog.java index 274277e1805..b68b7ed67d7 100644 --- a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalog.java +++ b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalog.java @@ -19,6 +19,7 @@ package org.apache.polaris.service.catalog.semanticmodel; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Splitter; import java.util.LinkedHashSet; import java.util.List; @@ -55,34 +56,30 @@ /** * Core create/list/load/update/drop logic for OSI semantic models. Mirrors {@link * org.apache.polaris.service.catalog.policy.PolicyCatalog}: the OSI document body is stored inside - * the entity {@code properties} map, writes validate the document and resolve every {@code - * dataset.source} to a {@code TABLE_LIKE} entity in the current catalog, and updates use optimistic - * concurrency on the entity version. + * the entity {@code properties} map, writes resolve every {@code dataset.source} to a {@code + * TABLE_LIKE} entity in the current catalog, and updates use optimistic concurrency on the entity + * version. Document schema validation is a separate concern (see {@link + * SemanticDocumentValidator}). */ public class SemanticModelCatalog { private static final Logger LOGGER = LoggerFactory.getLogger(SemanticModelCatalog.class); + private static final ObjectMapper MAPPER = new ObjectMapper(); private final CallContext callContext; private final PolarisResolutionManifestCatalogView resolvedEntityView; private final CatalogEntity catalogEntity; private final long catalogId; private final PolarisMetaStoreManager metaStoreManager; - private final int maxDocumentBytes; - private final int maxExpressionBytes; public SemanticModelCatalog( PolarisMetaStoreManager metaStoreManager, CallContext callContext, - PolarisResolutionManifestCatalogView resolvedEntityView, - int maxDocumentBytes, - int maxExpressionBytes) { + PolarisResolutionManifestCatalogView resolvedEntityView) { this.callContext = callContext; this.resolvedEntityView = resolvedEntityView; this.catalogEntity = resolvedEntityView.getResolvedCatalogEntity(); this.catalogId = catalogEntity.getId(); this.metaStoreManager = metaStoreManager; - this.maxDocumentBytes = maxDocumentBytes; - this.maxExpressionBytes = maxExpressionBytes; } public LoadSemanticModelResponse createSemanticModel( @@ -105,7 +102,7 @@ public LoadSemanticModelResponse createSemanticModel( throw new AlreadyExistsException("Semantic model already exists: %s", identifier.getName()); } - // Validate against the bundled OSI schema and size caps, then resolve source tables. + // Parse the document and resolve its source tables before persisting. validateDocumentAndSources(document); SemanticModelEntity entity = @@ -263,17 +260,24 @@ private SemanticModelEntity resolveModelOrThrow(SemanticModelIdentifier identifi } /** - * Validates the document against the bundled OSI schema and size caps, then resolves every {@code - * dataset.source} to a {@code TABLE_LIKE} entity. Shared by create and update. + * Parses the OSI document body and resolves every {@code dataset.source} to a {@code TABLE_LIKE} + * entity. Shared by create and update. Schema/size validation is a separate concern handled by a + * {@link SemanticDocumentValidator} implementation (not yet wired in this phase). */ private void validateDocumentAndSources(SemanticModelDocument document) { - JsonNode semanticModel = - OsiDocumentValidator.validate( - document.getVersion(), - document.getSemanticModel(), - maxDocumentBytes, - maxExpressionBytes); + String body = document.getSemanticModel(); + if (body == null || body.isBlank()) { + return; + } + JsonNode semanticModel; + try { + semanticModel = MAPPER.readTree(body); + } catch (com.fasterxml.jackson.core.JsonProcessingException e) { + throw new BadRequestException( + "Field 'semantic_model' is not valid JSON: %s", e.getOriginalMessage()); + } resolveSourcesOrThrow(semanticModel); + // TODO call the SemanticDocumentValidator when it's ready } /** diff --git a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandler.java b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandler.java index 8bf75a1e1af..7859210bf9e 100644 --- a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandler.java +++ b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandler.java @@ -55,12 +55,7 @@ public abstract class SemanticModelCatalogHandler extends CatalogHandler { @Override protected void initializeCatalog() { this.semanticModelCatalog = - new SemanticModelCatalog( - metaStoreManager(), - callContext(), - this.resolutionManifest, - realmConfig().getConfig(FeatureConfiguration.SEMANTIC_MODEL_MAX_DOCUMENT_BYTES), - realmConfig().getConfig(FeatureConfiguration.SEMANTIC_MODEL_MAX_EXPRESSION_BYTES)); + new SemanticModelCatalog(metaStoreManager(), callContext(), this.resolutionManifest); } public LoadSemanticModelResponse createSemanticModel( diff --git a/extensions/semantic-models/src/main/resources/osi/v0.1.1/osi-schema.json b/extensions/semantic-models/src/main/resources/osi/v0.1.1/osi-schema.json deleted file mode 100644 index a4a6f92af51..00000000000 --- a/extensions/semantic-models/src/main/resources/osi/v0.1.1/osi-schema.json +++ /dev/null @@ -1,344 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/open-semantic-interchange/OSI/core-spec/osi-schema.json", - "title": "OSI Core Metadata Specification", - "description": "JSON Schema for validating OSI (Open Semantic Interoperability) semantic model definitions", - "type": "object", - "properties": { - "version": { - "type": "string", - "const": "0.1.1", - "description": "OSI specification version" - }, - "dialects": { - "type": "array", - "description": "Supported expression language dialects (enumeration definition)", - "items": { - "$ref": "#/$defs/Dialect" - } - }, - "vendors": { - "type": "array", - "description": "Supported vendors for custom extensions (enumeration definition)", - "items": { - "$ref": "#/$defs/Vendor" - } - }, - "semantic_model": { - "type": "array", - "description": "Collection of semantic model definitions", - "items": { - "$ref": "#/$defs/SemanticModel" - } - } - }, - "required": ["version", "semantic_model"], - "additionalProperties": false, - "$defs": { - "Dialect": { - "type": "string", - "enum": ["ANSI_SQL", "SNOWFLAKE", "MDX", "TABLEAU", "DATABRICKS", "MAQL"], - "description": "Supported SQL and expression language dialects" - }, - "Vendor": { - "type": "string", - "enum": ["COMMON", "SNOWFLAKE", "SALESFORCE", "DBT", "DATABRICKS", "GOODDATA"], - "description": "Supported vendors for custom extensions" - }, - "AIContext": { - "description": "Additional context for AI tools", - "oneOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": { - "instructions": { - "type": "string", - "description": "Instructions for AI on how to use this entity" - }, - "synonyms": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Alternative names and terms" - }, - "examples": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Sample questions or use cases" - } - }, - "additionalProperties": true - } - ] - }, - "CustomExtension": { - "type": "object", - "description": "Vendor-specific attributes for extensibility", - "properties": { - "vendor_name": { - "$ref": "#/$defs/Vendor" - }, - "data": { - "type": "string", - "description": "JSON string containing vendor-specific data" - } - }, - "required": ["vendor_name", "data"], - "additionalProperties": false - }, - "DialectExpression": { - "type": "object", - "description": "Expression in a specific dialect", - "properties": { - "dialect": { - "$ref": "#/$defs/Dialect" - }, - "expression": { - "type": "string", - "description": "SQL or dialect-specific expression" - } - }, - "required": ["dialect", "expression"], - "additionalProperties": false - }, - "Expression": { - "type": "object", - "description": "Expression definition with multi-dialect support", - "properties": { - "dialects": { - "type": "array", - "items": { - "$ref": "#/$defs/DialectExpression" - }, - "minItems": 1 - } - }, - "required": ["dialects"], - "additionalProperties": false - }, - "Dimension": { - "type": "object", - "description": "Dimension metadata", - "properties": { - "is_time": { - "type": "boolean", - "description": "Indicates if this is a time-based dimension for temporal filtering" - } - }, - "additionalProperties": false - }, - "Field": { - "type": "object", - "description": "Row-level attribute for grouping, filtering, and metric expressions", - "properties": { - "name": { - "type": "string", - "description": "Unique identifier for the field within the dataset" - }, - "expression": { - "$ref": "#/$defs/Expression" - }, - "dimension": { - "$ref": "#/$defs/Dimension" - }, - "label": { - "type": "string", - "description": "Label for categorization" - }, - "description": { - "type": "string", - "description": "Human-readable description" - }, - "ai_context": { - "$ref": "#/$defs/AIContext" - }, - "custom_extensions": { - "type": "array", - "items": { - "$ref": "#/$defs/CustomExtension" - } - } - }, - "required": ["name", "expression"], - "additionalProperties": false - }, - "Dataset": { - "type": "object", - "description": "Logical dataset representing a business entity (fact or dimension table)", - "properties": { - "name": { - "type": "string", - "description": "Unique identifier for the dataset" - }, - "source": { - "type": "string", - "description": "Reference to underlying physical table/view (database.schema.table) or query" - }, - "primary_key": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Primary key columns (single or composite)" - }, - "unique_keys": { - "type": "array", - "items": { - "type": "array", - "items": { - "type": "string" - } - }, - "description": "Array of unique key definitions (each can be single or composite)" - }, - "description": { - "type": "string", - "description": "Human-readable description" - }, - "ai_context": { - "$ref": "#/$defs/AIContext" - }, - "fields": { - "type": "array", - "items": { - "$ref": "#/$defs/Field" - } - }, - "custom_extensions": { - "type": "array", - "items": { - "$ref": "#/$defs/CustomExtension" - } - } - }, - "required": ["name", "source"], - "additionalProperties": false - }, - "Relationship": { - "type": "object", - "description": "Foreign key relationship between datasets", - "properties": { - "name": { - "type": "string", - "description": "Unique identifier for the relationship" - }, - "from": { - "type": "string", - "description": "Dataset on the many side of the relationship" - }, - "to": { - "type": "string", - "description": "Dataset on the one side of the relationship" - }, - "from_columns": { - "type": "array", - "items": { - "type": "string" - }, - "minItems": 1, - "description": "Foreign key columns in the 'from' dataset" - }, - "to_columns": { - "type": "array", - "items": { - "type": "string" - }, - "minItems": 1, - "description": "Primary/unique key columns in the 'to' dataset" - }, - "ai_context": { - "$ref": "#/$defs/AIContext" - }, - "custom_extensions": { - "type": "array", - "items": { - "$ref": "#/$defs/CustomExtension" - } - } - }, - "required": ["name", "from", "to", "from_columns", "to_columns"], - "additionalProperties": false - }, - "Metric": { - "type": "object", - "description": "Quantitative measure defined on business data", - "properties": { - "name": { - "type": "string", - "description": "Unique identifier for the metric" - }, - "expression": { - "$ref": "#/$defs/Expression" - }, - "description": { - "type": "string", - "description": "Human-readable description of what the metric measures" - }, - "ai_context": { - "$ref": "#/$defs/AIContext" - }, - "custom_extensions": { - "type": "array", - "items": { - "$ref": "#/$defs/CustomExtension" - } - } - }, - "required": ["name", "expression"], - "additionalProperties": false - }, - "SemanticModel": { - "type": "object", - "description": "Top-level container representing a complete semantic model", - "properties": { - "name": { - "type": "string", - "description": "Unique identifier for the semantic model" - }, - "description": { - "type": "string", - "description": "Human-readable description" - }, - "ai_context": { - "$ref": "#/$defs/AIContext" - }, - "datasets": { - "type": "array", - "items": { - "$ref": "#/$defs/Dataset" - }, - "minItems": 1, - "description": "Collection of logical datasets" - }, - "relationships": { - "type": "array", - "items": { - "$ref": "#/$defs/Relationship" - }, - "description": "Defines how datasets are connected" - }, - "metrics": { - "type": "array", - "items": { - "$ref": "#/$defs/Metric" - }, - "description": "Quantifiable measures spanning datasets" - }, - "custom_extensions": { - "type": "array", - "items": { - "$ref": "#/$defs/CustomExtension" - } - } - }, - "required": ["name", "datasets"], - "additionalProperties": false - } - } -} diff --git a/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/OsiDocumentValidatorTest.java b/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/OsiDocumentValidatorTest.java deleted file mode 100644 index 2f8adc8c014..00000000000 --- a/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/OsiDocumentValidatorTest.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.catalog.semanticmodel; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -import org.apache.iceberg.exceptions.BadRequestException; -import org.junit.jupiter.api.Test; - -class OsiDocumentValidatorTest { - - private static final int MAX_DOC = 1024 * 1024; - private static final int MAX_EXPR = 16 * 1024; - - private static final String VALID_MODEL = - "[{\"name\":\"m\",\"datasets\":[{\"name\":\"d\",\"source\":\"sales.store_sales\"}]}]"; - - @Test - void acceptsValidDocument() { - var node = OsiDocumentValidator.validate("0.1.1", VALID_MODEL, MAX_DOC, MAX_EXPR); - assertThat(node.isArray()).isTrue(); - assertThat(node.get(0).get("name").asText()).isEqualTo("m"); - } - - @Test - void rejectsUnsupportedSpecVersion() { - assertThatThrownBy(() -> OsiDocumentValidator.validate("0.2.0", VALID_MODEL, MAX_DOC, MAX_EXPR)) - .isInstanceOf(BadRequestException.class) - .hasMessageContaining("Unsupported OSI spec version"); - } - - @Test - void rejectsUnknownField() { - String withUnknown = - "[{\"name\":\"m\",\"bogus\":1,\"datasets\":[{\"name\":\"d\",\"source\":\"a.b\"}]}]"; - assertThatThrownBy(() -> OsiDocumentValidator.validate("0.1.1", withUnknown, MAX_DOC, MAX_EXPR)) - .isInstanceOf(BadRequestException.class) - .hasMessageContaining("OSI schema validation"); - } - - @Test - void rejectsMissingRequiredField() { - // dataset missing the required 'source' - String missingSource = "[{\"name\":\"m\",\"datasets\":[{\"name\":\"d\"}]}]"; - assertThatThrownBy( - () -> OsiDocumentValidator.validate("0.1.1", missingSource, MAX_DOC, MAX_EXPR)) - .isInstanceOf(BadRequestException.class) - .hasMessageContaining("OSI schema validation"); - } - - @Test - void rejectsInvalidJson() { - assertThatThrownBy(() -> OsiDocumentValidator.validate("0.1.1", "{not json", MAX_DOC, MAX_EXPR)) - .isInstanceOf(BadRequestException.class) - .hasMessageContaining("not valid JSON"); - } - - @Test - void rejectsOversizeDocument() { - assertThatThrownBy(() -> OsiDocumentValidator.validate("0.1.1", VALID_MODEL, 10, MAX_EXPR)) - .isInstanceOf(BadRequestException.class) - .hasMessageContaining("exceeds the maximum"); - } - - @Test - void rejectsOversizeExpression() { - String bigExpr = "SUM(" + "x".repeat(100) + ")"; - String model = - "[{\"name\":\"m\",\"datasets\":[{\"name\":\"d\",\"source\":\"a.b\"}]," - + "\"metrics\":[{\"name\":\"total\",\"expression\":{\"dialects\":" - + "[{\"dialect\":\"ANSI_SQL\",\"expression\":\"" - + bigExpr - + "\"}]}}]}]"; - // Schema-valid, but the expression exceeds a tiny per-expression cap. - assertThatThrownBy(() -> OsiDocumentValidator.validate("0.1.1", model, MAX_DOC, 10)) - .isInstanceOf(BadRequestException.class) - .hasMessageContaining("expression"); - } -} diff --git a/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogTest.java b/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogTest.java index 5ee4a994243..411ac7ca294 100644 --- a/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogTest.java +++ b/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogTest.java @@ -76,8 +76,6 @@ class SemanticModelCatalogTest { SemanticModelIdentifier.builder().setNamespace(List.of("sales")).setName(MODEL).build(); private static final String VALID_MODEL_JSON = "[{\"name\":\"m\",\"datasets\":[{\"name\":\"d\",\"source\":\"sales.store_sales\"}]}]"; - private static final int MAX_DOC = 1024 * 1024; - private static final int MAX_EXPR = 16 * 1024; private PolarisResolutionManifestCatalogView view; private PolarisMetaStoreManager metaStoreManager; @@ -100,7 +98,7 @@ void setUp() { when(view.getResolvedPath(ResolvedPathKey.ofNamespace(NS))) .thenReturn(path(catalogEntity, namespaceEntity)); - catalog = new SemanticModelCatalog(metaStoreManager, callContext, view, MAX_DOC, MAX_EXPR); + catalog = new SemanticModelCatalog(metaStoreManager, callContext, view); } private SemanticModelDocument doc(String semanticModelJson) { diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 96490e2fbdc..2e9b8983249 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -82,7 +82,6 @@ jakarta-servlet-api = { module = "jakarta.servlet:jakarta.servlet-api", version jakarta-validation-api = { module = "jakarta.validation:jakarta.validation-api", version = "3.1.1" } jakarta-ws-rs-api = { module = "jakarta.ws.rs:jakarta.ws.rs-api", version = "4.0.0" } javax-servlet-api = { module = "javax.servlet:javax.servlet-api", version = "4.0.1" } -json-schema-validator = { module = "com.networknt:json-schema-validator", version = "1.5.9" } jspecify = { module = "org.jspecify:jspecify", version = "1.0.0" } jcstress-core = { module = "org.openjdk.jcstress:jcstress-core", version = "0.16" } jmh-core = { module = "org.openjdk.jmh:jmh-core", version.ref = "jmh" } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java index cdb1812df42..87aecac3026 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java @@ -517,24 +517,6 @@ public static void enforceFeatureEnabledOrThrow( .defaultValue(false) // beta feature, keep it off by default .buildFeatureConfiguration(); - public static final FeatureConfiguration SEMANTIC_MODEL_MAX_DOCUMENT_BYTES = - PolarisConfiguration.builder() - .key("SEMANTIC_MODEL_MAX_DOCUMENT_BYTES") - .description( - "The maximum size, in bytes, of a stored OSI semantic-model document. Writes whose " - + "document exceeds this cap are rejected with 400.") - .defaultValue(1024 * 1024) // 1 MB - .buildFeatureConfiguration(); - - public static final FeatureConfiguration SEMANTIC_MODEL_MAX_EXPRESSION_BYTES = - PolarisConfiguration.builder() - .key("SEMANTIC_MODEL_MAX_EXPRESSION_BYTES") - .description( - "The maximum size, in bytes, of a single expression fragment inside an OSI " - + "semantic-model document. Guards against metadata-as-blob abuse.") - .defaultValue(16 * 1024) // 16 KB - .buildFeatureConfiguration(); - public static final FeatureConfiguration> SUPPORTED_CATALOG_CONNECTION_TYPES = PolarisConfiguration.>builder() .key("SUPPORTED_CATALOG_CONNECTION_TYPES") diff --git a/runtime/server/distribution/LICENSE b/runtime/server/distribution/LICENSE index 33d9d4891f4..014efbe619b 100644 --- a/runtime/server/distribution/LICENSE +++ b/runtime/server/distribution/LICENSE @@ -516,15 +516,6 @@ License: MIT License -------------------------------------------------------------------------------- -This product bundles ethlo Internet Time Utility. - -* Maven group:artifact IDs: com.ethlo.time:itu - -Project URL: https://github.com/ethlo/itu -License: Apache License 2.0 - https://www.apache.org/licenses/LICENSE-2.0.txt - --------------------------------------------------------------------------------- - This product bundles FasterXML Java Classmate. * Maven group:artifact IDs: com.fasterxml:classmate @@ -926,15 +917,6 @@ License: MIT License -------------------------------------------------------------------------------- -This product bundles networknt JSON Schema Validator. - -* Maven group:artifact IDs: com.networknt:json-schema-validator - -Project URL: https://github.com/networknt/json-schema-validator -License: Apache License 2.0 - https://www.apache.org/licenses/LICENSE-2.0.txt - --------------------------------------------------------------------------------- - This product bundles Nimbus Jose JWT. * Maven group:artifact IDs: com.nimbusds:nimbus-jose-jwt From 5a60dd8cb3e9f879e6276012153dda9388b61003 Mon Sep 17 00:00:00 2001 From: Yufei Gu Date: Wed, 8 Jul 2026 18:29:55 -0700 Subject: [PATCH 04/13] Split schema validation and dataset parsing --- .../semanticmodel/SemanticModelCatalog.java | 58 ++++++++++--------- .../SemanticModelCatalogTest.java | 16 +++++ 2 files changed, 48 insertions(+), 26 deletions(-) diff --git a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalog.java b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalog.java index b68b7ed67d7..ccead8253ec 100644 --- a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalog.java +++ b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalog.java @@ -18,6 +18,7 @@ */ package org.apache.polaris.service.catalog.semanticmodel; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Splitter; @@ -102,8 +103,9 @@ public LoadSemanticModelResponse createSemanticModel( throw new AlreadyExistsException("Semantic model already exists: %s", identifier.getName()); } - // Parse the document and resolve its source tables before persisting. - validateDocumentAndSources(document); + // Validate the document schema, then resolve its source tables before persisting. + JsonNode semanticModel = validateDocument(document); + resolveAndValidateSources(semanticModel); SemanticModelEntity entity = new SemanticModelEntity.Builder(namespace, identifier.getName()) @@ -190,7 +192,8 @@ public LoadSemanticModelResponse updateSemanticModel( expectedVersion, currentVersion)); } - validateDocumentAndSources(document); + JsonNode semanticModel = validateDocument(document); + resolveAndValidateSources(semanticModel); SemanticModelEntity newEntity = new SemanticModelEntity.Builder(current) @@ -240,8 +243,6 @@ public void dropSemanticModel(SemanticModelIdentifier identifier) { } } - // ---- helpers ---- - private PolarisResolvedPathWrapper resolveModelPathOrThrow(SemanticModelIdentifier identifier) { Namespace namespace = toNamespace(identifier); PolarisResolvedPathWrapper resolved = @@ -260,46 +261,51 @@ private SemanticModelEntity resolveModelOrThrow(SemanticModelIdentifier identifi } /** - * Parses the OSI document body and resolves every {@code dataset.source} to a {@code TABLE_LIKE} - * entity. Shared by create and update. Schema/size validation is a separate concern handled by a - * {@link SemanticDocumentValidator} implementation (not yet wired in this phase). + * Validates the OSI document against the Ossie JSON schema and returns the parsed {@code + * semantic_model} body. Currently only checks that the required body is present and well-formed + * JSON; full schema validation is deferred to a {@link SemanticDocumentValidator} implementation. + * + * @return the parsed {@code semantic_model} node + * @throws BadRequestException if the body is missing/blank or not valid JSON */ - private void validateDocumentAndSources(SemanticModelDocument document) { + private JsonNode validateDocument(SemanticModelDocument document) { String body = document.getSemanticModel(); if (body == null || body.isBlank()) { - return; + throw new BadRequestException("Semantic model document must not be empty"); } - JsonNode semanticModel; try { - semanticModel = MAPPER.readTree(body); - } catch (com.fasterxml.jackson.core.JsonProcessingException e) { + // TODO: delegate full Ossie JSON-schema validation to SemanticDocumentValidator. + return MAPPER.readTree(body); + } catch (JsonProcessingException e) { throw new BadRequestException( "Field 'semantic_model' is not valid JSON: %s", e.getOriginalMessage()); } - resolveSourcesOrThrow(semanticModel); - // TODO call the SemanticDocumentValidator when it's ready } /** - * Resolves every {@code dataset.source} in the OSI document to a {@code TABLE_LIKE} entity in the - * current catalog. Fails with 400 and a JSON-Pointer to the offending dataset if any source does - * not resolve. Column-level checks are deferred (F5). + * Resolves and validates every {@code dataset.source} in the parsed OSI document against the + * current catalog. Every dataset must define a string {@code source}; a missing or non-string + * source, or one that does not resolve to a {@code TABLE_LIKE} entity, fails with 400 and a + * JSON-Pointer to the offending dataset. Column-level checks are deferred (F5). */ - private void resolveSourcesOrThrow(JsonNode semanticModel) { + private void resolveAndValidateSources(JsonNode semanticModel) { if (!semanticModel.isArray()) { return; } - for (int m = 0; m < semanticModel.size(); m++) { - JsonNode datasets = semanticModel.get(m).get("datasets"); + + for (int modelIdx = 0; modelIdx < semanticModel.size(); modelIdx++) { + JsonNode datasets = semanticModel.get(modelIdx).get("datasets"); if (datasets == null || !datasets.isArray()) { continue; } - for (int d = 0; d < datasets.size(); d++) { - JsonNode source = datasets.get(d).get("source"); - if (source != null && source.isTextual()) { - String pointer = String.format("/semantic_model/%d/datasets/%d/source", m, d); - resolveSourceOrThrow(source.asText(), pointer); + for (int datasetIdx = 0; datasetIdx < datasets.size(); datasetIdx++) { + String pointer = String.format("/semantic_model/%d/datasets/%d/source", modelIdx, datasetIdx); + JsonNode source = datasets.get(datasetIdx).get("source"); + if (source == null || !source.isTextual()) { + throw new BadRequestException( + "Semantic model dataset at %s must define a string 'source'", pointer); } + resolveSourceOrThrow(source.asText(), pointer); } } } diff --git a/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogTest.java b/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogTest.java index 411ac7ca294..fdfe7d88369 100644 --- a/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogTest.java +++ b/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogTest.java @@ -155,6 +155,22 @@ void createResolvesSourcesAndPersists() { assertThat(response.getEntityVersion()).isEqualTo("1"); } + @Test + void createRejectsEmptyDocument() { + assertThatThrownBy(() -> catalog.createSemanticModel(IDENTIFIER, doc(""))) + .isInstanceOf(BadRequestException.class) + .hasMessageContaining("must not be empty"); + } + + @Test + void createRejectsDatasetWithoutSource() { + String noSource = "[{\"name\":\"m\",\"datasets\":[{\"name\":\"d\"}]}]"; + assertThatThrownBy(() -> catalog.createSemanticModel(IDENTIFIER, doc(noSource))) + .isInstanceOf(BadRequestException.class) + .hasMessageContaining("/semantic_model/0/datasets/0/source") + .hasMessageContaining("must define a string 'source'"); + } + @Test void createRejectsUnresolvedSource() { // No stub for the source table -> passthrough resolution returns null. From a40ffaab488b898d2cd3904598aaee5f12fb4ca0 Mon Sep 17 00:00:00 2001 From: Yufei Gu Date: Wed, 8 Jul 2026 18:38:00 -0700 Subject: [PATCH 05/13] Extract dataset.source separator into a named constant Address review feedback: define the '.' used to split an OSI dataset.source into a documented SOURCE_SEPARATOR constant so the dot-splitting per the IRC object-identifier scheme is explicit. --- .../catalog/semanticmodel/SemanticModelCatalog.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalog.java b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalog.java index ccead8253ec..80a18d67182 100644 --- a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalog.java +++ b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalog.java @@ -66,6 +66,12 @@ public class SemanticModelCatalog { private static final Logger LOGGER = LoggerFactory.getLogger(SemanticModelCatalog.class); private static final ObjectMapper MAPPER = new ObjectMapper(); + /** + * Separator between the namespace path and the name in an OSI {@code dataset.source}, per the IRC + * catalog object-identifier scheme (e.g. {@code sales.store_sales}). + */ + private static final char SOURCE_SEPARATOR = '.'; + private final CallContext callContext; private final PolarisResolutionManifestCatalogView resolvedEntityView; private final CatalogEntity catalogEntity; @@ -299,7 +305,8 @@ private void resolveAndValidateSources(JsonNode semanticModel) { continue; } for (int datasetIdx = 0; datasetIdx < datasets.size(); datasetIdx++) { - String pointer = String.format("/semantic_model/%d/datasets/%d/source", modelIdx, datasetIdx); + String pointer = + String.format("/semantic_model/%d/datasets/%d/source", modelIdx, datasetIdx); JsonNode source = datasets.get(datasetIdx).get("source"); if (source == null || !source.isTextual()) { throw new BadRequestException( @@ -325,7 +332,7 @@ private void resolveSourceOrThrow(String source, String pointer) { } private TableIdentifier parseSource(String source, String pointer) { - List parts = Splitter.on('.').splitToList(source); + List parts = Splitter.on(SOURCE_SEPARATOR).splitToList(source); if (parts.size() < 2 || parts.stream().anyMatch(String::isEmpty)) { throw new BadRequestException( "Semantic model source '%s' at %s must be a namespace-qualified identifier " From e6ea1ce824a81485a2716a98ca066109837c704c Mon Sep 17 00:00:00 2001 From: Yufei Gu Date: Wed, 8 Jul 2026 18:46:58 -0700 Subject: [PATCH 06/13] Refactor --- .../semanticmodel/SemanticModelCatalogHandler.java | 5 ----- .../service/catalog/common/CatalogHandler.java | 14 ++++++++++++++ .../catalog/iceberg/IcebergCatalogHandler.java | 5 ----- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandler.java b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandler.java index 7859210bf9e..60da706a100 100644 --- a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandler.java +++ b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandler.java @@ -22,7 +22,6 @@ import org.apache.iceberg.catalog.Namespace; import org.apache.polaris.core.auth.PolarisAuthorizableOperation; import org.apache.polaris.core.catalog.PolarisCatalogHelpers; -import org.apache.polaris.core.config.FeatureConfiguration; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; import org.apache.polaris.core.persistence.pagination.PageToken; @@ -100,10 +99,6 @@ public void dropSemanticModel(SemanticModelIdentifier identifier) { semanticModelCatalog.dropSemanticModel(identifier); } - private boolean shouldDecodeToken() { - return realmConfig().getConfig(FeatureConfiguration.LIST_PAGINATION_ENABLED); - } - private void authorizeBasicSemanticModelOperationOrThrow( PolarisAuthorizableOperation op, SemanticModelIdentifier identifier) { Namespace namespace = Namespace.of(identifier.getNamespace().toArray(new String[0])); diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/common/CatalogHandler.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/common/CatalogHandler.java index 2995bdba7ec..82e1df81dea 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/common/CatalogHandler.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/common/CatalogHandler.java @@ -40,9 +40,11 @@ import org.apache.polaris.core.auth.RenameAuthorizationIntent; import org.apache.polaris.core.auth.SingleTargetAuthorizationIntent; import org.apache.polaris.core.catalog.PolarisCatalogHelpers; +import org.apache.polaris.core.config.FeatureConfiguration; import org.apache.polaris.core.config.RealmConfig; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; @@ -95,6 +97,18 @@ protected PolarisResolutionManifest newResolutionManifest() { /** Initialize the catalog once authorized. Called after all `authorize...` methods. */ protected abstract void initializeCatalog(); + /** + * Whether opaque list page-tokens should be decoded, honoring the {@code LIST_PAGINATION_ENABLED} + * setting (with a per-catalog override when a catalog has been resolved). Shared by catalog + * handlers that page list results. + */ + protected boolean shouldDecodeToken() { + CatalogEntity catalogEntity = resolutionManifest.getResolvedCatalogEntity(); + return catalogEntity == null + ? realmConfig().getConfig(FeatureConfiguration.LIST_PAGINATION_ENABLED) + : realmConfig().getConfig(FeatureConfiguration.LIST_PAGINATION_ENABLED, catalogEntity); + } + protected void authorizeBasicNamespaceOperationOrThrow( PolarisAuthorizableOperation op, Namespace namespace) { authorizeBasicNamespaceOperationOrThrow(op, namespace, null, null, null); diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java index d60194306bf..3ccc2c31579 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java @@ -20,7 +20,6 @@ import static java.util.Objects.requireNonNull; import static org.apache.polaris.core.config.FeatureConfiguration.ALLOW_FEDERATED_CATALOGS_CREDENTIAL_VENDING; -import static org.apache.polaris.core.config.FeatureConfiguration.LIST_PAGINATION_ENABLED; import static org.apache.polaris.service.catalog.AccessDelegationMode.VENDED_CREDENTIALS; import static org.apache.polaris.service.catalog.common.ExceptionUtils.alreadyExistsExceptionForTableLikeEntity; import static org.apache.polaris.service.catalog.common.ExceptionUtils.notFoundExceptionForTableLikeEntity; @@ -211,10 +210,6 @@ private CatalogEntity getResolvedCatalogEntity() { return catalogEntity; } - private boolean shouldDecodeToken() { - return realmConfig().getConfig(LIST_PAGINATION_ENABLED, getResolvedCatalogEntity()); - } - @Override protected void initializeCatalog() { CatalogEntity resolvedCatalogEntity = getResolvedCatalogEntity(); From 5f50fc863d7464829bc37f97d5a14939df389cf0 Mon Sep 17 00:00:00 2001 From: Yufei Gu Date: Thu, 9 Jul 2026 14:49:01 -0700 Subject: [PATCH 07/13] add todo --- .../nosql/coretypes/mapping/EntityObjMappings.java | 4 ++-- .../nosql/coretypes/mapping/TestEntityObjMappings.java | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/persistence/nosql/persistence/metastore-types/src/main/java/org/apache/polaris/persistence/nosql/coretypes/mapping/EntityObjMappings.java b/persistence/nosql/persistence/metastore-types/src/main/java/org/apache/polaris/persistence/nosql/coretypes/mapping/EntityObjMappings.java index c4f8104b940..939ac3f22c6 100644 --- a/persistence/nosql/persistence/metastore-types/src/main/java/org/apache/polaris/persistence/nosql/coretypes/mapping/EntityObjMappings.java +++ b/persistence/nosql/persistence/metastore-types/src/main/java/org/apache/polaris/persistence/nosql/coretypes/mapping/EntityObjMappings.java @@ -90,8 +90,8 @@ public final class EntityObjMappings { static { var missingEntityTypes = new HashSet<>(Set.of(PolarisEntityType.values())); missingEntityTypes.remove(PolarisEntityType.NULL_TYPE); - // SEMANTIC_MODEL is not yet supported by the NoSQL backend; its mapping lands in a follow-up. - // Exclude it here so the NoSQL module still loads while the type exists in the shared enum. + // TODO: SEMANTIC_MODEL is not yet supported by the NoSQL backend; Excluded here so the NoSQL + // module still loads while the type exists in the shared enum. missingEntityTypes.remove(PolarisEntityType.SEMANTIC_MODEL); missingEntityTypes.removeAll(BY_ENTITY_TYPE.keySet()); checkState( diff --git a/persistence/nosql/persistence/metastore-types/src/test/java/org/apache/polaris/persistence/nosql/coretypes/mapping/TestEntityObjMappings.java b/persistence/nosql/persistence/metastore-types/src/test/java/org/apache/polaris/persistence/nosql/coretypes/mapping/TestEntityObjMappings.java index 50a84e4da55..f7db74e2d95 100644 --- a/persistence/nosql/persistence/metastore-types/src/test/java/org/apache/polaris/persistence/nosql/coretypes/mapping/TestEntityObjMappings.java +++ b/persistence/nosql/persistence/metastore-types/src/test/java/org/apache/polaris/persistence/nosql/coretypes/mapping/TestEntityObjMappings.java @@ -119,7 +119,8 @@ public void entityObjMapping(PolarisEntityType entityType, PolarisEntitySubType static Stream entityObjMapping() { return Arrays.stream(PolarisEntityType.values()) .filter(t -> t != PolarisEntityType.NULL_TYPE) - // SEMANTIC_MODEL is intentionally not mapped by the NoSQL backend yet (follow-up). + // TODO: SEMANTIC_MODEL is not yet mapped by the NoSQL backend; drop this filter once its + // mapping is added. .filter(t -> t != PolarisEntityType.SEMANTIC_MODEL) .flatMap( t -> { From 57cc7494a4fd2e3027965088bda3b2d6659ec8e0 Mon Sep 17 00:00:00 2001 From: Yufei Gu Date: Thu, 9 Jul 2026 16:01:47 -0700 Subject: [PATCH 08/13] resolve comments --- .../apache/polaris/core/auth/RbacOperationSemantics.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/auth/RbacOperationSemantics.java b/polaris-core/src/main/java/org/apache/polaris/core/auth/RbacOperationSemantics.java index 20fa79f6525..e0628805f33 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/auth/RbacOperationSemantics.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/auth/RbacOperationSemantics.java @@ -414,9 +414,9 @@ private static void register( register(LIST_POLICY, POLICY_LIST); // Semantic-model operations. - // Interim (Phase 3): gated behind the coarse CATALOG_MANAGE_CONTENT privilege, matching the - // design's "coarser-grained access reuses existing higher-level privileges" posture. The - // dedicated SEMANTIC_MODEL_* privilege matrix and read-time enforcement modes land in Phase 2. + // Interim: gated behind the coarse CATALOG_MANAGE_CONTENT privilege, matching the + // design's "coarser-grained access reuses existing higher-level privileges" posture. + // TODO: The dedicated SEMANTIC_MODEL_* privilege matrix will be implemented in a followup. register(CREATE_SEMANTIC_MODEL, CATALOG_MANAGE_CONTENT); register(LOAD_SEMANTIC_MODEL, CATALOG_MANAGE_CONTENT); register(UPDATE_SEMANTIC_MODEL, CATALOG_MANAGE_CONTENT); From 4a57220cad1a8ef74d6b29c22a8175325a4ccc34 Mon Sep 17 00:00:00 2001 From: Yufei Gu Date: Fri, 10 Jul 2026 10:06:17 -0700 Subject: [PATCH 09/13] Fix semantic-model create/update resolution-manifest failures Create and update crashed at runtime against the real resolution manifest: both probed keys via getPassthroughResolvedPath that were never registered via addPassthroughPath, and that method hard-throws on an unknown key. The mock-based unit test stubbed the view, so the failure was invisible. - Drop the create existence pre-check that probed the (unregistered) semantic-model path; rely on createEntityIfNotExists returning ENTITY_ALREADY_EXISTS, which is already handled. - Resolve each dataset.source with a fresh single-use PolarisResolutionManifest instead of the request manifest, since sources are parsed from the opaque document and cannot be pre-registered. SemanticModelCatalog now takes a ResolutionManifestFactory and PolarisPrincipal. Add real-manifest handler tests (SemanticModelCatalogHandlerCrudTest, SemanticModelCatalogHandlerAuthzTest) driven through TestServices, sharing a common bootstrap base. These exercise the strict manifest that the Mockito unit test cannot, and cover the CATALOG_MANAGE_CONTENT deny path. --- extensions/semantic-models/build.gradle.kts | 11 ++ .../semanticmodel/SemanticModelCatalog.java | 38 ++-- .../SemanticModelCatalogHandler.java | 7 +- ...stractSemanticModelCatalogHandlerTest.java | 175 ++++++++++++++++++ .../SemanticModelCatalogHandlerAuthzTest.java | 94 ++++++++++ .../SemanticModelCatalogHandlerCrudTest.java | 87 +++++++++ .../SemanticModelCatalogTest.java | 24 ++- 7 files changed, 421 insertions(+), 15 deletions(-) create mode 100644 extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/AbstractSemanticModelCatalogHandlerTest.java create mode 100644 extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandlerAuthzTest.java create mode 100644 extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandlerCrudTest.java diff --git a/extensions/semantic-models/build.gradle.kts b/extensions/semantic-models/build.gradle.kts index ff03fd9da01..a2a7b3d1f3f 100644 --- a/extensions/semantic-models/build.gradle.kts +++ b/extensions/semantic-models/build.gradle.kts @@ -73,6 +73,17 @@ dependencies { testImplementation(platform(libs.junit.bom)) testImplementation("org.junit.jupiter:junit-jupiter") testImplementation(libs.assertj.core) + + // Real-manifest handler tests bootstrap services via the runtime-service test fixtures + // (TestServices), which is the only way to reproduce the strict resolution-manifest behavior + // that the pure-Mockito unit test cannot. + testImplementation(testFixtures(project(":polaris-runtime-service"))) + testImplementation(project(":polaris-api-management-model")) + testImplementation(project(":polaris-api-management-service")) + testImplementation(project(":polaris-api-iceberg-service")) + testImplementation(project(":polaris-api-catalog-service")) + testCompileOnly(project(":polaris-immutables")) + testAnnotationProcessor(project(":polaris-immutables", configuration = "processor")) } val rootDir = rootProject.layout.projectDirectory diff --git a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalog.java b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalog.java index 80a18d67182..b613f7b0617 100644 --- a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalog.java +++ b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalog.java @@ -31,6 +31,8 @@ import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.exceptions.AlreadyExistsException; import org.apache.iceberg.exceptions.BadRequestException; +import org.apache.polaris.core.auth.PolarisPrincipal; +import org.apache.polaris.core.catalog.PolarisCatalogHelpers; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.PolarisBaseEntity; @@ -42,8 +44,11 @@ import org.apache.polaris.core.persistence.dao.entity.EntityResult; import org.apache.polaris.core.persistence.dao.entity.ListEntitiesResult; import org.apache.polaris.core.persistence.pagination.PageToken; +import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifest; import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifestCatalogView; +import org.apache.polaris.core.persistence.resolver.ResolutionManifestFactory; import org.apache.polaris.core.persistence.resolver.ResolvedPathKey; +import org.apache.polaris.core.persistence.resolver.ResolverPath; import org.apache.polaris.core.semantic.SemanticModelEntity; import org.apache.polaris.core.semantic.exceptions.NoSuchSemanticModelException; import org.apache.polaris.core.semantic.exceptions.SemanticModelVersionMismatchException; @@ -77,16 +82,22 @@ public class SemanticModelCatalog { private final CatalogEntity catalogEntity; private final long catalogId; private final PolarisMetaStoreManager metaStoreManager; + private final ResolutionManifestFactory resolutionManifestFactory; + private final PolarisPrincipal principal; public SemanticModelCatalog( PolarisMetaStoreManager metaStoreManager, CallContext callContext, - PolarisResolutionManifestCatalogView resolvedEntityView) { + PolarisResolutionManifestCatalogView resolvedEntityView, + ResolutionManifestFactory resolutionManifestFactory, + PolarisPrincipal principal) { this.callContext = callContext; this.resolvedEntityView = resolvedEntityView; this.catalogEntity = resolvedEntityView.getResolvedCatalogEntity(); this.catalogId = catalogEntity.getId(); this.metaStoreManager = metaStoreManager; + this.resolutionManifestFactory = resolutionManifestFactory; + this.principal = principal; } public LoadSemanticModelResponse createSemanticModel( @@ -101,15 +112,10 @@ public LoadSemanticModelResponse createSemanticModel( } List catalogPath = resolvedParent.getRawFullPath(); - PolarisResolvedPathWrapper existing = - resolvedEntityView.getPassthroughResolvedPath( - ResolvedPathKey.ofSemanticModel(namespace, identifier.getName()), - PolarisEntitySubType.NULL_SUBTYPE); - if (existing != null && existing.getRawLeafEntity() != null) { - throw new AlreadyExistsException("Semantic model already exists: %s", identifier.getName()); - } - - // Validate the document schema, then resolve its source tables before persisting. + // Validate the document schema, then resolve its source tables before persisting. Duplicate + // names are caught by createEntityIfNotExists below (ENTITY_ALREADY_EXISTS), so there is no + // separate existence pre-check: the semantic-model path is not registered on this request's + // resolution manifest, and probing it there would fail. JsonNode semanticModel = validateDocument(document); resolveAndValidateSources(semanticModel); @@ -319,8 +325,18 @@ private void resolveAndValidateSources(JsonNode semanticModel) { private void resolveSourceOrThrow(String source, String pointer) { TableIdentifier tableIdentifier = parseSource(source, pointer); + // Sources are parsed from the opaque document, so they cannot be pre-registered on this + // request's resolution manifest. Resolve each one with a fresh single-use manifest that + // registers the table path as an optional passthrough. + PolarisResolutionManifest manifest = + resolutionManifestFactory.createResolutionManifest(principal, catalogEntity.getName()); + manifest.addPassthroughPath( + new ResolverPath( + PolarisCatalogHelpers.tableIdentifierToList(tableIdentifier), + PolarisEntityType.TABLE_LIKE, + true /* optional */)); PolarisResolvedPathWrapper resolved = - resolvedEntityView.getPassthroughResolvedPath( + manifest.getPassthroughResolvedPath( ResolvedPathKey.ofTableLike(tableIdentifier), PolarisEntitySubType.ANY_SUBTYPE); if (resolved == null || resolved.getRawLeafEntity() == null diff --git a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandler.java b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandler.java index 60da706a100..13fc8a8306e 100644 --- a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandler.java +++ b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandler.java @@ -54,7 +54,12 @@ public abstract class SemanticModelCatalogHandler extends CatalogHandler { @Override protected void initializeCatalog() { this.semanticModelCatalog = - new SemanticModelCatalog(metaStoreManager(), callContext(), this.resolutionManifest); + new SemanticModelCatalog( + metaStoreManager(), + callContext(), + this.resolutionManifest, + resolutionManifestFactory(), + polarisPrincipal()); } public LoadSemanticModelResponse createSemanticModel( diff --git a/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/AbstractSemanticModelCatalogHandlerTest.java b/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/AbstractSemanticModelCatalogHandlerTest.java new file mode 100644 index 00000000000..111bffe3e0c --- /dev/null +++ b/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/AbstractSemanticModelCatalogHandlerTest.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.catalog.semanticmodel; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; + +import jakarta.ws.rs.core.Response; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.rest.requests.CreateNamespaceRequest; +import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.CatalogProperties; +import org.apache.polaris.core.admin.model.CreateCatalogRequest; +import org.apache.polaris.core.admin.model.FileStorageConfigInfo; +import org.apache.polaris.core.admin.model.StorageConfigInfo; +import org.apache.polaris.core.auth.AuthorizationState; +import org.apache.polaris.core.auth.PolarisAuthorizer; +import org.apache.polaris.service.TestServices; +import org.apache.polaris.service.catalog.semanticmodel.types.CreateSemanticModelRequest; +import org.apache.polaris.service.catalog.semanticmodel.types.SemanticModelDocument; +import org.apache.polaris.service.catalog.semanticmodel.types.SemanticModelIdentifier; +import org.apache.polaris.service.types.CreateGenericTableRequest; +import org.junit.jupiter.api.BeforeEach; +import org.mockito.Mockito; + +/** + * Shared bootstrap for {@link SemanticModelCatalogHandler} tests that run against the real + * resolution manifest via {@link TestServices}: it stands up a catalog, a namespace, and a source + * table ({@code ns1.t1}) so semantic-model {@code dataset.source} references resolve, and builds + * handlers with a caller-supplied authorizer. + */ +abstract class AbstractSemanticModelCatalogHandlerTest { + + protected static final String CATALOG_NAME = "sm-catalog"; + protected static final String CATALOG_BASE_LOCATION = "file:///tmp/polaris-sm-test"; + protected static final Namespace NS = Namespace.of("ns1"); + protected static final String SOURCE_TABLE = "t1"; + private static final UUID IDEMPOTENCY_KEY = new UUID(116617318654508422L, -7820829973016961092L); + + protected TestServices services; + + @BeforeEach + void bootstrap() { + services = + TestServices.builder() + .config( + Map.of( + "ALLOW_INSECURE_STORAGE_TYPES", + "true", + "SUPPORTED_CATALOG_STORAGE_TYPES", + List.of("FILE"))) + .build(); + createCatalogNamespaceAndTable(); + } + + /** Builds a handler wired to the given authorizer against the bootstrapped services. */ + protected SemanticModelCatalogHandler handler(PolarisAuthorizer authorizer) { + return ImmutableSemanticModelCatalogHandler.builder() + .catalogName(CATALOG_NAME) + .polarisPrincipal(services.principal()) + .callContext(services.newCallContext()) + .resolutionManifestFactory(services.resolutionManifestFactory()) + .metaStoreManager(services.metaStoreManager()) + .authorizer(authorizer) + .build(); + } + + /** + * Handler that skips authorization; use for fixtures and non-authz assertions. The authorizer is + * a mock stubbed to resolve the manifest (mirroring {@link TestServices}), since the base handler + * now drives resolution through {@code resolveAuthorizationInputs} rather than resolving inline. + */ + protected SemanticModelCatalogHandler passthroughHandler() { + PolarisAuthorizer authorizer = Mockito.mock(PolarisAuthorizer.class); + Mockito.doAnswer( + invocation -> { + AuthorizationState authzState = invocation.getArgument(0); + authzState.getResolutionManifest().resolveAll(); + return null; + }) + .when(authorizer) + .resolveAuthorizationInputs(any(), any()); + return handler(authorizer); + } + + protected static SemanticModelIdentifier identifier(String name) { + return SemanticModelIdentifier.builder() + .setNamespace(List.of(NS.levels())) + .setName(name) + .build(); + } + + protected static CreateSemanticModelRequest createRequest(String name, String model) { + return CreateSemanticModelRequest.builder().setName(name).setDocument(doc(model)).build(); + } + + protected static SemanticModelDocument doc(String model) { + return SemanticModelDocument.builder().setVersion("0.1.1").setSemanticModel(model).build(); + } + + protected static String modelJson(String source) { + return "[{\"name\":\"m\",\"datasets\":[{\"name\":\"d\",\"source\":\"" + source + "\"}]}]"; + } + + private void createCatalogNamespaceAndTable() { + Catalog catalogObject = + new Catalog( + Catalog.TypeEnum.INTERNAL, + CATALOG_NAME, + CatalogProperties.builder() + .setDefaultBaseLocation(CATALOG_BASE_LOCATION + "/" + CATALOG_NAME) + .build(), + 1725487592064L, + 1725487592064L, + 1, + FileStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.FILE) + .build()); + try (Response response = + services + .catalogsApi() + .createCatalog( + new CreateCatalogRequest(catalogObject), + services.realmContext(), + services.securityContext())) { + assertThat(response.getStatus()).isEqualTo(Response.Status.CREATED.getStatusCode()); + } + + try (Response response = + services + .restApi() + .createNamespace( + CATALOG_NAME, + CreateNamespaceRequest.builder().withNamespace(NS).build(), + IDEMPOTENCY_KEY, + services.realmContext(), + services.securityContext())) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + try (Response response = + services + .genericTableApi() + .createGenericTable( + CATALOG_NAME, + NS.toString(), + CreateGenericTableRequest.builder(SOURCE_TABLE, "iceberg") + .setBaseLocation(CATALOG_BASE_LOCATION + "/" + CATALOG_NAME + "/ns1/t1") + .build(), + null, + services.realmContext(), + services.securityContext())) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + } +} diff --git a/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandlerAuthzTest.java b/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandlerAuthzTest.java new file mode 100644 index 00000000000..84a0b41d1b5 --- /dev/null +++ b/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandlerAuthzTest.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.catalog.semanticmodel; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import org.apache.iceberg.exceptions.ForbiddenException; +import org.apache.polaris.core.auth.PolarisAuthorizerImpl; +import org.apache.polaris.service.TestServices; +import org.apache.polaris.service.catalog.semanticmodel.types.UpdateSemanticModelRequest; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Verifies that {@link SemanticModelCatalogHandler} enforces authorization: the five operations are + * gated behind {@code CATALOG_MANAGE_CONTENT} (see {@code RbacOperationSemantics}), so the + * unprivileged {@code test-principal} that {@link TestServices} bootstraps is rejected with {@link + * ForbiddenException}. This guards against silent regression of the gate. The allow path (an + * authorized principal succeeds) is covered end-to-end by {@link + * SemanticModelCatalogHandlerCrudTest} via the pass-through authorizer; the privilege-grant + * plumbing needed to build an authorized principal is not reachable through the {@link + * TestServices} surface. + */ +class SemanticModelCatalogHandlerAuthzTest extends AbstractSemanticModelCatalogHandlerTest { + + @BeforeEach + void seedModel() { + // Seed a model with a pass-through authorizer so load/update/drop reach the authz check + // (rather than failing with "not found") when exercised by the unprivileged handler. + passthroughHandler().createSemanticModel(NS, createRequest("m1", modelJson("ns1.t1"))); + } + + @Test + void createDeniedWithoutManageContent() { + assertThatThrownBy( + () -> + enforcingHandler() + .createSemanticModel(NS, createRequest("m2", modelJson("ns1.t1")))) + .isInstanceOf(ForbiddenException.class); + } + + @Test + void listDeniedWithoutManageContent() { + assertThatThrownBy(() -> enforcingHandler().listSemanticModels(NS, null, null)) + .isInstanceOf(ForbiddenException.class); + } + + @Test + void loadDeniedWithoutManageContent() { + assertThatThrownBy(() -> enforcingHandler().loadSemanticModel(identifier("m1"))) + .isInstanceOf(ForbiddenException.class); + } + + @Test + void updateDeniedWithoutManageContent() { + assertThatThrownBy( + () -> + enforcingHandler() + .updateSemanticModel( + identifier("m1"), + UpdateSemanticModelRequest.builder() + .setDocument(doc(modelJson("ns1.t1"))) + .setEntityVersion("1") + .build())) + .isInstanceOf(ForbiddenException.class); + } + + @Test + void dropDeniedWithoutManageContent() { + assertThatThrownBy(() -> enforcingHandler().dropSemanticModel(identifier("m1"))) + .isInstanceOf(ForbiddenException.class); + } + + /** Handler that actually enforces authorization for the unprivileged {@code test-principal}. */ + private SemanticModelCatalogHandler enforcingHandler() { + return handler(new PolarisAuthorizerImpl(services.realmConfig())); + } +} diff --git a/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandlerCrudTest.java b/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandlerCrudTest.java new file mode 100644 index 00000000000..c5ed2733a13 --- /dev/null +++ b/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandlerCrudTest.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.catalog.semanticmodel; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import org.apache.iceberg.exceptions.BadRequestException; +import org.apache.polaris.core.semantic.exceptions.NoSuchSemanticModelException; +import org.apache.polaris.service.TestServices; +import org.apache.polaris.service.catalog.semanticmodel.types.ListSemanticModelsResponse; +import org.apache.polaris.service.catalog.semanticmodel.types.LoadSemanticModelResponse; +import org.apache.polaris.service.catalog.semanticmodel.types.UpdateSemanticModelRequest; +import org.junit.jupiter.api.Test; + +/** + * Exercises {@link SemanticModelCatalogHandler} end-to-end against the real resolution manifest via + * {@link TestServices}. Unlike {@link SemanticModelCatalogTest} (pure Mockito), this drives the + * handler that builds a strict {@code PolarisResolutionManifest} — the only way to catch the + * passthrough-registration failures that broke create/update. Authorization is intentionally a + * pass-through mock here; privilege enforcement is covered by {@link + * SemanticModelCatalogHandlerAuthzTest}. + */ +class SemanticModelCatalogHandlerCrudTest extends AbstractSemanticModelCatalogHandlerTest { + + @Test + void createThenLoadRoundTrips() { + LoadSemanticModelResponse created = + passthroughHandler().createSemanticModel(NS, createRequest("m1", modelJson("ns1.t1"))); + + assertThat(created.getDocument().getSemanticModel()).isEqualTo(modelJson("ns1.t1")); + + LoadSemanticModelResponse loaded = passthroughHandler().loadSemanticModel(identifier("m1")); + assertThat(loaded.getDocument().getSemanticModel()).isEqualTo(modelJson("ns1.t1")); + assertThat(loaded.getEntityVersion()).isEqualTo(created.getEntityVersion()); + } + + @Test + void fullLifecycleCreateListUpdateDrop() { + passthroughHandler().createSemanticModel(NS, createRequest("m1", modelJson("ns1.t1"))); + + ListSemanticModelsResponse listed = passthroughHandler().listSemanticModels(NS, null, null); + assertThat(listed.getIdentifiers()).anySatisfy(id -> assertThat(id.getName()).isEqualTo("m1")); + + LoadSemanticModelResponse current = passthroughHandler().loadSemanticModel(identifier("m1")); + String updatedModel = modelJson("ns1.t1").replace("\"d\"", "\"d_renamed\""); + LoadSemanticModelResponse updated = + passthroughHandler() + .updateSemanticModel( + identifier("m1"), + UpdateSemanticModelRequest.builder() + .setDocument(doc(updatedModel)) + .setEntityVersion(current.getEntityVersion()) + .build()); + assertThat(updated.getDocument().getSemanticModel()).isEqualTo(updatedModel); + + passthroughHandler().dropSemanticModel(identifier("m1")); + assertThatThrownBy(() -> passthroughHandler().loadSemanticModel(identifier("m1"))) + .isInstanceOf(NoSuchSemanticModelException.class); + } + + @Test + void createRejectsUnknownSource() { + assertThatThrownBy( + () -> + passthroughHandler() + .createSemanticModel(NS, createRequest("m2", modelJson("ns1.missing")))) + .isInstanceOf(BadRequestException.class) + .hasMessageContaining("ns1.missing"); + } +} diff --git a/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogTest.java b/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogTest.java index fdfe7d88369..4601da75d8d 100644 --- a/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogTest.java +++ b/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogTest.java @@ -33,6 +33,7 @@ import org.apache.iceberg.exceptions.AlreadyExistsException; import org.apache.iceberg.exceptions.BadRequestException; import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.auth.PolarisPrincipal; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.EntityNameLookupRecord; @@ -43,13 +44,16 @@ import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; import org.apache.polaris.core.persistence.ResolvedPolarisEntity; +import org.apache.polaris.core.persistence.dao.entity.BaseResult; import org.apache.polaris.core.persistence.dao.entity.DropEntityResult; import org.apache.polaris.core.persistence.dao.entity.EntityResult; import org.apache.polaris.core.persistence.dao.entity.GenerateEntityIdResult; import org.apache.polaris.core.persistence.dao.entity.ListEntitiesResult; import org.apache.polaris.core.persistence.pagination.Page; import org.apache.polaris.core.persistence.pagination.PageToken; +import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifest; import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifestCatalogView; +import org.apache.polaris.core.persistence.resolver.ResolutionManifestFactory; import org.apache.polaris.core.persistence.resolver.ResolvedPathKey; import org.apache.polaris.core.semantic.SemanticModelEntity; import org.apache.polaris.core.semantic.exceptions.NoSuchSemanticModelException; @@ -79,6 +83,8 @@ class SemanticModelCatalogTest { private PolarisResolutionManifestCatalogView view; private PolarisMetaStoreManager metaStoreManager; + private ResolutionManifestFactory resolutionManifestFactory; + private PolarisResolutionManifest sourceManifest; private SemanticModelCatalog catalog; private PolarisEntity catalogEntity; @@ -88,6 +94,9 @@ class SemanticModelCatalogTest { void setUp() { view = mock(PolarisResolutionManifestCatalogView.class); metaStoreManager = mock(PolarisMetaStoreManager.class); + resolutionManifestFactory = mock(ResolutionManifestFactory.class); + sourceManifest = mock(PolarisResolutionManifest.class); + PolarisPrincipal principal = mock(PolarisPrincipal.class); CallContext callContext = mock(CallContext.class); PolarisCallContext polarisCallContext = mock(PolarisCallContext.class); when(callContext.getPolarisCallContext()).thenReturn(polarisCallContext); @@ -97,8 +106,13 @@ void setUp() { when(view.getResolvedCatalogEntity()).thenReturn(new CatalogEntity(catalogEntity)); when(view.getResolvedPath(ResolvedPathKey.ofNamespace(NS))) .thenReturn(path(catalogEntity, namespaceEntity)); + // Source tables are resolved through a fresh single-use manifest, not the request view. + when(resolutionManifestFactory.createResolutionManifest(any(), any())) + .thenReturn(sourceManifest); - catalog = new SemanticModelCatalog(metaStoreManager, callContext, view); + catalog = + new SemanticModelCatalog( + metaStoreManager, callContext, view, resolutionManifestFactory, principal); } private SemanticModelDocument doc(String semanticModelJson) { @@ -118,7 +132,7 @@ private void stubResolvableSource() { .setParentId(2L) .setName("store_sales") .build(); - when(view.getPassthroughResolvedPath( + when(sourceManifest.getPassthroughResolvedPath( eq(ResolvedPathKey.ofTableLike(TableIdentifier.of(NS, "store_sales"))), eq(PolarisEntitySubType.ANY_SUBTYPE))) .thenReturn(path(catalogEntity, namespaceEntity, table)); @@ -182,7 +196,11 @@ void createRejectsUnresolvedSource() { @Test void createRejectsExistingModel() { - stubExistingModel(1); + stubResolvableSource(); + when(metaStoreManager.generateNewEntityId(any())).thenReturn(new GenerateEntityIdResult(10L)); + // Duplicates are surfaced by the persistence layer, not a pre-check. + when(metaStoreManager.createEntityIfNotExists(any(), any(), any())) + .thenReturn(new EntityResult(BaseResult.ReturnStatus.ENTITY_ALREADY_EXISTS, null)); assertThatThrownBy(() -> catalog.createSemanticModel(IDENTIFIER, doc(VALID_MODEL_JSON))) .isInstanceOf(AlreadyExistsException.class); } From c6fbcc025e34e287d946bb8ebf7681b4c41f1555 Mon Sep 17 00:00:00 2001 From: Yufei Gu Date: Fri, 24 Jul 2026 16:52:56 -0700 Subject: [PATCH 10/13] Keep shouldDecodeToken local to each handler Revert the extraction of shouldDecodeToken into the base CatalogHandler so this PR stays focused on semantic models. IcebergCatalogHandler keeps its original private method, and SemanticModelCatalogHandler holds its own copy. The dedup can land in a separate, focused PR. --- .../semanticmodel/SemanticModelCatalogHandler.java | 9 +++++++++ .../service/catalog/common/CatalogHandler.java | 14 -------------- .../catalog/iceberg/IcebergCatalogHandler.java | 5 +++++ 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandler.java b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandler.java index 13fc8a8306e..df73bd8b98f 100644 --- a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandler.java +++ b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandler.java @@ -22,6 +22,8 @@ import org.apache.iceberg.catalog.Namespace; import org.apache.polaris.core.auth.PolarisAuthorizableOperation; import org.apache.polaris.core.catalog.PolarisCatalogHelpers; +import org.apache.polaris.core.config.FeatureConfiguration; +import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; import org.apache.polaris.core.persistence.pagination.PageToken; @@ -104,6 +106,13 @@ public void dropSemanticModel(SemanticModelIdentifier identifier) { semanticModelCatalog.dropSemanticModel(identifier); } + private boolean shouldDecodeToken() { + CatalogEntity catalogEntity = resolutionManifest.getResolvedCatalogEntity(); + return catalogEntity == null + ? realmConfig().getConfig(FeatureConfiguration.LIST_PAGINATION_ENABLED) + : realmConfig().getConfig(FeatureConfiguration.LIST_PAGINATION_ENABLED, catalogEntity); + } + private void authorizeBasicSemanticModelOperationOrThrow( PolarisAuthorizableOperation op, SemanticModelIdentifier identifier) { Namespace namespace = Namespace.of(identifier.getNamespace().toArray(new String[0])); diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/common/CatalogHandler.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/common/CatalogHandler.java index 82e1df81dea..2995bdba7ec 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/common/CatalogHandler.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/common/CatalogHandler.java @@ -40,11 +40,9 @@ import org.apache.polaris.core.auth.RenameAuthorizationIntent; import org.apache.polaris.core.auth.SingleTargetAuthorizationIntent; import org.apache.polaris.core.catalog.PolarisCatalogHelpers; -import org.apache.polaris.core.config.FeatureConfiguration; import org.apache.polaris.core.config.RealmConfig; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; -import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; @@ -97,18 +95,6 @@ protected PolarisResolutionManifest newResolutionManifest() { /** Initialize the catalog once authorized. Called after all `authorize...` methods. */ protected abstract void initializeCatalog(); - /** - * Whether opaque list page-tokens should be decoded, honoring the {@code LIST_PAGINATION_ENABLED} - * setting (with a per-catalog override when a catalog has been resolved). Shared by catalog - * handlers that page list results. - */ - protected boolean shouldDecodeToken() { - CatalogEntity catalogEntity = resolutionManifest.getResolvedCatalogEntity(); - return catalogEntity == null - ? realmConfig().getConfig(FeatureConfiguration.LIST_PAGINATION_ENABLED) - : realmConfig().getConfig(FeatureConfiguration.LIST_PAGINATION_ENABLED, catalogEntity); - } - protected void authorizeBasicNamespaceOperationOrThrow( PolarisAuthorizableOperation op, Namespace namespace) { authorizeBasicNamespaceOperationOrThrow(op, namespace, null, null, null); diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java index 3ccc2c31579..d60194306bf 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java @@ -20,6 +20,7 @@ import static java.util.Objects.requireNonNull; import static org.apache.polaris.core.config.FeatureConfiguration.ALLOW_FEDERATED_CATALOGS_CREDENTIAL_VENDING; +import static org.apache.polaris.core.config.FeatureConfiguration.LIST_PAGINATION_ENABLED; import static org.apache.polaris.service.catalog.AccessDelegationMode.VENDED_CREDENTIALS; import static org.apache.polaris.service.catalog.common.ExceptionUtils.alreadyExistsExceptionForTableLikeEntity; import static org.apache.polaris.service.catalog.common.ExceptionUtils.notFoundExceptionForTableLikeEntity; @@ -210,6 +211,10 @@ private CatalogEntity getResolvedCatalogEntity() { return catalogEntity; } + private boolean shouldDecodeToken() { + return realmConfig().getConfig(LIST_PAGINATION_ENABLED, getResolvedCatalogEntity()); + } + @Override protected void initializeCatalog() { CatalogEntity resolvedCatalogEntity = getResolvedCatalogEntity(); From 42c1b02df98f7a3bcbcee8e8d3ddd63c2b616653 Mon Sep 17 00:00:00 2001 From: Yufei Date: Thu, 30 Jul 2026 16:33:25 -0700 Subject: [PATCH 11/13] Address semantic model review feedback --- extensions/semantic-models/build.gradle.kts | 4 +- .../SemanticDocumentValidator.java | 13 +++--- .../semanticmodel/SemanticModelCatalog.java | 39 ++++++++++-------- .../SemanticModelCatalogAdapter.java | 2 +- .../SemanticModelCatalogHandler.java | 20 +++++++-- .../SemanticModelCatalogHandlerAuthzTest.java | 41 +++++++++++++++++++ .../core/semantic/SemanticModelEntity.java | 14 +++---- .../common/PolarisSecurableMapper.java | 14 +++++++ .../common/PolarisSecurableMapperTest.java | 12 ++++++ 9 files changed, 123 insertions(+), 36 deletions(-) diff --git a/extensions/semantic-models/build.gradle.kts b/extensions/semantic-models/build.gradle.kts index a2a7b3d1f3f..4fb24a92e6a 100644 --- a/extensions/semantic-models/build.gradle.kts +++ b/extensions/semantic-models/build.gradle.kts @@ -41,8 +41,8 @@ dependencies { // CatalogAdapter and other shared catalog service abstractions. implementation(project(":polaris-runtime-service")) - // Dependencies required by the generated OSI semantic-model REST API code (mirrors the set used - // by the polaris-api-catalog-service generated code). + // Dependencies required by the generated Ossie semantic-model REST API code (mirrors the set + // used by the polaris-api-catalog-service generated code). implementation(platform(libs.iceberg.bom)) implementation("org.apache.iceberg:iceberg-api") implementation("org.apache.iceberg:iceberg-core") diff --git a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticDocumentValidator.java b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticDocumentValidator.java index 9308212f6d0..97ecda06da0 100644 --- a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticDocumentValidator.java +++ b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticDocumentValidator.java @@ -21,17 +21,18 @@ import org.apache.polaris.service.catalog.semanticmodel.types.SemanticModelDocument; /** - * Contract for validating an OSI (Open Semantic Interchange) semantic-model document at write time. + * Contract for validating an Apache Ossie semantic-model document at write time. * - *

Only the interface ships in this phase; a concrete implementation (schema validation against - * the bundled OSI JSON Schema, size caps, etc.) lands in a follow-up. Implementations are expected - * to signal an invalid document by throwing an exception that maps to HTTP 400 (e.g. {@link - * org.apache.iceberg.exceptions.BadRequestException}) with field-level detail. + *

Only the interface ships in this phase. The concrete implementation (schema validation against + * the bundled Ossie JSON Schema, size caps, etc.) is tracked by apache/polaris#4522. Implementations are + * expected to signal an invalid document by throwing an exception that maps to HTTP 400 (e.g. + * {@link org.apache.iceberg.exceptions.BadRequestException}) with field-level detail. */ public interface SemanticDocumentValidator { /** - * Validates the given OSI document, throwing on failure. + * Validates the given Ossie document, throwing on failure. * * @param document the document to validate (its {@code version} and {@code semantic_model} body) */ diff --git a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalog.java b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalog.java index b613f7b0617..1433c84a7c0 100644 --- a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalog.java +++ b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalog.java @@ -24,6 +24,7 @@ import com.google.common.base.Splitter; import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -60,9 +61,11 @@ import org.slf4j.LoggerFactory; /** - * Core create/list/load/update/drop logic for OSI semantic models. Mirrors {@link - * org.apache.polaris.service.catalog.policy.PolicyCatalog}: the OSI document body is stored inside - * the entity {@code properties} map, writes resolve every {@code dataset.source} to a {@code + * Core create/list/load/update/drop logic for Apache Ossie semantic models. + * + *

Following the storage pattern used by {@link + * org.apache.polaris.service.catalog.policy.PolicyCatalog}, the Ossie document body is stored + * inside the entity {@code properties} map. Writes resolve every {@code dataset.source} to a {@code * TABLE_LIKE} entity in the current catalog, and updates use optimistic concurrency on the entity * version. Document schema validation is a separate concern (see {@link * SemanticDocumentValidator}). @@ -72,8 +75,8 @@ public class SemanticModelCatalog { private static final ObjectMapper MAPPER = new ObjectMapper(); /** - * Separator between the namespace path and the name in an OSI {@code dataset.source}, per the IRC - * catalog object-identifier scheme (e.g. {@code sales.store_sales}). + * Separator between the namespace path and the name in an Ossie {@code dataset.source}, per the + * IRC catalog object-identifier scheme (e.g. {@code sales.store_sales}). */ private static final char SOURCE_SEPARATOR = '.'; @@ -112,10 +115,10 @@ public LoadSemanticModelResponse createSemanticModel( } List catalogPath = resolvedParent.getRawFullPath(); - // Validate the document schema, then resolve its source tables before persisting. Duplicate - // names are caught by createEntityIfNotExists below (ENTITY_ALREADY_EXISTS), so there is no - // separate existence pre-check: the semantic-model path is not registered on this request's - // resolution manifest, and probing it there would fail. + // Parse the document, then resolve its source tables before persisting. Duplicate names are + // caught by createEntityIfNotExists below (ENTITY_ALREADY_EXISTS), so there is no separate + // existence pre-check: the semantic-model path is not registered on this request's resolution + // manifest, and probing it there would fail. JsonNode semanticModel = validateDocument(document); resolveAndValidateSources(semanticModel); @@ -245,7 +248,7 @@ public void dropSemanticModel(SemanticModelIdentifier identifier) { callContext.getPolarisCallContext(), PolarisEntity.toCoreList(catalogPath), entity, - java.util.Map.of(), + Map.of(), false); if (!result.isSuccess()) { throw new IllegalStateException( @@ -273,9 +276,9 @@ private SemanticModelEntity resolveModelOrThrow(SemanticModelIdentifier identifi } /** - * Validates the OSI document against the Ossie JSON schema and returns the parsed {@code - * semantic_model} body. Currently only checks that the required body is present and well-formed - * JSON; full schema validation is deferred to a {@link SemanticDocumentValidator} implementation. + * Parses the Ossie document and returns the {@code semantic_model} body. Currently only checks + * that the required body is present and well-formed JSON; full schema validation is deferred to a + * {@link SemanticDocumentValidator} implementation. * * @return the parsed {@code semantic_model} node * @throws BadRequestException if the body is missing/blank or not valid JSON @@ -295,10 +298,12 @@ private JsonNode validateDocument(SemanticModelDocument document) { } /** - * Resolves and validates every {@code dataset.source} in the parsed OSI document against the - * current catalog. Every dataset must define a string {@code source}; a missing or non-string - * source, or one that does not resolve to a {@code TABLE_LIKE} entity, fails with 400 and a - * JSON-Pointer to the offending dataset. Column-level checks are deferred (F5). + * Resolves and validates every {@code dataset.source} in the parsed Ossie document against the + * current catalog. Unlike engine-specific view SQL, {@code dataset.source} is a structured + * catalog identifier, so validating it here prevents every client from persisting dangling + * references. Every dataset must define a string {@code source}; a missing or non-string source, + * or one that does not resolve to a {@code TABLE_LIKE} entity, fails with 400 and a JSON-Pointer + * to the offending dataset. Column-level checks are deferred (F5). */ private void resolveAndValidateSources(JsonNode semanticModel) { if (!semanticModel.isArray()) { diff --git a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogAdapter.java b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogAdapter.java index 38fbf099d74..7f4d2d557f5 100644 --- a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogAdapter.java +++ b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogAdapter.java @@ -38,7 +38,7 @@ import org.apache.polaris.service.catalog.semanticmodel.types.UpdateSemanticModelRequest; /** - * Adapter for the OSI semantic-model API. The endpoints are gated by {@link + * Adapter for the Apache Ossie semantic-model API. The endpoints are gated by {@link * FeatureConfiguration#ENABLE_SEMANTIC_MODELS} and dispatch to {@link SemanticModelCatalogHandler} * for authorization, validation, source resolution, and persistence. */ diff --git a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandler.java b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandler.java index df73bd8b98f..0b990cc241c 100644 --- a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandler.java +++ b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandler.java @@ -20,7 +20,10 @@ import java.util.List; import org.apache.iceberg.catalog.Namespace; +import org.apache.polaris.core.auth.AuthorizationRequest; +import org.apache.polaris.core.auth.AuthorizationState; import org.apache.polaris.core.auth.PolarisAuthorizableOperation; +import org.apache.polaris.core.auth.SingleTargetAuthorizationIntent; import org.apache.polaris.core.catalog.PolarisCatalogHelpers; import org.apache.polaris.core.config.FeatureConfiguration; import org.apache.polaris.core.entity.CatalogEntity; @@ -32,6 +35,7 @@ import org.apache.polaris.core.semantic.exceptions.NoSuchSemanticModelException; import org.apache.polaris.immutables.PolarisImmutable; import org.apache.polaris.service.catalog.common.CatalogHandler; +import org.apache.polaris.service.catalog.common.PolarisSecurableMapper; import org.apache.polaris.service.catalog.semanticmodel.types.CreateSemanticModelRequest; import org.apache.polaris.service.catalog.semanticmodel.types.ListSemanticModelsResponse; import org.apache.polaris.service.catalog.semanticmodel.types.LoadSemanticModelResponse; @@ -39,8 +43,8 @@ import org.apache.polaris.service.catalog.semanticmodel.types.UpdateSemanticModelRequest; /** - * Authorizes and delegates OSI semantic-model operations to {@link SemanticModelCatalog}. Mirrors - * {@link org.apache.polaris.service.catalog.policy.PolicyCatalogHandler}. + * Authorizes and delegates Apache Ossie semantic-model operations to {@link SemanticModelCatalog}. + * Mirrors {@link org.apache.polaris.service.catalog.policy.PolicyCatalogHandler}. * *

Authorization is intentionally minimal in this phase: operations are gated by the coarse * {@code CATALOG_MANAGE_CONTENT} privilege (see {@code RbacOperationSemantics}). The dedicated @@ -122,7 +126,17 @@ private void authorizeBasicSemanticModelOperationOrThrow( PolarisCatalogHelpers.identifierToList(namespace, identifier.getName()), PolarisEntityType.SEMANTIC_MODEL, true /* optional */)); - resolutionManifest.resolveAll(); + AuthorizationState authorizationState = new AuthorizationState(resolutionManifest); + authorizer() + .resolveAuthorizationInputs( + authorizationState, + new AuthorizationRequest( + polarisPrincipal(), + List.of( + new SingleTargetAuthorizationIntent( + op, + PolarisSecurableMapper.semanticModel( + catalogName(), namespace, identifier.getName()))))); PolarisResolvedPathWrapper target = resolutionManifest.getResolvedPath( diff --git a/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandlerAuthzTest.java b/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandlerAuthzTest.java index 84a0b41d1b5..df38223b96a 100644 --- a/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandlerAuthzTest.java +++ b/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandlerAuthzTest.java @@ -18,14 +18,26 @@ */ package org.apache.polaris.service.catalog.semanticmodel; +import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import org.apache.iceberg.exceptions.ForbiddenException; +import org.apache.polaris.core.auth.AuthorizationRequest; +import org.apache.polaris.core.auth.AuthorizationState; +import org.apache.polaris.core.auth.PolarisAuthorizableOperation; +import org.apache.polaris.core.auth.PolarisAuthorizer; import org.apache.polaris.core.auth.PolarisAuthorizerImpl; +import org.apache.polaris.core.auth.SingleTargetAuthorizationIntent; import org.apache.polaris.service.TestServices; +import org.apache.polaris.service.catalog.common.PolarisSecurableMapper; import org.apache.polaris.service.catalog.semanticmodel.types.UpdateSemanticModelRequest; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; /** * Verifies that {@link SemanticModelCatalogHandler} enforces authorization: the five operations are @@ -87,6 +99,35 @@ void dropDeniedWithoutManageContent() { .isInstanceOf(ForbiddenException.class); } + @Test + void loadResolvesAuthorizationInputsForSemanticModel() { + PolarisAuthorizer authorizer = mock(PolarisAuthorizer.class); + doAnswer( + invocation -> { + AuthorizationState authorizationState = invocation.getArgument(0); + authorizationState.getResolutionManifest().resolveAll(); + return null; + }) + .when(authorizer) + .resolveAuthorizationInputs(any(), any()); + + handler(authorizer).loadSemanticModel(identifier("m1")); + + ArgumentCaptor requestCaptor = + ArgumentCaptor.forClass(AuthorizationRequest.class); + verify(authorizer).resolveAuthorizationInputs(any(), requestCaptor.capture()); + assertThat(requestCaptor.getValue().intents()) + .singleElement() + .isInstanceOfSatisfying( + SingleTargetAuthorizationIntent.class, + intent -> { + assertThat(intent.operation()) + .isEqualTo(PolarisAuthorizableOperation.LOAD_SEMANTIC_MODEL); + assertThat(intent.target()) + .isEqualTo(PolarisSecurableMapper.semanticModel(CATALOG_NAME, NS, "m1")); + }); + } + /** Handler that actually enforces authorization for the unprivileged {@code test-principal}. */ private SemanticModelCatalogHandler enforcingHandler() { return handler(new PolarisAuthorizerImpl(services.realmConfig())); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/semantic/SemanticModelEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/semantic/SemanticModelEntity.java index 6ad2201641a..6ecd690406e 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/semantic/SemanticModelEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/semantic/SemanticModelEntity.java @@ -30,19 +30,19 @@ import org.jspecify.annotations.Nullable; /** - * A Polaris entity that stores an Open Semantic Interchange (OSI) semantic-model document. + * A Polaris entity that stores an Apache Ossie semantic-model document. * *

Following the precedent set by {@link org.apache.polaris.core.policy.PolicyEntity}, the full - * document body is stored inside the entity {@code properties} map: the declared OSI spec version - * under {@link #SPEC_VERSION_KEY} and the OSI document (as a JSON string) under {@link + * document body is stored inside the entity {@code properties} map: the declared Ossie spec version + * under {@link #SPEC_VERSION_KEY} and the Ossie document (as a JSON string) under {@link * #CONTENT_KEY}. */ public class SemanticModelEntity extends PolarisEntity { - /** The declared OSI spec version of the stored document (e.g. {@code 0.1.1}). */ + /** The declared Ossie spec version of the stored document (e.g. {@code 0.1.1}). */ public static final String SPEC_VERSION_KEY = "semantic-model.spec-version"; - /** The OSI document serialized as a JSON string. */ + /** The Ossie document serialized as a JSON string. */ public static final String CONTENT_KEY = "semantic-model.content"; public SemanticModelEntity(PolarisBaseEntity sourceEntity) { @@ -62,13 +62,13 @@ public SemanticModelEntity(PolarisBaseEntity sourceEntity) { return null; } - /** The declared OSI spec version of the stored document. */ + /** The declared Ossie spec version of the stored document. */ @JsonIgnore public String getSpecVersion() { return getPropertiesAsMap().get(SPEC_VERSION_KEY); } - /** The OSI document body, serialized as a JSON string. */ + /** The Ossie document body, serialized as a JSON string. */ @JsonIgnore public String getContent() { return getPropertiesAsMap().get(CONTENT_KEY); diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/common/PolarisSecurableMapper.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/common/PolarisSecurableMapper.java index 733aa5a2256..7a2ceb39b12 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/common/PolarisSecurableMapper.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/common/PolarisSecurableMapper.java @@ -87,6 +87,20 @@ public static PolarisSecurable policy(String catalogName, PolicyIdentifier ident .build(); } + public static PolarisSecurable semanticModel( + String catalogName, Namespace namespace, String name) { + if (namespace.isEmpty()) { + throw new IllegalArgumentException("Semantic-model target cannot have an empty namespace"); + } + ImmutablePolarisSecurable.Builder builder = + ImmutablePolarisSecurable.builder() + .addPathSegment(new PathSegment(PolarisEntityType.CATALOG, catalogName)); + Arrays.stream(namespace.levels()) + .map(level -> new PathSegment(PolarisEntityType.NAMESPACE, level)) + .forEach(builder::addPathSegment); + return builder.addPathSegment(new PathSegment(PolarisEntityType.SEMANTIC_MODEL, name)).build(); + } + public static PolarisSecurable policyAttachmentTarget( String catalogName, PolicyAttachmentTarget target) { ImmutablePolarisSecurable.Builder builder = diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/common/PolarisSecurableMapperTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/common/PolarisSecurableMapperTest.java index 2a1ef675b90..c7af48d71e1 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/common/PolarisSecurableMapperTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/common/PolarisSecurableMapperTest.java @@ -49,4 +49,16 @@ void tableLikeMapsNamespaceAndTableName() { new PathSegment(PolarisEntityType.NAMESPACE, "ns2"), new PathSegment(PolarisEntityType.TABLE_LIKE, "table")); } + + @Test + void semanticModelMapsNamespaceAndModelName() { + assertThat( + PolarisSecurableMapper.semanticModel("catalog", Namespace.of("ns1", "ns2"), "model") + .getPathSegments()) + .containsExactly( + new PathSegment(PolarisEntityType.CATALOG, "catalog"), + new PathSegment(PolarisEntityType.NAMESPACE, "ns1"), + new PathSegment(PolarisEntityType.NAMESPACE, "ns2"), + new PathSegment(PolarisEntityType.SEMANTIC_MODEL, "model")); + } } From 7e0d859d920cb037fb78652e718ba81eb6677e1b Mon Sep 17 00:00:00 2001 From: Yufei Date: Thu, 30 Jul 2026 16:36:49 -0700 Subject: [PATCH 12/13] Resolve comments --- .../catalog/semanticmodel/SemanticModelCatalog.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalog.java b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalog.java index 1433c84a7c0..267b6860662 100644 --- a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalog.java +++ b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalog.java @@ -63,12 +63,10 @@ /** * Core create/list/load/update/drop logic for Apache Ossie semantic models. * - *

Following the storage pattern used by {@link - * org.apache.polaris.service.catalog.policy.PolicyCatalog}, the Ossie document body is stored - * inside the entity {@code properties} map. Writes resolve every {@code dataset.source} to a {@code - * TABLE_LIKE} entity in the current catalog, and updates use optimistic concurrency on the entity - * version. Document schema validation is a separate concern (see {@link - * SemanticDocumentValidator}). + *

The Ossie document body is stored inside the entity {@code properties} map. Writes resolve + * every {@code dataset.source} to a {@code TABLE_LIKE} entity in the current catalog, and updates + * use optimistic concurrency on the entity version. Document schema validation is a separate + * concern (see {@link SemanticDocumentValidator}). */ public class SemanticModelCatalog { private static final Logger LOGGER = LoggerFactory.getLogger(SemanticModelCatalog.class); From 347fd6088031141b1a6a346b4cefebf6e9e82201 Mon Sep 17 00:00:00 2001 From: Yufei Date: Thu, 30 Jul 2026 22:37:55 -0700 Subject: [PATCH 13/13] Resolve comments --- .../service/catalog/semanticmodel/SemanticModelCatalog.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalog.java b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalog.java index 267b6860662..b6ad08f5cc7 100644 --- a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalog.java +++ b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalog.java @@ -301,7 +301,7 @@ private JsonNode validateDocument(SemanticModelDocument document) { * catalog identifier, so validating it here prevents every client from persisting dangling * references. Every dataset must define a string {@code source}; a missing or non-string source, * or one that does not resolve to a {@code TABLE_LIKE} entity, fails with 400 and a JSON-Pointer - * to the offending dataset. Column-level checks are deferred (F5). + * to the offending dataset. */ private void resolveAndValidateSources(JsonNode semanticModel) { if (!semanticModel.isArray()) {