diff --git a/extensions/semantic-models/build.gradle.kts b/extensions/semantic-models/build.gradle.kts index ff03fd9da01..4fb24a92e6a 100644 --- a/extensions/semantic-models/build.gradle.kts +++ b/extensions/semantic-models/build.gradle.kts @@ -41,8 +41,8 @@ dependencies { // CatalogAdapter and other shared catalog service abstractions. implementation(project(":polaris-runtime-service")) - // Dependencies required by the generated OSI semantic-model REST API code (mirrors the set used - // by the polaris-api-catalog-service generated code). + // Dependencies required by the generated Ossie semantic-model REST API code (mirrors the set + // used by the polaris-api-catalog-service generated code). implementation(platform(libs.iceberg.bom)) implementation("org.apache.iceberg:iceberg-api") implementation("org.apache.iceberg:iceberg-core") @@ -73,6 +73,17 @@ dependencies { testImplementation(platform(libs.junit.bom)) testImplementation("org.junit.jupiter:junit-jupiter") testImplementation(libs.assertj.core) + + // Real-manifest handler tests bootstrap services via the runtime-service test fixtures + // (TestServices), which is the only way to reproduce the strict resolution-manifest behavior + // that the pure-Mockito unit test cannot. + testImplementation(testFixtures(project(":polaris-runtime-service"))) + testImplementation(project(":polaris-api-management-model")) + testImplementation(project(":polaris-api-management-service")) + testImplementation(project(":polaris-api-iceberg-service")) + testImplementation(project(":polaris-api-catalog-service")) + testCompileOnly(project(":polaris-immutables")) + testAnnotationProcessor(project(":polaris-immutables", configuration = "processor")) } val rootDir = rootProject.layout.projectDirectory diff --git a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticDocumentValidator.java b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticDocumentValidator.java new file mode 100644 index 00000000000..97ecda06da0 --- /dev/null +++ b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticDocumentValidator.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.catalog.semanticmodel; + +import org.apache.polaris.service.catalog.semanticmodel.types.SemanticModelDocument; + +/** + * Contract for validating an Apache Ossie semantic-model document at write time. + * + *

Only the interface ships in this phase. The concrete implementation (schema validation against + * the bundled Ossie JSON Schema, size caps, etc.) is tracked by apache/polaris#4522. Implementations are + * expected to signal an invalid document by throwing an exception that maps to HTTP 400 (e.g. + * {@link org.apache.iceberg.exceptions.BadRequestException}) with field-level detail. + */ +public interface SemanticDocumentValidator { + + /** + * Validates the given Ossie document, throwing on failure. + * + * @param document the document to validate (its {@code version} and {@code semantic_model} body) + */ + void validate(SemanticModelDocument document); +} diff --git a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalog.java b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalog.java new file mode 100644 index 00000000000..b6ad08f5cc7 --- /dev/null +++ b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalog.java @@ -0,0 +1,381 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.catalog.semanticmodel; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Splitter; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.AlreadyExistsException; +import org.apache.iceberg.exceptions.BadRequestException; +import org.apache.polaris.core.auth.PolarisPrincipal; +import org.apache.polaris.core.catalog.PolarisCatalogHelpers; +import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.entity.CatalogEntity; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisEntity; +import org.apache.polaris.core.entity.PolarisEntitySubType; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.persistence.PolarisMetaStoreManager; +import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; +import org.apache.polaris.core.persistence.dao.entity.EntityResult; +import org.apache.polaris.core.persistence.dao.entity.ListEntitiesResult; +import org.apache.polaris.core.persistence.pagination.PageToken; +import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifest; +import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifestCatalogView; +import org.apache.polaris.core.persistence.resolver.ResolutionManifestFactory; +import org.apache.polaris.core.persistence.resolver.ResolvedPathKey; +import org.apache.polaris.core.persistence.resolver.ResolverPath; +import org.apache.polaris.core.semantic.SemanticModelEntity; +import org.apache.polaris.core.semantic.exceptions.NoSuchSemanticModelException; +import org.apache.polaris.core.semantic.exceptions.SemanticModelVersionMismatchException; +import org.apache.polaris.service.catalog.semanticmodel.types.ListSemanticModelsResponse; +import org.apache.polaris.service.catalog.semanticmodel.types.LoadSemanticModelResponse; +import org.apache.polaris.service.catalog.semanticmodel.types.SemanticModelDocument; +import org.apache.polaris.service.catalog.semanticmodel.types.SemanticModelIdentifier; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Core create/list/load/update/drop logic for Apache Ossie semantic models. + * + *

The Ossie document body is stored inside the entity {@code properties} map. Writes resolve + * every {@code dataset.source} to a {@code TABLE_LIKE} entity in the current catalog, and updates + * use optimistic concurrency on the entity version. Document schema validation is a separate + * concern (see {@link SemanticDocumentValidator}). + */ +public class SemanticModelCatalog { + private static final Logger LOGGER = LoggerFactory.getLogger(SemanticModelCatalog.class); + private static final ObjectMapper MAPPER = new ObjectMapper(); + + /** + * Separator between the namespace path and the name in an Ossie {@code dataset.source}, per the + * IRC catalog object-identifier scheme (e.g. {@code sales.store_sales}). + */ + private static final char SOURCE_SEPARATOR = '.'; + + private final CallContext callContext; + private final PolarisResolutionManifestCatalogView resolvedEntityView; + private final CatalogEntity catalogEntity; + private final long catalogId; + private final PolarisMetaStoreManager metaStoreManager; + private final ResolutionManifestFactory resolutionManifestFactory; + private final PolarisPrincipal principal; + + public SemanticModelCatalog( + PolarisMetaStoreManager metaStoreManager, + CallContext callContext, + PolarisResolutionManifestCatalogView resolvedEntityView, + ResolutionManifestFactory resolutionManifestFactory, + PolarisPrincipal principal) { + this.callContext = callContext; + this.resolvedEntityView = resolvedEntityView; + this.catalogEntity = resolvedEntityView.getResolvedCatalogEntity(); + this.catalogId = catalogEntity.getId(); + this.metaStoreManager = metaStoreManager; + this.resolutionManifestFactory = resolutionManifestFactory; + this.principal = principal; + } + + public LoadSemanticModelResponse createSemanticModel( + SemanticModelIdentifier identifier, SemanticModelDocument document) { + Namespace namespace = toNamespace(identifier); + PolarisResolvedPathWrapper resolvedParent = + resolvedEntityView.getResolvedPath(ResolvedPathKey.ofNamespace(namespace)); + if (resolvedParent == null) { + // Illegal state because the namespace should've already been in the static resolution set. + throw new IllegalStateException( + String.format("Failed to fetch resolved parent for semantic model '%s'", identifier)); + } + List catalogPath = resolvedParent.getRawFullPath(); + + // Parse the document, then resolve its source tables before persisting. Duplicate names are + // caught by createEntityIfNotExists below (ENTITY_ALREADY_EXISTS), so there is no separate + // existence pre-check: the semantic-model path is not registered on this request's resolution + // manifest, and probing it there would fail. + JsonNode semanticModel = validateDocument(document); + resolveAndValidateSources(semanticModel); + + SemanticModelEntity entity = + new SemanticModelEntity.Builder(namespace, identifier.getName()) + .setCatalogId(catalogId) + .setParentId(resolvedParent.getRawLeafEntity().getId()) + .setSpecVersion(document.getVersion()) + .setContent(document.getSemanticModel()) + .setId( + metaStoreManager.generateNewEntityId(callContext.getPolarisCallContext()).getId()) + .setCreateTimestamp(System.currentTimeMillis()) + .build(); + + EntityResult res = + metaStoreManager.createEntityIfNotExists( + callContext.getPolarisCallContext(), PolarisEntity.toCoreList(catalogPath), entity); + if (!res.isSuccess()) { + switch (res.getReturnStatus()) { + case ENTITY_ALREADY_EXISTS -> + throw new AlreadyExistsException( + "Semantic model already exists: %s", identifier.getName()); + default -> + throw new IllegalStateException( + String.format( + "Unknown error status for identifier %s: %s with extraInfo: %s", + identifier, res.getReturnStatus(), res.getExtraInformation())); + } + } + + SemanticModelEntity result = SemanticModelEntity.of(res.getEntity()); + LOGGER.debug("Created semantic model entity {} with identifier {}", result, identifier); + return toLoadResponse(result); + } + + public ListSemanticModelsResponse listSemanticModels(Namespace namespace, PageToken pageToken) { + PolarisResolvedPathWrapper resolvedEntities = + resolvedEntityView.getResolvedPath(ResolvedPathKey.ofNamespace(namespace)); + if (resolvedEntities == null) { + throw new IllegalStateException( + String.format("Failed to fetch resolved namespace '%s'", namespace)); + } + List catalogPath = resolvedEntities.getRawFullPath(); + + ListEntitiesResult result = + metaStoreManager.listEntities( + callContext.getPolarisCallContext(), + PolarisEntity.toCoreList(catalogPath), + PolarisEntityType.SEMANTIC_MODEL, + PolarisEntitySubType.NULL_SUBTYPE, + pageToken); + if (!result.isSuccess()) { + throw new IllegalStateException("Failed to list semantic models in namespace: " + namespace); + } + + // LinkedHashSet keeps the page order stable in the serialized response. + Set identifiers = + result.getEntities().stream() + .map( + record -> + SemanticModelIdentifier.builder() + .setNamespace(List.of(namespace.levels())) + .setName(record.getName()) + .build()) + .collect(Collectors.toCollection(LinkedHashSet::new)); + return ListSemanticModelsResponse.builder() + .setIdentifiers(identifiers) + .setNextPageToken(result.getPage().encodedResponseToken()) + .build(); + } + + public LoadSemanticModelResponse loadSemanticModel(SemanticModelIdentifier identifier) { + return toLoadResponse(resolveModelOrThrow(identifier)); + } + + public LoadSemanticModelResponse updateSemanticModel( + SemanticModelIdentifier identifier, SemanticModelDocument document, String expectedVersion) { + PolarisResolvedPathWrapper resolvedPath = resolveModelPathOrThrow(identifier); + SemanticModelEntity current = SemanticModelEntity.of(resolvedPath.getRawLeafEntity()); + + String currentVersion = Integer.toString(current.getEntityVersion()); + if (!currentVersion.equals(expectedVersion)) { + throw new SemanticModelVersionMismatchException( + String.format( + "Semantic model version mismatch. Given version is %s, current version is %s", + expectedVersion, currentVersion)); + } + + JsonNode semanticModel = validateDocument(document); + resolveAndValidateSources(semanticModel); + + SemanticModelEntity newEntity = + new SemanticModelEntity.Builder(current) + .setSpecVersion(document.getVersion()) + .setContent(document.getSemanticModel()) + .build(); + + List catalogPath = resolvedPath.getRawParentPath(); + SemanticModelEntity updated = + Optional.ofNullable( + metaStoreManager + .updateEntityPropertiesIfNotChanged( + callContext.getPolarisCallContext(), + PolarisEntity.toCoreList(catalogPath), + newEntity) + .getEntity()) + .map(SemanticModelEntity::of) + .orElse(null); + if (updated == null) { + // Lost the optimistic-concurrency race with a concurrent writer. + throw new SemanticModelVersionMismatchException( + String.format( + "Semantic model %s was modified concurrently; retry after reloading", + identifier.getName())); + } + + return toLoadResponse(updated); + } + + public void dropSemanticModel(SemanticModelIdentifier identifier) { + PolarisResolvedPathWrapper resolvedPath = resolveModelPathOrThrow(identifier); + List catalogPath = resolvedPath.getRawParentPath(); + PolarisBaseEntity entity = resolvedPath.getRawLeafEntity(); + + var result = + metaStoreManager.dropEntityIfExists( + callContext.getPolarisCallContext(), + PolarisEntity.toCoreList(catalogPath), + entity, + Map.of(), + false); + if (!result.isSuccess()) { + throw new IllegalStateException( + String.format( + "Failed to drop semantic model %s error status: %s with extraInfo: %s", + identifier, result.getReturnStatus(), result.getExtraInformation())); + } + } + + private PolarisResolvedPathWrapper resolveModelPathOrThrow(SemanticModelIdentifier identifier) { + Namespace namespace = toNamespace(identifier); + PolarisResolvedPathWrapper resolved = + resolvedEntityView.getPassthroughResolvedPath( + ResolvedPathKey.ofSemanticModel(namespace, identifier.getName()), + PolarisEntitySubType.NULL_SUBTYPE); + if (resolved == null || resolved.getRawLeafEntity() == null) { + throw new NoSuchSemanticModelException( + String.format("Semantic model does not exist: %s", identifier.getName())); + } + return resolved; + } + + private SemanticModelEntity resolveModelOrThrow(SemanticModelIdentifier identifier) { + return SemanticModelEntity.of(resolveModelPathOrThrow(identifier).getRawLeafEntity()); + } + + /** + * Parses the Ossie document and returns the {@code semantic_model} body. Currently only checks + * that the required body is present and well-formed JSON; full schema validation is deferred to a + * {@link SemanticDocumentValidator} implementation. + * + * @return the parsed {@code semantic_model} node + * @throws BadRequestException if the body is missing/blank or not valid JSON + */ + private JsonNode validateDocument(SemanticModelDocument document) { + String body = document.getSemanticModel(); + if (body == null || body.isBlank()) { + throw new BadRequestException("Semantic model document must not be empty"); + } + try { + // TODO: delegate full Ossie JSON-schema validation to SemanticDocumentValidator. + return MAPPER.readTree(body); + } catch (JsonProcessingException e) { + throw new BadRequestException( + "Field 'semantic_model' is not valid JSON: %s", e.getOriginalMessage()); + } + } + + /** + * Resolves and validates every {@code dataset.source} in the parsed Ossie document against the + * current catalog. Unlike engine-specific view SQL, {@code dataset.source} is a structured + * catalog identifier, so validating it here prevents every client from persisting dangling + * references. Every dataset must define a string {@code source}; a missing or non-string source, + * or one that does not resolve to a {@code TABLE_LIKE} entity, fails with 400 and a JSON-Pointer + * to the offending dataset. + */ + private void resolveAndValidateSources(JsonNode semanticModel) { + if (!semanticModel.isArray()) { + return; + } + + for (int modelIdx = 0; modelIdx < semanticModel.size(); modelIdx++) { + JsonNode datasets = semanticModel.get(modelIdx).get("datasets"); + if (datasets == null || !datasets.isArray()) { + continue; + } + for (int datasetIdx = 0; datasetIdx < datasets.size(); datasetIdx++) { + String pointer = + String.format("/semantic_model/%d/datasets/%d/source", modelIdx, datasetIdx); + JsonNode source = datasets.get(datasetIdx).get("source"); + if (source == null || !source.isTextual()) { + throw new BadRequestException( + "Semantic model dataset at %s must define a string 'source'", pointer); + } + resolveSourceOrThrow(source.asText(), pointer); + } + } + } + + private void resolveSourceOrThrow(String source, String pointer) { + TableIdentifier tableIdentifier = parseSource(source, pointer); + // Sources are parsed from the opaque document, so they cannot be pre-registered on this + // request's resolution manifest. Resolve each one with a fresh single-use manifest that + // registers the table path as an optional passthrough. + PolarisResolutionManifest manifest = + resolutionManifestFactory.createResolutionManifest(principal, catalogEntity.getName()); + manifest.addPassthroughPath( + new ResolverPath( + PolarisCatalogHelpers.tableIdentifierToList(tableIdentifier), + PolarisEntityType.TABLE_LIKE, + true /* optional */)); + PolarisResolvedPathWrapper resolved = + manifest.getPassthroughResolvedPath( + ResolvedPathKey.ofTableLike(tableIdentifier), PolarisEntitySubType.ANY_SUBTYPE); + if (resolved == null + || resolved.getRawLeafEntity() == null + || resolved.getRawLeafEntity().getType() != PolarisEntityType.TABLE_LIKE) { + throw new BadRequestException( + "Semantic model source '%s' at %s does not resolve to a table or view in catalog '%s'", + source, pointer, catalogEntity.getName()); + } + } + + private TableIdentifier parseSource(String source, String pointer) { + List parts = Splitter.on(SOURCE_SEPARATOR).splitToList(source); + if (parts.size() < 2 || parts.stream().anyMatch(String::isEmpty)) { + throw new BadRequestException( + "Semantic model source '%s' at %s must be a namespace-qualified identifier " + + "'.'", + source, pointer); + } + String name = parts.get(parts.size() - 1); + String[] levels = parts.subList(0, parts.size() - 1).toArray(new String[0]); + return TableIdentifier.of(Namespace.of(levels), name); + } + + private static Namespace toNamespace(SemanticModelIdentifier identifier) { + return Namespace.of(identifier.getNamespace().toArray(new String[0])); + } + + private static LoadSemanticModelResponse toLoadResponse(SemanticModelEntity entity) { + SemanticModelDocument document = + SemanticModelDocument.builder() + .setVersion(entity.getSpecVersion()) + .setSemanticModel(entity.getContent()) + .build(); + return LoadSemanticModelResponse.builder() + .setDocument(document) + .setEntityVersion(Integer.toString(entity.getEntityVersion())) + .build(); + } +} diff --git a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogAdapter.java b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogAdapter.java index a569a120370..7f4d2d557f5 100644 --- a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogAdapter.java +++ b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogAdapter.java @@ -22,39 +22,57 @@ import jakarta.inject.Inject; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.SecurityContext; +import java.util.List; +import org.apache.iceberg.catalog.Namespace; +import org.apache.polaris.core.auth.PolarisPrincipal; import org.apache.polaris.core.config.FeatureConfiguration; import org.apache.polaris.core.config.RealmConfig; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.rest.NamespaceUtils; +import org.apache.polaris.service.catalog.CatalogPrefixParser; import org.apache.polaris.service.catalog.common.CatalogAdapter; import org.apache.polaris.service.catalog.semanticmodel.api.PolarisCatalogSemanticModelApiService; import org.apache.polaris.service.catalog.semanticmodel.types.CreateSemanticModelRequest; +import org.apache.polaris.service.catalog.semanticmodel.types.SemanticModelIdentifier; import org.apache.polaris.service.catalog.semanticmodel.types.UpdateSemanticModelRequest; /** - * Stub adapter for the OSI semantic-model API. The endpoints are wired and gated by {@link - * FeatureConfiguration#ENABLE_SEMANTIC_MODELS}, but every operation returns {@code 501 Not - * Implemented}. Persistence, validation, authorization, and source-link resolution land in - * subsequent phases. + * Adapter for the Apache Ossie semantic-model API. The endpoints are gated by {@link + * FeatureConfiguration#ENABLE_SEMANTIC_MODELS} and dispatch to {@link SemanticModelCatalogHandler} + * for authorization, validation, source resolution, and persistence. */ @RequestScoped public class SemanticModelCatalogAdapter implements PolarisCatalogSemanticModelApiService, CatalogAdapter { private final RealmConfig realmConfig; + private final CatalogPrefixParser prefixParser; + private final SemanticModelCatalogHandlerFactory handlerFactory; @Inject - public SemanticModelCatalogAdapter(CallContext callContext) { + public SemanticModelCatalogAdapter( + CallContext callContext, + CatalogPrefixParser prefixParser, + SemanticModelCatalogHandlerFactory handlerFactory) { this.realmConfig = callContext.getRealmConfig(); + this.prefixParser = prefixParser; + this.handlerFactory = handlerFactory; } - private void ensureEnabled() { + private SemanticModelCatalogHandler newHandler(SecurityContext securityContext, String prefix) { FeatureConfiguration.enforceFeatureEnabledOrThrow( realmConfig, FeatureConfiguration.ENABLE_SEMANTIC_MODELS); + PolarisPrincipal principal = validatePrincipal(securityContext); + String catalogName = prefixParser.prefixToCatalogName(prefix); + return handlerFactory.createHandler(catalogName, principal); } - private static Response notImplemented() { - return Response.status(Response.Status.NOT_IMPLEMENTED).build(); + private static SemanticModelIdentifier identifier(Namespace namespace, String name) { + return SemanticModelIdentifier.builder() + .setNamespace(List.of(namespace.levels())) + .setName(name) + .build(); } @Override @@ -64,10 +82,10 @@ public Response createSemanticModel( CreateSemanticModelRequest createSemanticModelRequest, RealmContext realmContext, SecurityContext securityContext) { - ensureEnabled(); - // TODO: authorize the principal, validate the OSI document against the bundled OSI JSON Schema, - // then persist a new semantic-model entity and return the stored document. - return notImplemented(); + Namespace ns = + NamespaceUtils.splitNamespace(namespace, NamespaceUtils.DEFAULT_NAMESPACE_SEPARATOR); + SemanticModelCatalogHandler handler = newHandler(securityContext, prefix); + return Response.ok(handler.createSemanticModel(ns, createSemanticModelRequest)).build(); } @Override @@ -78,10 +96,10 @@ public Response listSemanticModels( Integer pageSize, RealmContext realmContext, SecurityContext securityContext) { - ensureEnabled(); - // TODO: authorize the principal, then page through the namespace's semantic-model entities and - // return their identifiers. - return notImplemented(); + Namespace ns = + NamespaceUtils.splitNamespace(namespace, NamespaceUtils.DEFAULT_NAMESPACE_SEPARATOR); + SemanticModelCatalogHandler handler = newHandler(securityContext, prefix); + return Response.ok(handler.listSemanticModels(ns, pageToken, pageSize)).build(); } @Override @@ -91,9 +109,10 @@ public Response loadSemanticModel( String semanticModelName, RealmContext realmContext, SecurityContext securityContext) { - ensureEnabled(); - // TODO: authorize the principal, then read and return the stored OSI document. - return notImplemented(); + Namespace ns = + NamespaceUtils.splitNamespace(namespace, NamespaceUtils.DEFAULT_NAMESPACE_SEPARATOR); + SemanticModelCatalogHandler handler = newHandler(securityContext, prefix); + return Response.ok(handler.loadSemanticModel(identifier(ns, semanticModelName))).build(); } @Override @@ -104,10 +123,13 @@ public Response updateSemanticModel( UpdateSemanticModelRequest updateSemanticModelRequest, RealmContext realmContext, SecurityContext securityContext) { - ensureEnabled(); - // TODO: authorize the principal, validate the OSI document, then replace the stored document - // and return it. - return notImplemented(); + Namespace ns = + NamespaceUtils.splitNamespace(namespace, NamespaceUtils.DEFAULT_NAMESPACE_SEPARATOR); + SemanticModelCatalogHandler handler = newHandler(securityContext, prefix); + return Response.ok( + handler.updateSemanticModel( + identifier(ns, semanticModelName), updateSemanticModelRequest)) + .build(); } @Override @@ -117,8 +139,10 @@ public Response dropSemanticModel( String semanticModelName, RealmContext realmContext, SecurityContext securityContext) { - ensureEnabled(); - // TODO: authorize the principal, then delete the semantic-model entity. - return notImplemented(); + Namespace ns = + NamespaceUtils.splitNamespace(namespace, NamespaceUtils.DEFAULT_NAMESPACE_SEPARATOR); + SemanticModelCatalogHandler handler = newHandler(securityContext, prefix); + handler.dropSemanticModel(identifier(ns, semanticModelName)); + return Response.noContent().build(); } } diff --git a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandler.java b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandler.java new file mode 100644 index 00000000000..0b990cc241c --- /dev/null +++ b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandler.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.catalog.semanticmodel; + +import java.util.List; +import org.apache.iceberg.catalog.Namespace; +import org.apache.polaris.core.auth.AuthorizationRequest; +import org.apache.polaris.core.auth.AuthorizationState; +import org.apache.polaris.core.auth.PolarisAuthorizableOperation; +import org.apache.polaris.core.auth.SingleTargetAuthorizationIntent; +import org.apache.polaris.core.catalog.PolarisCatalogHelpers; +import org.apache.polaris.core.config.FeatureConfiguration; +import org.apache.polaris.core.entity.CatalogEntity; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; +import org.apache.polaris.core.persistence.pagination.PageToken; +import org.apache.polaris.core.persistence.resolver.ResolvedPathKey; +import org.apache.polaris.core.persistence.resolver.ResolverPath; +import org.apache.polaris.core.semantic.exceptions.NoSuchSemanticModelException; +import org.apache.polaris.immutables.PolarisImmutable; +import org.apache.polaris.service.catalog.common.CatalogHandler; +import org.apache.polaris.service.catalog.common.PolarisSecurableMapper; +import org.apache.polaris.service.catalog.semanticmodel.types.CreateSemanticModelRequest; +import org.apache.polaris.service.catalog.semanticmodel.types.ListSemanticModelsResponse; +import org.apache.polaris.service.catalog.semanticmodel.types.LoadSemanticModelResponse; +import org.apache.polaris.service.catalog.semanticmodel.types.SemanticModelIdentifier; +import org.apache.polaris.service.catalog.semanticmodel.types.UpdateSemanticModelRequest; + +/** + * Authorizes and delegates Apache Ossie semantic-model operations to {@link SemanticModelCatalog}. + * Mirrors {@link org.apache.polaris.service.catalog.policy.PolicyCatalogHandler}. + * + *

Authorization is intentionally minimal in this phase: operations are gated by the coarse + * {@code CATALOG_MANAGE_CONTENT} privilege (see {@code RbacOperationSemantics}). The dedicated + * {@code SEMANTIC_MODEL_*} privilege matrix, the write-time source-access check, and the + * independent/propagated read-time enforcement modes land in the authorization phase. + */ +@PolarisImmutable +@SuppressWarnings("immutables:incompat") +public abstract class SemanticModelCatalogHandler extends CatalogHandler { + + private SemanticModelCatalog semanticModelCatalog; + + @Override + protected void initializeCatalog() { + this.semanticModelCatalog = + new SemanticModelCatalog( + metaStoreManager(), + callContext(), + this.resolutionManifest, + resolutionManifestFactory(), + polarisPrincipal()); + } + + public LoadSemanticModelResponse createSemanticModel( + Namespace namespace, CreateSemanticModelRequest request) { + PolarisAuthorizableOperation op = PolarisAuthorizableOperation.CREATE_SEMANTIC_MODEL; + authorizeBasicNamespaceOperationOrThrow(op, namespace); + + SemanticModelIdentifier identifier = + SemanticModelIdentifier.builder() + .setNamespace(List.of(namespace.levels())) + .setName(request.getName()) + .build(); + return semanticModelCatalog.createSemanticModel(identifier, request.getDocument()); + } + + public ListSemanticModelsResponse listSemanticModels( + Namespace namespace, String pageToken, Integer pageSize) { + PolarisAuthorizableOperation op = PolarisAuthorizableOperation.LIST_SEMANTIC_MODEL; + authorizeBasicNamespaceOperationOrThrow(op, namespace); + + PageToken pageRequest = PageToken.build(pageToken, pageSize, this::shouldDecodeToken); + return semanticModelCatalog.listSemanticModels(namespace, pageRequest); + } + + public LoadSemanticModelResponse loadSemanticModel(SemanticModelIdentifier identifier) { + PolarisAuthorizableOperation op = PolarisAuthorizableOperation.LOAD_SEMANTIC_MODEL; + authorizeBasicSemanticModelOperationOrThrow(op, identifier); + return semanticModelCatalog.loadSemanticModel(identifier); + } + + public LoadSemanticModelResponse updateSemanticModel( + SemanticModelIdentifier identifier, UpdateSemanticModelRequest request) { + PolarisAuthorizableOperation op = PolarisAuthorizableOperation.UPDATE_SEMANTIC_MODEL; + authorizeBasicSemanticModelOperationOrThrow(op, identifier); + return semanticModelCatalog.updateSemanticModel( + identifier, request.getDocument(), request.getEntityVersion()); + } + + public void dropSemanticModel(SemanticModelIdentifier identifier) { + PolarisAuthorizableOperation op = PolarisAuthorizableOperation.DROP_SEMANTIC_MODEL; + authorizeBasicSemanticModelOperationOrThrow(op, identifier); + semanticModelCatalog.dropSemanticModel(identifier); + } + + private boolean shouldDecodeToken() { + CatalogEntity catalogEntity = resolutionManifest.getResolvedCatalogEntity(); + return catalogEntity == null + ? realmConfig().getConfig(FeatureConfiguration.LIST_PAGINATION_ENABLED) + : realmConfig().getConfig(FeatureConfiguration.LIST_PAGINATION_ENABLED, catalogEntity); + } + + private void authorizeBasicSemanticModelOperationOrThrow( + PolarisAuthorizableOperation op, SemanticModelIdentifier identifier) { + Namespace namespace = Namespace.of(identifier.getNamespace().toArray(new String[0])); + resolutionManifest = newResolutionManifest(); + resolutionManifest.addPassthroughPath( + new ResolverPath( + PolarisCatalogHelpers.identifierToList(namespace, identifier.getName()), + PolarisEntityType.SEMANTIC_MODEL, + true /* optional */)); + AuthorizationState authorizationState = new AuthorizationState(resolutionManifest); + authorizer() + .resolveAuthorizationInputs( + authorizationState, + new AuthorizationRequest( + polarisPrincipal(), + List.of( + new SingleTargetAuthorizationIntent( + op, + PolarisSecurableMapper.semanticModel( + catalogName(), namespace, identifier.getName()))))); + + PolarisResolvedPathWrapper target = + resolutionManifest.getResolvedPath( + ResolvedPathKey.ofSemanticModel(namespace, identifier.getName()), true); + if (target == null) { + throw new NoSuchSemanticModelException( + String.format("Semantic model does not exist: %s", identifier.getName())); + } + + authorizer() + .authorizeOrThrow( + polarisPrincipal(), + resolutionManifest.getAllActivatedCatalogRoleAndPrincipalRoles(), + op, + target, + null /* secondary */); + + initializeCatalog(); + } +} diff --git a/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandlerFactory.java b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandlerFactory.java new file mode 100644 index 00000000000..16a9b9e5517 --- /dev/null +++ b/extensions/semantic-models/src/main/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandlerFactory.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.catalog.semanticmodel; + +import jakarta.enterprise.context.RequestScoped; +import jakarta.inject.Inject; +import org.apache.polaris.core.auth.PolarisAuthorizer; +import org.apache.polaris.core.auth.PolarisPrincipal; +import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.persistence.PolarisMetaStoreManager; +import org.apache.polaris.core.persistence.resolver.ResolutionManifestFactory; + +@RequestScoped +public class SemanticModelCatalogHandlerFactory { + + @Inject CallContext callContext; + @Inject ResolutionManifestFactory resolutionManifestFactory; + @Inject PolarisMetaStoreManager metaStoreManager; + @Inject PolarisAuthorizer authorizer; + + public SemanticModelCatalogHandler createHandler(String catalogName, PolarisPrincipal principal) { + return ImmutableSemanticModelCatalogHandler.builder() + .catalogName(catalogName) + .polarisPrincipal(principal) + .callContext(callContext) + .resolutionManifestFactory(resolutionManifestFactory) + .metaStoreManager(metaStoreManager) + .authorizer(authorizer) + .build(); + } +} diff --git a/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/AbstractSemanticModelCatalogHandlerTest.java b/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/AbstractSemanticModelCatalogHandlerTest.java new file mode 100644 index 00000000000..111bffe3e0c --- /dev/null +++ b/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/AbstractSemanticModelCatalogHandlerTest.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.catalog.semanticmodel; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; + +import jakarta.ws.rs.core.Response; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.rest.requests.CreateNamespaceRequest; +import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.CatalogProperties; +import org.apache.polaris.core.admin.model.CreateCatalogRequest; +import org.apache.polaris.core.admin.model.FileStorageConfigInfo; +import org.apache.polaris.core.admin.model.StorageConfigInfo; +import org.apache.polaris.core.auth.AuthorizationState; +import org.apache.polaris.core.auth.PolarisAuthorizer; +import org.apache.polaris.service.TestServices; +import org.apache.polaris.service.catalog.semanticmodel.types.CreateSemanticModelRequest; +import org.apache.polaris.service.catalog.semanticmodel.types.SemanticModelDocument; +import org.apache.polaris.service.catalog.semanticmodel.types.SemanticModelIdentifier; +import org.apache.polaris.service.types.CreateGenericTableRequest; +import org.junit.jupiter.api.BeforeEach; +import org.mockito.Mockito; + +/** + * Shared bootstrap for {@link SemanticModelCatalogHandler} tests that run against the real + * resolution manifest via {@link TestServices}: it stands up a catalog, a namespace, and a source + * table ({@code ns1.t1}) so semantic-model {@code dataset.source} references resolve, and builds + * handlers with a caller-supplied authorizer. + */ +abstract class AbstractSemanticModelCatalogHandlerTest { + + protected static final String CATALOG_NAME = "sm-catalog"; + protected static final String CATALOG_BASE_LOCATION = "file:///tmp/polaris-sm-test"; + protected static final Namespace NS = Namespace.of("ns1"); + protected static final String SOURCE_TABLE = "t1"; + private static final UUID IDEMPOTENCY_KEY = new UUID(116617318654508422L, -7820829973016961092L); + + protected TestServices services; + + @BeforeEach + void bootstrap() { + services = + TestServices.builder() + .config( + Map.of( + "ALLOW_INSECURE_STORAGE_TYPES", + "true", + "SUPPORTED_CATALOG_STORAGE_TYPES", + List.of("FILE"))) + .build(); + createCatalogNamespaceAndTable(); + } + + /** Builds a handler wired to the given authorizer against the bootstrapped services. */ + protected SemanticModelCatalogHandler handler(PolarisAuthorizer authorizer) { + return ImmutableSemanticModelCatalogHandler.builder() + .catalogName(CATALOG_NAME) + .polarisPrincipal(services.principal()) + .callContext(services.newCallContext()) + .resolutionManifestFactory(services.resolutionManifestFactory()) + .metaStoreManager(services.metaStoreManager()) + .authorizer(authorizer) + .build(); + } + + /** + * Handler that skips authorization; use for fixtures and non-authz assertions. The authorizer is + * a mock stubbed to resolve the manifest (mirroring {@link TestServices}), since the base handler + * now drives resolution through {@code resolveAuthorizationInputs} rather than resolving inline. + */ + protected SemanticModelCatalogHandler passthroughHandler() { + PolarisAuthorizer authorizer = Mockito.mock(PolarisAuthorizer.class); + Mockito.doAnswer( + invocation -> { + AuthorizationState authzState = invocation.getArgument(0); + authzState.getResolutionManifest().resolveAll(); + return null; + }) + .when(authorizer) + .resolveAuthorizationInputs(any(), any()); + return handler(authorizer); + } + + protected static SemanticModelIdentifier identifier(String name) { + return SemanticModelIdentifier.builder() + .setNamespace(List.of(NS.levels())) + .setName(name) + .build(); + } + + protected static CreateSemanticModelRequest createRequest(String name, String model) { + return CreateSemanticModelRequest.builder().setName(name).setDocument(doc(model)).build(); + } + + protected static SemanticModelDocument doc(String model) { + return SemanticModelDocument.builder().setVersion("0.1.1").setSemanticModel(model).build(); + } + + protected static String modelJson(String source) { + return "[{\"name\":\"m\",\"datasets\":[{\"name\":\"d\",\"source\":\"" + source + "\"}]}]"; + } + + private void createCatalogNamespaceAndTable() { + Catalog catalogObject = + new Catalog( + Catalog.TypeEnum.INTERNAL, + CATALOG_NAME, + CatalogProperties.builder() + .setDefaultBaseLocation(CATALOG_BASE_LOCATION + "/" + CATALOG_NAME) + .build(), + 1725487592064L, + 1725487592064L, + 1, + FileStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.FILE) + .build()); + try (Response response = + services + .catalogsApi() + .createCatalog( + new CreateCatalogRequest(catalogObject), + services.realmContext(), + services.securityContext())) { + assertThat(response.getStatus()).isEqualTo(Response.Status.CREATED.getStatusCode()); + } + + try (Response response = + services + .restApi() + .createNamespace( + CATALOG_NAME, + CreateNamespaceRequest.builder().withNamespace(NS).build(), + IDEMPOTENCY_KEY, + services.realmContext(), + services.securityContext())) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + try (Response response = + services + .genericTableApi() + .createGenericTable( + CATALOG_NAME, + NS.toString(), + CreateGenericTableRequest.builder(SOURCE_TABLE, "iceberg") + .setBaseLocation(CATALOG_BASE_LOCATION + "/" + CATALOG_NAME + "/ns1/t1") + .build(), + null, + services.realmContext(), + services.securityContext())) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + } +} diff --git a/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandlerAuthzTest.java b/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandlerAuthzTest.java new file mode 100644 index 00000000000..df38223b96a --- /dev/null +++ b/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandlerAuthzTest.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.catalog.semanticmodel; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +import org.apache.iceberg.exceptions.ForbiddenException; +import org.apache.polaris.core.auth.AuthorizationRequest; +import org.apache.polaris.core.auth.AuthorizationState; +import org.apache.polaris.core.auth.PolarisAuthorizableOperation; +import org.apache.polaris.core.auth.PolarisAuthorizer; +import org.apache.polaris.core.auth.PolarisAuthorizerImpl; +import org.apache.polaris.core.auth.SingleTargetAuthorizationIntent; +import org.apache.polaris.service.TestServices; +import org.apache.polaris.service.catalog.common.PolarisSecurableMapper; +import org.apache.polaris.service.catalog.semanticmodel.types.UpdateSemanticModelRequest; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +/** + * Verifies that {@link SemanticModelCatalogHandler} enforces authorization: the five operations are + * gated behind {@code CATALOG_MANAGE_CONTENT} (see {@code RbacOperationSemantics}), so the + * unprivileged {@code test-principal} that {@link TestServices} bootstraps is rejected with {@link + * ForbiddenException}. This guards against silent regression of the gate. The allow path (an + * authorized principal succeeds) is covered end-to-end by {@link + * SemanticModelCatalogHandlerCrudTest} via the pass-through authorizer; the privilege-grant + * plumbing needed to build an authorized principal is not reachable through the {@link + * TestServices} surface. + */ +class SemanticModelCatalogHandlerAuthzTest extends AbstractSemanticModelCatalogHandlerTest { + + @BeforeEach + void seedModel() { + // Seed a model with a pass-through authorizer so load/update/drop reach the authz check + // (rather than failing with "not found") when exercised by the unprivileged handler. + passthroughHandler().createSemanticModel(NS, createRequest("m1", modelJson("ns1.t1"))); + } + + @Test + void createDeniedWithoutManageContent() { + assertThatThrownBy( + () -> + enforcingHandler() + .createSemanticModel(NS, createRequest("m2", modelJson("ns1.t1")))) + .isInstanceOf(ForbiddenException.class); + } + + @Test + void listDeniedWithoutManageContent() { + assertThatThrownBy(() -> enforcingHandler().listSemanticModels(NS, null, null)) + .isInstanceOf(ForbiddenException.class); + } + + @Test + void loadDeniedWithoutManageContent() { + assertThatThrownBy(() -> enforcingHandler().loadSemanticModel(identifier("m1"))) + .isInstanceOf(ForbiddenException.class); + } + + @Test + void updateDeniedWithoutManageContent() { + assertThatThrownBy( + () -> + enforcingHandler() + .updateSemanticModel( + identifier("m1"), + UpdateSemanticModelRequest.builder() + .setDocument(doc(modelJson("ns1.t1"))) + .setEntityVersion("1") + .build())) + .isInstanceOf(ForbiddenException.class); + } + + @Test + void dropDeniedWithoutManageContent() { + assertThatThrownBy(() -> enforcingHandler().dropSemanticModel(identifier("m1"))) + .isInstanceOf(ForbiddenException.class); + } + + @Test + void loadResolvesAuthorizationInputsForSemanticModel() { + PolarisAuthorizer authorizer = mock(PolarisAuthorizer.class); + doAnswer( + invocation -> { + AuthorizationState authorizationState = invocation.getArgument(0); + authorizationState.getResolutionManifest().resolveAll(); + return null; + }) + .when(authorizer) + .resolveAuthorizationInputs(any(), any()); + + handler(authorizer).loadSemanticModel(identifier("m1")); + + ArgumentCaptor requestCaptor = + ArgumentCaptor.forClass(AuthorizationRequest.class); + verify(authorizer).resolveAuthorizationInputs(any(), requestCaptor.capture()); + assertThat(requestCaptor.getValue().intents()) + .singleElement() + .isInstanceOfSatisfying( + SingleTargetAuthorizationIntent.class, + intent -> { + assertThat(intent.operation()) + .isEqualTo(PolarisAuthorizableOperation.LOAD_SEMANTIC_MODEL); + assertThat(intent.target()) + .isEqualTo(PolarisSecurableMapper.semanticModel(CATALOG_NAME, NS, "m1")); + }); + } + + /** Handler that actually enforces authorization for the unprivileged {@code test-principal}. */ + private SemanticModelCatalogHandler enforcingHandler() { + return handler(new PolarisAuthorizerImpl(services.realmConfig())); + } +} diff --git a/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandlerCrudTest.java b/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandlerCrudTest.java new file mode 100644 index 00000000000..c5ed2733a13 --- /dev/null +++ b/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogHandlerCrudTest.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.catalog.semanticmodel; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import org.apache.iceberg.exceptions.BadRequestException; +import org.apache.polaris.core.semantic.exceptions.NoSuchSemanticModelException; +import org.apache.polaris.service.TestServices; +import org.apache.polaris.service.catalog.semanticmodel.types.ListSemanticModelsResponse; +import org.apache.polaris.service.catalog.semanticmodel.types.LoadSemanticModelResponse; +import org.apache.polaris.service.catalog.semanticmodel.types.UpdateSemanticModelRequest; +import org.junit.jupiter.api.Test; + +/** + * Exercises {@link SemanticModelCatalogHandler} end-to-end against the real resolution manifest via + * {@link TestServices}. Unlike {@link SemanticModelCatalogTest} (pure Mockito), this drives the + * handler that builds a strict {@code PolarisResolutionManifest} — the only way to catch the + * passthrough-registration failures that broke create/update. Authorization is intentionally a + * pass-through mock here; privilege enforcement is covered by {@link + * SemanticModelCatalogHandlerAuthzTest}. + */ +class SemanticModelCatalogHandlerCrudTest extends AbstractSemanticModelCatalogHandlerTest { + + @Test + void createThenLoadRoundTrips() { + LoadSemanticModelResponse created = + passthroughHandler().createSemanticModel(NS, createRequest("m1", modelJson("ns1.t1"))); + + assertThat(created.getDocument().getSemanticModel()).isEqualTo(modelJson("ns1.t1")); + + LoadSemanticModelResponse loaded = passthroughHandler().loadSemanticModel(identifier("m1")); + assertThat(loaded.getDocument().getSemanticModel()).isEqualTo(modelJson("ns1.t1")); + assertThat(loaded.getEntityVersion()).isEqualTo(created.getEntityVersion()); + } + + @Test + void fullLifecycleCreateListUpdateDrop() { + passthroughHandler().createSemanticModel(NS, createRequest("m1", modelJson("ns1.t1"))); + + ListSemanticModelsResponse listed = passthroughHandler().listSemanticModels(NS, null, null); + assertThat(listed.getIdentifiers()).anySatisfy(id -> assertThat(id.getName()).isEqualTo("m1")); + + LoadSemanticModelResponse current = passthroughHandler().loadSemanticModel(identifier("m1")); + String updatedModel = modelJson("ns1.t1").replace("\"d\"", "\"d_renamed\""); + LoadSemanticModelResponse updated = + passthroughHandler() + .updateSemanticModel( + identifier("m1"), + UpdateSemanticModelRequest.builder() + .setDocument(doc(updatedModel)) + .setEntityVersion(current.getEntityVersion()) + .build()); + assertThat(updated.getDocument().getSemanticModel()).isEqualTo(updatedModel); + + passthroughHandler().dropSemanticModel(identifier("m1")); + assertThatThrownBy(() -> passthroughHandler().loadSemanticModel(identifier("m1"))) + .isInstanceOf(NoSuchSemanticModelException.class); + } + + @Test + void createRejectsUnknownSource() { + assertThatThrownBy( + () -> + passthroughHandler() + .createSemanticModel(NS, createRequest("m2", modelJson("ns1.missing")))) + .isInstanceOf(BadRequestException.class) + .hasMessageContaining("ns1.missing"); + } +} diff --git a/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogTest.java b/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogTest.java new file mode 100644 index 00000000000..4601da75d8d --- /dev/null +++ b/extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/SemanticModelCatalogTest.java @@ -0,0 +1,301 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.catalog.semanticmodel; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.Arrays; +import java.util.List; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.AlreadyExistsException; +import org.apache.iceberg.exceptions.BadRequestException; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.auth.PolarisPrincipal; +import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.entity.CatalogEntity; +import org.apache.polaris.core.entity.EntityNameLookupRecord; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisEntity; +import org.apache.polaris.core.entity.PolarisEntitySubType; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.persistence.PolarisMetaStoreManager; +import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; +import org.apache.polaris.core.persistence.ResolvedPolarisEntity; +import org.apache.polaris.core.persistence.dao.entity.BaseResult; +import org.apache.polaris.core.persistence.dao.entity.DropEntityResult; +import org.apache.polaris.core.persistence.dao.entity.EntityResult; +import org.apache.polaris.core.persistence.dao.entity.GenerateEntityIdResult; +import org.apache.polaris.core.persistence.dao.entity.ListEntitiesResult; +import org.apache.polaris.core.persistence.pagination.Page; +import org.apache.polaris.core.persistence.pagination.PageToken; +import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifest; +import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifestCatalogView; +import org.apache.polaris.core.persistence.resolver.ResolutionManifestFactory; +import org.apache.polaris.core.persistence.resolver.ResolvedPathKey; +import org.apache.polaris.core.semantic.SemanticModelEntity; +import org.apache.polaris.core.semantic.exceptions.NoSuchSemanticModelException; +import org.apache.polaris.core.semantic.exceptions.SemanticModelVersionMismatchException; +import org.apache.polaris.service.catalog.semanticmodel.types.LoadSemanticModelResponse; +import org.apache.polaris.service.catalog.semanticmodel.types.SemanticModelDocument; +import org.apache.polaris.service.catalog.semanticmodel.types.SemanticModelIdentifier; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.invocation.InvocationOnMock; + +/** + * Unit tests for {@link SemanticModelCatalog}. Following the {@code extensions/auth/opa} pattern, + * these are plain JUnit + Mockito tests that drive the catalog directly against a mocked resolution + * view and metastore manager — no Quarkus bootstrap. Full-stack coverage is left to the + * integration-test layer. + */ +class SemanticModelCatalogTest { + + private static final long CATALOG_ID = 1L; + private static final Namespace NS = Namespace.of("sales"); + private static final String MODEL = "m"; + private static final SemanticModelIdentifier IDENTIFIER = + SemanticModelIdentifier.builder().setNamespace(List.of("sales")).setName(MODEL).build(); + private static final String VALID_MODEL_JSON = + "[{\"name\":\"m\",\"datasets\":[{\"name\":\"d\",\"source\":\"sales.store_sales\"}]}]"; + + private PolarisResolutionManifestCatalogView view; + private PolarisMetaStoreManager metaStoreManager; + private ResolutionManifestFactory resolutionManifestFactory; + private PolarisResolutionManifest sourceManifest; + private SemanticModelCatalog catalog; + + private PolarisEntity catalogEntity; + private PolarisEntity namespaceEntity; + + @BeforeEach + void setUp() { + view = mock(PolarisResolutionManifestCatalogView.class); + metaStoreManager = mock(PolarisMetaStoreManager.class); + resolutionManifestFactory = mock(ResolutionManifestFactory.class); + sourceManifest = mock(PolarisResolutionManifest.class); + PolarisPrincipal principal = mock(PolarisPrincipal.class); + CallContext callContext = mock(CallContext.class); + PolarisCallContext polarisCallContext = mock(PolarisCallContext.class); + when(callContext.getPolarisCallContext()).thenReturn(polarisCallContext); + + catalogEntity = entity(PolarisEntityType.CATALOG, CATALOG_ID, "cat", 0L); + namespaceEntity = entity(PolarisEntityType.NAMESPACE, 2L, "sales", CATALOG_ID); + when(view.getResolvedCatalogEntity()).thenReturn(new CatalogEntity(catalogEntity)); + when(view.getResolvedPath(ResolvedPathKey.ofNamespace(NS))) + .thenReturn(path(catalogEntity, namespaceEntity)); + // Source tables are resolved through a fresh single-use manifest, not the request view. + when(resolutionManifestFactory.createResolutionManifest(any(), any())) + .thenReturn(sourceManifest); + + catalog = + new SemanticModelCatalog( + metaStoreManager, callContext, view, resolutionManifestFactory, principal); + } + + private SemanticModelDocument doc(String semanticModelJson) { + return SemanticModelDocument.builder() + .setVersion("0.1.1") + .setSemanticModel(semanticModelJson) + .build(); + } + + private void stubResolvableSource() { + PolarisEntity table = + new PolarisEntity.Builder() + .setType(PolarisEntityType.TABLE_LIKE) + .setSubType(PolarisEntitySubType.ICEBERG_TABLE) + .setId(5L) + .setCatalogId(CATALOG_ID) + .setParentId(2L) + .setName("store_sales") + .build(); + when(sourceManifest.getPassthroughResolvedPath( + eq(ResolvedPathKey.ofTableLike(TableIdentifier.of(NS, "store_sales"))), + eq(PolarisEntitySubType.ANY_SUBTYPE))) + .thenReturn(path(catalogEntity, namespaceEntity, table)); + } + + private void stubExistingModel(int entityVersion) { + SemanticModelEntity stored = + new SemanticModelEntity.Builder(NS, MODEL) + .setSpecVersion("0.1.1") + .setContent(VALID_MODEL_JSON) + .setId(10L) + .setCatalogId(CATALOG_ID) + .setParentId(2L) + .setEntityVersion(entityVersion) + .build(); + when(view.getPassthroughResolvedPath( + eq(ResolvedPathKey.ofSemanticModel(NS, MODEL)), eq(PolarisEntitySubType.NULL_SUBTYPE))) + .thenReturn(path(catalogEntity, namespaceEntity, stored)); + } + + @Test + void createResolvesSourcesAndPersists() { + stubResolvableSource(); + when(metaStoreManager.generateNewEntityId(any())).thenReturn(new GenerateEntityIdResult(10L)); + // Echo the entity back as the persisted result. + when(metaStoreManager.createEntityIfNotExists(any(), any(), any())) + .thenAnswer(SemanticModelCatalogTest::echoPersistedEntity); + + LoadSemanticModelResponse response = + catalog.createSemanticModel(IDENTIFIER, doc(VALID_MODEL_JSON)); + + assertThat(response.getDocument().getSemanticModel()).isEqualTo(VALID_MODEL_JSON); + assertThat(response.getDocument().getVersion()).isEqualTo("0.1.1"); + assertThat(response.getEntityVersion()).isEqualTo("1"); + } + + @Test + void createRejectsEmptyDocument() { + assertThatThrownBy(() -> catalog.createSemanticModel(IDENTIFIER, doc(""))) + .isInstanceOf(BadRequestException.class) + .hasMessageContaining("must not be empty"); + } + + @Test + void createRejectsDatasetWithoutSource() { + String noSource = "[{\"name\":\"m\",\"datasets\":[{\"name\":\"d\"}]}]"; + assertThatThrownBy(() -> catalog.createSemanticModel(IDENTIFIER, doc(noSource))) + .isInstanceOf(BadRequestException.class) + .hasMessageContaining("/semantic_model/0/datasets/0/source") + .hasMessageContaining("must define a string 'source'"); + } + + @Test + void createRejectsUnresolvedSource() { + // No stub for the source table -> passthrough resolution returns null. + assertThatThrownBy(() -> catalog.createSemanticModel(IDENTIFIER, doc(VALID_MODEL_JSON))) + .isInstanceOf(BadRequestException.class) + .hasMessageContaining("/semantic_model/0/datasets/0/source") + .hasMessageContaining("store_sales"); + } + + @Test + void createRejectsExistingModel() { + stubResolvableSource(); + when(metaStoreManager.generateNewEntityId(any())).thenReturn(new GenerateEntityIdResult(10L)); + // Duplicates are surfaced by the persistence layer, not a pre-check. + when(metaStoreManager.createEntityIfNotExists(any(), any(), any())) + .thenReturn(new EntityResult(BaseResult.ReturnStatus.ENTITY_ALREADY_EXISTS, null)); + assertThatThrownBy(() -> catalog.createSemanticModel(IDENTIFIER, doc(VALID_MODEL_JSON))) + .isInstanceOf(AlreadyExistsException.class); + } + + @Test + void loadReturnsStoredDocument() { + stubExistingModel(4); + LoadSemanticModelResponse response = catalog.loadSemanticModel(IDENTIFIER); + assertThat(response.getDocument().getSemanticModel()).isEqualTo(VALID_MODEL_JSON); + assertThat(response.getEntityVersion()).isEqualTo("4"); + } + + @Test + void loadRejectsMissingModel() { + assertThatThrownBy(() -> catalog.loadSemanticModel(IDENTIFIER)) + .isInstanceOf(NoSuchSemanticModelException.class); + } + + @Test + void updateRejectsVersionMismatch() { + stubExistingModel(3); + assertThatThrownBy(() -> catalog.updateSemanticModel(IDENTIFIER, doc(VALID_MODEL_JSON), "1")) + .isInstanceOf(SemanticModelVersionMismatchException.class); + } + + @Test + void updatePersistsWhenVersionMatches() { + stubExistingModel(3); + stubResolvableSource(); + when(metaStoreManager.updateEntityPropertiesIfNotChanged(any(), any(), any())) + .thenAnswer(SemanticModelCatalogTest::echoPersistedEntity); + + LoadSemanticModelResponse response = + catalog.updateSemanticModel(IDENTIFIER, doc(VALID_MODEL_JSON), "3"); + assertThat(response.getDocument().getSemanticModel()).isEqualTo(VALID_MODEL_JSON); + } + + @Test + void dropRemovesModel() { + stubExistingModel(1); + when(metaStoreManager.dropEntityIfExists(any(), any(), any(), any(), eq(false))) + .thenReturn(new DropEntityResult()); + assertThatCode(() -> catalog.dropSemanticModel(IDENTIFIER)).doesNotThrowAnyException(); + } + + @Test + void dropRejectsMissingModel() { + assertThatThrownBy(() -> catalog.dropSemanticModel(IDENTIFIER)) + .isInstanceOf(NoSuchSemanticModelException.class); + } + + @Test + void listReturnsIdentifiers() { + EntityNameLookupRecord record = + new EntityNameLookupRecord( + entity(PolarisEntityType.SEMANTIC_MODEL, 10L, MODEL, CATALOG_ID)); + when(metaStoreManager.listEntities( + any(), + any(), + eq(PolarisEntityType.SEMANTIC_MODEL), + eq(PolarisEntitySubType.NULL_SUBTYPE), + any())) + .thenReturn(new ListEntitiesResult(Page.fromItems(List.of(record)))); + + var response = catalog.listSemanticModels(NS, PageToken.readEverything()); + assertThat(response.getIdentifiers()) + .singleElement() + .satisfies( + id -> { + assertThat(id.getName()).isEqualTo(MODEL); + assertThat(id.getNamespace()).containsExactly("sales"); + }); + } + + // ---- helpers ---- + + /** Mockito answer that echoes the entity passed to a persist call back as the stored result. */ + private static EntityResult echoPersistedEntity(InvocationOnMock invocation) { + return new EntityResult((PolarisBaseEntity) invocation.getArgument(2)); + } + + private static PolarisEntity entity( + PolarisEntityType type, long id, String name, long catalogId) { + return new PolarisEntity.Builder() + .setType(type) + .setId(id) + .setCatalogId(catalogId) + .setName(name) + .build(); + } + + private static PolarisResolvedPathWrapper path(PolarisEntity... entities) { + return new PolarisResolvedPathWrapper( + Arrays.stream(entities) + .map(e -> new ResolvedPolarisEntity(e, List.of(), List.of())) + .toList()); + } +} diff --git a/persistence/nosql/persistence/metastore-types/src/main/java/org/apache/polaris/persistence/nosql/coretypes/mapping/EntityObjMappings.java b/persistence/nosql/persistence/metastore-types/src/main/java/org/apache/polaris/persistence/nosql/coretypes/mapping/EntityObjMappings.java index 1349b50ebb9..939ac3f22c6 100644 --- a/persistence/nosql/persistence/metastore-types/src/main/java/org/apache/polaris/persistence/nosql/coretypes/mapping/EntityObjMappings.java +++ b/persistence/nosql/persistence/metastore-types/src/main/java/org/apache/polaris/persistence/nosql/coretypes/mapping/EntityObjMappings.java @@ -90,6 +90,9 @@ public final class EntityObjMappings { static { var missingEntityTypes = new HashSet<>(Set.of(PolarisEntityType.values())); missingEntityTypes.remove(PolarisEntityType.NULL_TYPE); + // TODO: SEMANTIC_MODEL is not yet supported by the NoSQL backend; Excluded here so the NoSQL + // module still loads while the type exists in the shared enum. + missingEntityTypes.remove(PolarisEntityType.SEMANTIC_MODEL); missingEntityTypes.removeAll(BY_ENTITY_TYPE.keySet()); checkState( missingEntityTypes.isEmpty(), diff --git a/persistence/nosql/persistence/metastore-types/src/test/java/org/apache/polaris/persistence/nosql/coretypes/mapping/TestEntityObjMappings.java b/persistence/nosql/persistence/metastore-types/src/test/java/org/apache/polaris/persistence/nosql/coretypes/mapping/TestEntityObjMappings.java index 7687c7ec2fc..f7db74e2d95 100644 --- a/persistence/nosql/persistence/metastore-types/src/test/java/org/apache/polaris/persistence/nosql/coretypes/mapping/TestEntityObjMappings.java +++ b/persistence/nosql/persistence/metastore-types/src/test/java/org/apache/polaris/persistence/nosql/coretypes/mapping/TestEntityObjMappings.java @@ -119,6 +119,9 @@ public void entityObjMapping(PolarisEntityType entityType, PolarisEntitySubType static Stream entityObjMapping() { return Arrays.stream(PolarisEntityType.values()) .filter(t -> t != PolarisEntityType.NULL_TYPE) + // TODO: SEMANTIC_MODEL is not yet mapped by the NoSQL backend; drop this filter once its + // mapping is added. + .filter(t -> t != PolarisEntityType.SEMANTIC_MODEL) .flatMap( t -> { var subTypes = diff --git a/polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizableOperation.java b/polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizableOperation.java index c016749459b..2fc3b6157da 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizableOperation.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizableOperation.java @@ -115,6 +115,11 @@ public enum PolarisAuthorizableOperation { GET_APPLICABLE_POLICIES_ON_TABLE, ADD_POLICY_GRANT_TO_CATALOG_ROLE, REVOKE_POLICY_GRANT_FROM_CATALOG_ROLE, + CREATE_SEMANTIC_MODEL, + LOAD_SEMANTIC_MODEL, + UPDATE_SEMANTIC_MODEL, + DROP_SEMANTIC_MODEL, + LIST_SEMANTIC_MODEL, ASSIGN_TABLE_UUID, UPGRADE_TABLE_FORMAT_VERSION, ADD_TABLE_SCHEMA, diff --git a/polaris-core/src/main/java/org/apache/polaris/core/auth/RbacOperationSemantics.java b/polaris-core/src/main/java/org/apache/polaris/core/auth/RbacOperationSemantics.java index f63999959dd..e0628805f33 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/auth/RbacOperationSemantics.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/auth/RbacOperationSemantics.java @@ -41,6 +41,7 @@ import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.CREATE_POLICY; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.CREATE_PRINCIPAL; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.CREATE_PRINCIPAL_ROLE; +import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.CREATE_SEMANTIC_MODEL; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.CREATE_TABLE_DIRECT; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.CREATE_TABLE_DIRECT_WITH_WRITE_DELEGATION; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.CREATE_TABLE_STAGED; @@ -55,6 +56,7 @@ import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.DETACH_POLICY_FROM_TABLE; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.DROP_NAMESPACE; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.DROP_POLICY; +import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.DROP_SEMANTIC_MODEL; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.DROP_TABLE_WITHOUT_PURGE; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.DROP_TABLE_WITH_PURGE; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.DROP_VIEW; @@ -76,10 +78,12 @@ import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.LIST_PRINCIPALS; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.LIST_PRINCIPAL_ROLES; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.LIST_PRINCIPAL_ROLES_ASSIGNED; +import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.LIST_SEMANTIC_MODEL; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.LIST_TABLES; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.LIST_VIEWS; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.LOAD_NAMESPACE_METADATA; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.LOAD_POLICY; +import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.LOAD_SEMANTIC_MODEL; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.LOAD_TABLE; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.LOAD_TABLE_WITH_READ_DELEGATION; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.LOAD_TABLE_WITH_WRITE_DELEGATION; @@ -126,6 +130,7 @@ import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.UPDATE_POLICY; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.UPDATE_PRINCIPAL; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.UPDATE_PRINCIPAL_ROLE; +import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.UPDATE_SEMANTIC_MODEL; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.UPDATE_TABLE; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.UPDATE_TABLE_FOR_STAGED_CREATE; import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.UPGRADE_TABLE_FORMAT_VERSION; @@ -135,6 +140,7 @@ import static org.apache.polaris.core.entity.PolarisPrivilege.CATALOG_DETACH_POLICY; import static org.apache.polaris.core.entity.PolarisPrivilege.CATALOG_DROP; import static org.apache.polaris.core.entity.PolarisPrivilege.CATALOG_LIST; +import static org.apache.polaris.core.entity.PolarisPrivilege.CATALOG_MANAGE_CONTENT; import static org.apache.polaris.core.entity.PolarisPrivilege.CATALOG_MANAGE_GRANTS_ON_SECURABLE; import static org.apache.polaris.core.entity.PolarisPrivilege.CATALOG_READ_PROPERTIES; import static org.apache.polaris.core.entity.PolarisPrivilege.CATALOG_ROLE_CREATE; @@ -407,6 +413,16 @@ private static void register( register(UPDATE_POLICY, POLICY_WRITE); register(LIST_POLICY, POLICY_LIST); + // Semantic-model operations. + // Interim: gated behind the coarse CATALOG_MANAGE_CONTENT privilege, matching the + // design's "coarser-grained access reuses existing higher-level privileges" posture. + // TODO: The dedicated SEMANTIC_MODEL_* privilege matrix will be implemented in a followup. + register(CREATE_SEMANTIC_MODEL, CATALOG_MANAGE_CONTENT); + register(LOAD_SEMANTIC_MODEL, CATALOG_MANAGE_CONTENT); + register(UPDATE_SEMANTIC_MODEL, CATALOG_MANAGE_CONTENT); + register(DROP_SEMANTIC_MODEL, CATALOG_MANAGE_CONTENT); + register(LIST_SEMANTIC_MODEL, CATALOG_MANAGE_CONTENT); + // Policy attachment operations (use CATALOG rooting) register( ATTACH_POLICY_TO_CATALOG, diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntityType.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntityType.java index 13aa1f465db..0896f314163 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntityType.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntityType.java @@ -34,7 +34,8 @@ public enum PolarisEntityType { TABLE_LIKE(7, NAMESPACE, false, false), TASK(8, ROOT, false, false), FILE(9, TABLE_LIKE, false, false), - POLICY(10, NAMESPACE, false, false); + POLICY(10, NAMESPACE, false, false), + SEMANTIC_MODEL(11, NAMESPACE, false, false); // to efficiently map a code to its corresponding entity type, use a reverse array which // is initialized below diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/ResolvedPathKey.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/ResolvedPathKey.java index d1acfc2ca45..934cd5bdbc9 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/ResolvedPathKey.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/ResolvedPathKey.java @@ -54,6 +54,11 @@ public static ResolvedPathKey ofPolicy(Namespace namespace, String name) { PolarisCatalogHelpers.identifierToList(namespace, name), PolarisEntityType.POLICY); } + public static ResolvedPathKey ofSemanticModel(Namespace namespace, String name) { + return new ResolvedPathKey( + PolarisCatalogHelpers.identifierToList(namespace, name), PolarisEntityType.SEMANTIC_MODEL); + } + public static ResolvedPathKey ofCatalogRole(String roleName) { return new ResolvedPathKey(List.of(roleName), PolarisEntityType.CATALOG_ROLE); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/semantic/SemanticModelEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/semantic/SemanticModelEntity.java new file mode 100644 index 00000000000..6ecd690406e --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/semantic/SemanticModelEntity.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.semantic; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.common.base.Preconditions; +import org.apache.iceberg.catalog.Namespace; +import org.apache.polaris.core.entity.NamespaceEntity; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisEntity; +import org.apache.polaris.core.entity.PolarisEntitySubType; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.entity.PolarisEntityUtils; +import org.jspecify.annotations.Nullable; + +/** + * A Polaris entity that stores an Apache Ossie semantic-model document. + * + *

Following the precedent set by {@link org.apache.polaris.core.policy.PolicyEntity}, the full + * document body is stored inside the entity {@code properties} map: the declared Ossie spec version + * under {@link #SPEC_VERSION_KEY} and the Ossie document (as a JSON string) under {@link + * #CONTENT_KEY}. + */ +public class SemanticModelEntity extends PolarisEntity { + + /** The declared Ossie spec version of the stored document (e.g. {@code 0.1.1}). */ + public static final String SPEC_VERSION_KEY = "semantic-model.spec-version"; + + /** The Ossie document serialized as a JSON string. */ + public static final String CONTENT_KEY = "semantic-model.content"; + + public SemanticModelEntity(PolarisBaseEntity sourceEntity) { + super(sourceEntity); + Preconditions.checkState( + getType() == PolarisEntityType.SEMANTIC_MODEL, "Invalid entity type: %s", getType()); + Preconditions.checkState( + getSubType() == PolarisEntitySubType.NULL_SUBTYPE, + "Invalid entity sub type: %s", + getSubType()); + } + + public static @Nullable SemanticModelEntity of(@Nullable PolarisBaseEntity sourceEntity) { + if (sourceEntity != null) { + return new SemanticModelEntity(sourceEntity); + } + return null; + } + + /** The declared Ossie spec version of the stored document. */ + @JsonIgnore + public String getSpecVersion() { + return getPropertiesAsMap().get(SPEC_VERSION_KEY); + } + + /** The Ossie document body, serialized as a JSON string. */ + @JsonIgnore + public String getContent() { + return getPropertiesAsMap().get(CONTENT_KEY); + } + + @JsonIgnore + public @Nullable Namespace getParentNamespace() { + String parentNamespace = getInternalPropertiesAsMap().get(NamespaceEntity.PARENT_NAMESPACE_KEY); + if (parentNamespace != null) { + return PolarisEntityUtils.decodeNamespace(parentNamespace); + } + return null; + } + + public static class Builder extends PolarisEntity.BaseBuilder { + public Builder(Namespace namespace, String modelName) { + super(); + setType(PolarisEntityType.SEMANTIC_MODEL); + setParentNamespace(namespace); + setName(modelName); + } + + public Builder(SemanticModelEntity original) { + super(original); + } + + @Override + public SemanticModelEntity build() { + Preconditions.checkArgument( + properties.containsKey(SPEC_VERSION_KEY), + "Semantic model spec version must be specified"); + Preconditions.checkArgument( + properties.containsKey(CONTENT_KEY), "Semantic model content must be specified"); + return new SemanticModelEntity(buildBase()); + } + + public Builder setParentNamespace(Namespace namespace) { + if (namespace != null && !namespace.isEmpty()) { + internalProperties.put( + NamespaceEntity.PARENT_NAMESPACE_KEY, PolarisEntityUtils.encodeNamespace(namespace)); + } + return this; + } + + public Builder setSpecVersion(String specVersion) { + properties.put(SPEC_VERSION_KEY, specVersion); + return this; + } + + public Builder setContent(String content) { + properties.put(CONTENT_KEY, content); + return this; + } + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/semantic/exceptions/NoSuchSemanticModelException.java b/polaris-core/src/main/java/org/apache/polaris/core/semantic/exceptions/NoSuchSemanticModelException.java new file mode 100644 index 00000000000..ecae524d764 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/semantic/exceptions/NoSuchSemanticModelException.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.semantic.exceptions; + +import org.apache.polaris.core.exceptions.PolarisException; + +/** Thrown when a semantic model cannot be found. Maps to HTTP 404. */ +public class NoSuchSemanticModelException extends PolarisException { + public NoSuchSemanticModelException(String message) { + super(message); + } + + public NoSuchSemanticModelException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/semantic/exceptions/SemanticModelVersionMismatchException.java b/polaris-core/src/main/java/org/apache/polaris/core/semantic/exceptions/SemanticModelVersionMismatchException.java new file mode 100644 index 00000000000..b7804392e22 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/semantic/exceptions/SemanticModelVersionMismatchException.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.semantic.exceptions; + +import org.apache.polaris.core.exceptions.PolarisException; + +/** + * Thrown when a semantic-model update supplies an {@code entity-version} that does not match the + * version currently stored in the catalog (optimistic-concurrency conflict). Maps to HTTP 409. + */ +public class SemanticModelVersionMismatchException extends PolarisException { + public SemanticModelVersionMismatchException(String message) { + super(message); + } + + public SemanticModelVersionMismatchException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/polaris-core/src/test/java/org/apache/polaris/core/entity/PolarisEntityTypeTest.java b/polaris-core/src/test/java/org/apache/polaris/core/entity/PolarisEntityTypeTest.java index c2b9cd6ea9c..11c090b61b0 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/entity/PolarisEntityTypeTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/entity/PolarisEntityTypeTest.java @@ -40,7 +40,8 @@ static Stream entityTypes() { Arguments.of(8, PolarisEntityType.TASK), Arguments.of(9, PolarisEntityType.FILE), Arguments.of(10, PolarisEntityType.POLICY), - Arguments.of(11, null)); + Arguments.of(11, PolarisEntityType.SEMANTIC_MODEL), + Arguments.of(12, null)); } @ParameterizedTest diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/common/PolarisSecurableMapper.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/common/PolarisSecurableMapper.java index 733aa5a2256..7a2ceb39b12 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/common/PolarisSecurableMapper.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/common/PolarisSecurableMapper.java @@ -87,6 +87,20 @@ public static PolarisSecurable policy(String catalogName, PolicyIdentifier ident .build(); } + public static PolarisSecurable semanticModel( + String catalogName, Namespace namespace, String name) { + if (namespace.isEmpty()) { + throw new IllegalArgumentException("Semantic-model target cannot have an empty namespace"); + } + ImmutablePolarisSecurable.Builder builder = + ImmutablePolarisSecurable.builder() + .addPathSegment(new PathSegment(PolarisEntityType.CATALOG, catalogName)); + Arrays.stream(namespace.levels()) + .map(level -> new PathSegment(PolarisEntityType.NAMESPACE, level)) + .forEach(builder::addPathSegment); + return builder.addPathSegment(new PathSegment(PolarisEntityType.SEMANTIC_MODEL, name)).build(); + } + public static PolarisSecurable policyAttachmentTarget( String catalogName, PolicyAttachmentTarget target) { ImmutablePolarisSecurable.Builder builder = diff --git a/runtime/service/src/main/java/org/apache/polaris/service/exception/PolarisExceptionMapper.java b/runtime/service/src/main/java/org/apache/polaris/service/exception/PolarisExceptionMapper.java index 1e9926bfd7c..443982ba035 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/exception/PolarisExceptionMapper.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/exception/PolarisExceptionMapper.java @@ -33,6 +33,8 @@ import org.apache.polaris.core.policy.exceptions.PolicyInUseException; import org.apache.polaris.core.policy.exceptions.PolicyVersionMismatchException; import org.apache.polaris.core.policy.validator.InvalidPolicyException; +import org.apache.polaris.core.semantic.exceptions.NoSuchSemanticModelException; +import org.apache.polaris.core.semantic.exceptions.SemanticModelVersionMismatchException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.event.Level; @@ -58,6 +60,9 @@ private Response.Status getStatus(PolarisException exception) { case PolicyMappingAlreadyExistsException policyMappingAlreadyExistsException -> Response.Status.CONFLICT; case PolicyInUseException policyInUseException -> Response.Status.BAD_REQUEST; + case NoSuchSemanticModelException noSuchSemanticModelException -> Response.Status.NOT_FOUND; + case SemanticModelVersionMismatchException semanticModelVersionMismatchException -> + Response.Status.CONFLICT; default -> Response.Status.INTERNAL_SERVER_ERROR; }; } diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/common/PolarisSecurableMapperTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/common/PolarisSecurableMapperTest.java index 2a1ef675b90..c7af48d71e1 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/common/PolarisSecurableMapperTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/common/PolarisSecurableMapperTest.java @@ -49,4 +49,16 @@ void tableLikeMapsNamespaceAndTableName() { new PathSegment(PolarisEntityType.NAMESPACE, "ns2"), new PathSegment(PolarisEntityType.TABLE_LIKE, "table")); } + + @Test + void semanticModelMapsNamespaceAndModelName() { + assertThat( + PolarisSecurableMapper.semanticModel("catalog", Namespace.of("ns1", "ns2"), "model") + .getPathSegments()) + .containsExactly( + new PathSegment(PolarisEntityType.CATALOG, "catalog"), + new PathSegment(PolarisEntityType.NAMESPACE, "ns1"), + new PathSegment(PolarisEntityType.NAMESPACE, "ns2"), + new PathSegment(PolarisEntityType.SEMANTIC_MODEL, "model")); + } }