From c0656d73a49fb7996c19c5f622572ea2b8791914 Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Wed, 3 Dec 2025 10:36:38 +0100 Subject: [PATCH 1/9] NoSQL: Metastore implementation --- bom/build.gradle.kts | 1 + gradle/projects.main.properties | 1 + .../persistence/metastore/build.gradle.kts | 73 + .../nosql/metastore/ContentIdentifier.java | 168 ++ .../nosql/metastore/NoSqlMetaStore.java | 1347 +++++++++++++++++ .../metastore/NoSqlMetaStoreManager.java | 831 ++++++++++ .../NoSqlMetaStoreManagerFactory.java | 350 +++++ .../nosql/metastore/NoSqlPaginationToken.java | 69 + .../NonFunctionalBasePersistence.java | 368 +++++ .../committers/CatalogChangeCommitter.java | 42 + .../CatalogChangeCommitterWrapper.java | 88 ++ .../metastore/committers/ChangeCommitter.java | 38 + .../committers/ChangeCommitterWrapper.java | 103 ++ .../metastore/committers/ChangeResult.java | 28 + .../committers/PrincipalsChangeCommitter.java | 39 + .../PrincipalsChangeCommitterWrapper.java | 84 + .../metastore/indexaccess/IndexUtils.java | 74 + .../indexaccess/IndexedContainerAccess.java | 73 + .../IndexedContainerAccessImpl.java | 170 +++ .../IndexedContainerAccessRoot.java | 187 +++ .../indexaccess/MemoizedIndexedAccess.java | 178 +++ .../metastore/mutation/EntityUpdate.java | 33 + .../metastore/mutation/GrantsMutation.java | 189 +++ .../metastore/mutation/MutationAttempt.java | 593 ++++++++ .../mutation/MutationAttemptRoot.java | 59 + .../metastore/mutation/MutationResults.java | 148 ++ .../metastore/mutation/PolicyMutation.java | 156 ++ .../mutation/PrincipalMutations.java | 348 +++++ .../UpdateKeyForCatalogAndEntityType.java | 38 + .../nosql/metastore/privs/GrantTriplet.java | 57 + .../privs/PolarisPrivilegesProvider.java | 42 + .../metastore/privs/SecurableAndGrantee.java | 60 + .../privs/SecurableGranteePrivilegeTuple.java | 28 + .../src/main/resources/META-INF/beans.xml | 24 + ...ore.persistence.pagination.Token$TokenType | 20 + .../metastore/TestContentIdentifier.java | 62 + .../metastore/TestNoSqlMetaStoreManager.java | 259 ++++ .../nosql/metastore/TestNoSqlResolver.java | 105 ++ .../TestIndexedContainerAccess.java | 312 ++++ .../indexaccess/TestMemoizedIndexAccess.java | 214 +++ .../src/test/resources/logback-test.xml | 35 + .../src/test/resources/weld.properties | 21 + .../nosql/metastore/CdiProducers.java | 60 + .../testFixtures/resources/META-INF/beans.xml | 24 + 44 files changed, 7199 insertions(+) create mode 100644 persistence/nosql/persistence/metastore/build.gradle.kts create mode 100644 persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/ContentIdentifier.java create mode 100644 persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NoSqlMetaStore.java create mode 100644 persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NoSqlMetaStoreManager.java create mode 100644 persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NoSqlMetaStoreManagerFactory.java create mode 100644 persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NoSqlPaginationToken.java create mode 100644 persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NonFunctionalBasePersistence.java create mode 100644 persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/committers/CatalogChangeCommitter.java create mode 100644 persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/committers/CatalogChangeCommitterWrapper.java create mode 100644 persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/committers/ChangeCommitter.java create mode 100644 persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/committers/ChangeCommitterWrapper.java create mode 100644 persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/committers/ChangeResult.java create mode 100644 persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/committers/PrincipalsChangeCommitter.java create mode 100644 persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/committers/PrincipalsChangeCommitterWrapper.java create mode 100644 persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/indexaccess/IndexUtils.java create mode 100644 persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/indexaccess/IndexedContainerAccess.java create mode 100644 persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/indexaccess/IndexedContainerAccessImpl.java create mode 100644 persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/indexaccess/IndexedContainerAccessRoot.java create mode 100644 persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/indexaccess/MemoizedIndexedAccess.java create mode 100644 persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/EntityUpdate.java create mode 100644 persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/GrantsMutation.java create mode 100644 persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/MutationAttempt.java create mode 100644 persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/MutationAttemptRoot.java create mode 100644 persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/MutationResults.java create mode 100644 persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/PolicyMutation.java create mode 100644 persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/PrincipalMutations.java create mode 100644 persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/UpdateKeyForCatalogAndEntityType.java create mode 100644 persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/privs/GrantTriplet.java create mode 100644 persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/privs/PolarisPrivilegesProvider.java create mode 100644 persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/privs/SecurableAndGrantee.java create mode 100644 persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/privs/SecurableGranteePrivilegeTuple.java create mode 100644 persistence/nosql/persistence/metastore/src/main/resources/META-INF/beans.xml create mode 100644 persistence/nosql/persistence/metastore/src/main/resources/META-INF/services/org.apache.polaris.core.persistence.pagination.Token$TokenType create mode 100644 persistence/nosql/persistence/metastore/src/test/java/org/apache/polaris/persistence/nosql/metastore/TestContentIdentifier.java create mode 100644 persistence/nosql/persistence/metastore/src/test/java/org/apache/polaris/persistence/nosql/metastore/TestNoSqlMetaStoreManager.java create mode 100644 persistence/nosql/persistence/metastore/src/test/java/org/apache/polaris/persistence/nosql/metastore/TestNoSqlResolver.java create mode 100644 persistence/nosql/persistence/metastore/src/test/java/org/apache/polaris/persistence/nosql/metastore/indexaccess/TestIndexedContainerAccess.java create mode 100644 persistence/nosql/persistence/metastore/src/test/java/org/apache/polaris/persistence/nosql/metastore/indexaccess/TestMemoizedIndexAccess.java create mode 100644 persistence/nosql/persistence/metastore/src/test/resources/logback-test.xml create mode 100644 persistence/nosql/persistence/metastore/src/test/resources/weld.properties create mode 100644 persistence/nosql/persistence/metastore/src/testFixtures/java/org/apache/polaris/persistence/nosql/metastore/CdiProducers.java create mode 100644 persistence/nosql/persistence/metastore/src/testFixtures/resources/META-INF/beans.xml diff --git a/bom/build.gradle.kts b/bom/build.gradle.kts index eb73c1d86f..213f31ed76 100644 --- a/bom/build.gradle.kts +++ b/bom/build.gradle.kts @@ -62,6 +62,7 @@ dependencies { api(project(":polaris-persistence-nosql-api")) api(project(":polaris-persistence-nosql-impl")) api(project(":polaris-persistence-nosql-benchmark")) + api(project(":polaris-persistence-nosql-metastore")) api(project(":polaris-persistence-nosql-metastore-types")) api(project(":polaris-persistence-nosql-correctness")) api(project(":polaris-persistence-nosql-cdi-common")) diff --git a/gradle/projects.main.properties b/gradle/projects.main.properties index 3706c6f132..85ff05b26d 100644 --- a/gradle/projects.main.properties +++ b/gradle/projects.main.properties @@ -77,6 +77,7 @@ polaris-persistence-nosql-realms-store-nosql=persistence/nosql/realms/store-nosq polaris-persistence-nosql-api=persistence/nosql/persistence/api polaris-persistence-nosql-impl=persistence/nosql/persistence/impl polaris-persistence-nosql-benchmark=persistence/nosql/persistence/benchmark +polaris-persistence-nosql-metastore=persistence/nosql/persistence/metastore polaris-persistence-nosql-metastore-types=persistence/nosql/persistence/metastore-types polaris-persistence-nosql-correctness=persistence/nosql/persistence/correctness polaris-persistence-nosql-cdi-common=persistence/nosql/persistence/cdi/common diff --git a/persistence/nosql/persistence/metastore/build.gradle.kts b/persistence/nosql/persistence/metastore/build.gradle.kts new file mode 100644 index 0000000000..ecb1c78c4e --- /dev/null +++ b/persistence/nosql/persistence/metastore/build.gradle.kts @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +plugins { + id("org.kordamp.gradle.jandex") + id("polaris-server") +} + +description = "Polaris NoSQL persistence - bridge to meta-store" + +dependencies { + implementation(project(":polaris-core")) + + implementation(project(":polaris-persistence-nosql-api")) + implementation(project(":polaris-persistence-nosql-metastore-types")) + implementation(project(":polaris-persistence-nosql-authz-api")) + implementation(project(":polaris-persistence-nosql-authz-spi")) + implementation(project(":polaris-persistence-nosql-realms-api")) + implementation(project(":polaris-idgen-api")) + implementation(project(":polaris-api-catalog-service")) + + implementation(platform(libs.iceberg.bom)) + implementation("org.apache.iceberg:iceberg-api") + implementation("org.apache.iceberg:iceberg-core") + + implementation(libs.guava) + implementation(libs.slf4j.api) + + implementation(platform(libs.jackson.bom)) + implementation("com.fasterxml.jackson.core:jackson-annotations") + implementation("com.fasterxml.jackson.core:jackson-core") + implementation("com.fasterxml.jackson.core:jackson-databind") + + compileOnly(project(":polaris-immutables")) + annotationProcessor(project(":polaris-immutables", configuration = "processor")) + + compileOnly(libs.jakarta.annotation.api) + compileOnly(libs.jakarta.validation.api) + compileOnly(libs.jakarta.inject.api) + compileOnly(libs.jakarta.enterprise.cdi.api) + compileOnly(libs.smallrye.common.annotation) + + testImplementation(testFixtures(project(":polaris-core"))) + testRuntimeOnly(project(":polaris-persistence-nosql-authz-impl")) + testRuntimeOnly(project(":polaris-persistence-nosql-authz-store-nosql")) + testRuntimeOnly(testFixtures(project(":polaris-persistence-nosql-cdi-weld"))) + testImplementation(libs.weld.se.core) + testImplementation(libs.weld.junit5) + testRuntimeOnly(libs.smallrye.jandex) + + testFixturesImplementation(project(":polaris-core")) + testFixturesImplementation(testFixtures(project(":polaris-core"))) + testFixturesImplementation(libs.jakarta.annotation.api) + testFixturesImplementation(libs.jakarta.validation.api) + testFixturesImplementation(libs.jakarta.enterprise.cdi.api) + testCompileOnly(libs.smallrye.common.annotation) +} diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/ContentIdentifier.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/ContentIdentifier.java new file mode 100644 index 0000000000..6f11f71e4c --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/ContentIdentifier.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.nosql.metastore; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; + +import com.fasterxml.jackson.annotation.JsonValue; +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import java.util.ArrayList; +import java.util.List; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.polaris.immutables.PolarisImmutable; +import org.apache.polaris.persistence.nosql.api.index.IndexKey; +import org.apache.polaris.service.types.PolicyIdentifier; +import org.immutables.value.Value; + +@Value.Style(underrideToString = "asDotDelimitedString") +@PolarisImmutable +public interface ContentIdentifier { + @Value.Parameter + @JsonValue + List elements(); + + static ContentIdentifier identifier(List elements) { + return ImmutableContentIdentifier.of(elements); + } + + static ContentIdentifier identifier(String[] namespace, String name) { + return ImmutableContentIdentifier.builder().addElements(namespace).addElements(name).build(); + } + + static ContentIdentifier identifier(String... elements) { + return ImmutableContentIdentifier.of(List.of(elements)); + } + + static ContentIdentifier identifierFor(PolicyIdentifier identifier) { + return identifier(identifier.getNamespace().levels(), identifier.getName()); + } + + static ContentIdentifier identifierFor(TableIdentifier identifier) { + return identifier(identifier.namespace().levels(), identifier.name()); + } + + static ContentIdentifier identifierFor(Namespace namespace) { + return identifier(namespace.levels()); + } + + static ContentIdentifier identifierFromLocationString(String locationString) { + var builder = builder(); + var len = locationString.length(); + var off = -1; + for (var i = 0; i < len; i++) { + var c = locationString.charAt(i); + checkArgument(c >= ' ', "Control characters are forbidden in locations"); + if (c == '/' || c == '\\') { + if (off != -1) { + builder.addElements(locationString.substring(off, i)); + off = -1; + } + } else { + if (off == -1) { + off = i; + } + } + } + if (off != -1) { + builder.addElements(locationString.substring(off)); + } + return builder.build(); + } + + default ContentIdentifier parent() { + var elems = elements(); + checkState(!elems.isEmpty(), "Empty namespace has no parent"); + return ImmutableContentIdentifier.of(elems.subList(0, elems.size() - 1)); + } + + default boolean isEmpty() { + return elements().isEmpty(); + } + + default int length() { + return elements().size(); + } + + default String leafName() { + var elems = elements(); + return elems.isEmpty() ? "" : elems.getLast(); + } + + default ContentIdentifier childOf(String childName) { + var elems = elements(); + var newElements = new ArrayList(elems.size() + 1); + newElements.addAll(elems); + newElements.add(childName); + return ImmutableContentIdentifier.of(newElements); + } + + default String asDotDelimitedString() { + return String.join(".", elements()); + } + + static ImmutableContentIdentifier.Builder builder() { + return ImmutableContentIdentifier.builder(); + } + + default IndexKey toIndexKey() { + return IndexKey.key(String.join("\u0000", elements())); + } + + default boolean startsWith(ContentIdentifier other) { + var elems = elements(); + var otherElems = other.elements(); + var otherSize = otherElems.size(); + if (otherSize > elems.size()) { + return false; + } + for (int i = 0; i < otherSize; i++) { + if (!elems.get(i).equals(otherElems.get(i))) { + return false; + } + } + return true; + } + + @CanIgnoreReturnValue + static ImmutableContentIdentifier.Builder indexKeyToIdentifierBuilder( + IndexKey indexKey, ImmutableContentIdentifier.Builder builder) { + var str = indexKey.toString(); + var l = str.length(); + for (var i = 0; i < l; ) { + var iNull = str.indexOf(0, i); + if (iNull == -1) { + builder.addElements(str.substring(i)); + return builder; + } + builder.addElements(str.substring(i, iNull)); + i = iNull + 1; + } + return builder; + } + + static ImmutableContentIdentifier.Builder indexKeyToIdentifierBuilder(IndexKey indexKey) { + return indexKeyToIdentifierBuilder(indexKey, ImmutableContentIdentifier.builder()); + } + + static ContentIdentifier indexKeyToIdentifier(IndexKey indexKey) { + return indexKeyToIdentifierBuilder(indexKey).build(); + } +} diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NoSqlMetaStore.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NoSqlMetaStore.java new file mode 100644 index 0000000000..dee536976b --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NoSqlMetaStore.java @@ -0,0 +1,1347 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.nosql.metastore; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; +import static org.apache.polaris.core.entity.PolarisEntityConstants.ENTITY_BASE_LOCATION; +import static org.apache.polaris.core.persistence.dao.entity.BaseResult.ReturnStatus.ENTITY_ALREADY_EXISTS; +import static org.apache.polaris.core.persistence.dao.entity.BaseResult.ReturnStatus.ENTITY_NOT_FOUND; +import static org.apache.polaris.persistence.nosql.api.index.IndexContainer.newUpdatableIndex; +import static org.apache.polaris.persistence.nosql.api.index.IndexKey.INDEX_KEY_SERIALIZER; +import static org.apache.polaris.persistence.nosql.api.obj.ObjRef.OBJ_REF_SERIALIZER; +import static org.apache.polaris.persistence.nosql.api.obj.ObjRef.objRef; +import static org.apache.polaris.persistence.nosql.coretypes.catalog.CatalogStateObj.CATALOG_STATE_REF_NAME_PATTERN; +import static org.apache.polaris.persistence.nosql.coretypes.catalog.EntityIdSet.ENTITY_ID_SET_SERIALIZER; +import static org.apache.polaris.persistence.nosql.coretypes.mapping.EntityObjMappings.containerTypeForEntityType; +import static org.apache.polaris.persistence.nosql.coretypes.mapping.EntityObjMappings.filterIsEntityType; +import static org.apache.polaris.persistence.nosql.coretypes.mapping.EntityObjMappings.isCatalogContent; +import static org.apache.polaris.persistence.nosql.coretypes.mapping.EntityObjMappings.mapToEntity; +import static org.apache.polaris.persistence.nosql.coretypes.mapping.EntityObjMappings.objTypeForPolarisTypeForFiltering; +import static org.apache.polaris.persistence.nosql.coretypes.mapping.EntityObjMappings.referenceName; +import static org.apache.polaris.persistence.nosql.coretypes.realm.PolicyMapping.POLICY_MAPPING_SERIALIZER; +import static org.apache.polaris.persistence.nosql.coretypes.realm.PolicyMappingsObj.POLICY_MAPPINGS_REF_NAME; +import static org.apache.polaris.persistence.nosql.coretypes.realm.PolicyMappingsObj.PolicyMappingKey.fromIndexKey; +import static org.apache.polaris.persistence.nosql.coretypes.realm.RootObj.ROOT_REF_NAME; +import static org.apache.polaris.persistence.nosql.coretypes.refs.References.catalogReferenceNames; +import static org.apache.polaris.persistence.nosql.metastore.ContentIdentifier.identifierFromLocationString; +import static org.apache.polaris.persistence.nosql.metastore.ContentIdentifier.indexKeyToIdentifier; +import static org.apache.polaris.persistence.nosql.metastore.indexaccess.MemoizedIndexedAccess.newMemoizedIndexedAccess; +import static org.apache.polaris.persistence.nosql.metastore.mutation.EntityUpdate.Operation.CREATE; +import static org.apache.polaris.persistence.nosql.metastore.mutation.EntityUpdate.Operation.UPDATE; +import static org.apache.polaris.persistence.nosql.metastore.mutation.UpdateKeyForCatalogAndEntityType.updateKeyForCatalogAndEntityType; + +import com.google.common.collect.Streams; +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.entity.AsyncTaskType; +import org.apache.polaris.core.entity.LocationBasedEntity; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisEntity; +import org.apache.polaris.core.entity.PolarisEntityConstants; +import org.apache.polaris.core.entity.PolarisEntityCore; +import org.apache.polaris.core.entity.PolarisEntitySubType; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.entity.PolarisGrantRecord; +import org.apache.polaris.core.entity.PolarisPrincipalSecrets; +import org.apache.polaris.core.entity.PolarisPrivilege; +import org.apache.polaris.core.entity.PolarisTaskConstants; +import org.apache.polaris.core.persistence.BaseMetaStoreManager; +import org.apache.polaris.core.persistence.PolarisObjectMapperUtil; +import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; +import org.apache.polaris.core.persistence.dao.entity.CreateCatalogResult; +import org.apache.polaris.core.persistence.dao.entity.CreatePrincipalResult; +import org.apache.polaris.core.persistence.dao.entity.DropEntityResult; +import org.apache.polaris.core.persistence.dao.entity.EntitiesResult; +import org.apache.polaris.core.persistence.dao.entity.EntityResult; +import org.apache.polaris.core.persistence.dao.entity.EntityWithPath; +import org.apache.polaris.core.persistence.dao.entity.LoadGrantsResult; +import org.apache.polaris.core.persistence.dao.entity.LoadPolicyMappingsResult; +import org.apache.polaris.core.persistence.dao.entity.PolicyAttachmentResult; +import org.apache.polaris.core.persistence.pagination.Page; +import org.apache.polaris.core.persistence.pagination.PageToken; +import org.apache.polaris.core.policy.PolarisPolicyMappingRecord; +import org.apache.polaris.core.policy.PolicyMappingUtil; +import org.apache.polaris.core.policy.PolicyType; +import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; +import org.apache.polaris.core.storage.PolarisStorageIntegration; +import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; +import org.apache.polaris.core.storage.StorageLocation; +import org.apache.polaris.persistence.nosql.api.Persistence; +import org.apache.polaris.persistence.nosql.api.index.Index; +import org.apache.polaris.persistence.nosql.api.index.IndexKey; +import org.apache.polaris.persistence.nosql.api.obj.ObjRef; +import org.apache.polaris.persistence.nosql.api.obj.ObjTypes; +import org.apache.polaris.persistence.nosql.authz.api.Privilege; +import org.apache.polaris.persistence.nosql.authz.api.PrivilegeSet; +import org.apache.polaris.persistence.nosql.authz.api.Privileges; +import org.apache.polaris.persistence.nosql.coretypes.ContainerObj; +import org.apache.polaris.persistence.nosql.coretypes.ObjBase; +import org.apache.polaris.persistence.nosql.coretypes.acl.AclObj; +import org.apache.polaris.persistence.nosql.coretypes.catalog.CatalogObj; +import org.apache.polaris.persistence.nosql.coretypes.catalog.CatalogRoleObj; +import org.apache.polaris.persistence.nosql.coretypes.catalog.CatalogRolesObj; +import org.apache.polaris.persistence.nosql.coretypes.catalog.CatalogStateObj; +import org.apache.polaris.persistence.nosql.coretypes.catalog.CatalogsObj; +import org.apache.polaris.persistence.nosql.coretypes.content.ContentObj; +import org.apache.polaris.persistence.nosql.coretypes.mapping.EntityObjMappings; +import org.apache.polaris.persistence.nosql.coretypes.principals.PrincipalObj; +import org.apache.polaris.persistence.nosql.coretypes.principals.PrincipalsObj; +import org.apache.polaris.persistence.nosql.coretypes.realm.PolicyMapping; +import org.apache.polaris.persistence.nosql.coretypes.realm.PolicyMappingsObj; +import org.apache.polaris.persistence.nosql.coretypes.realm.RootObj; +import org.apache.polaris.persistence.nosql.metastore.committers.CatalogChangeCommitterWrapper; +import org.apache.polaris.persistence.nosql.metastore.committers.ChangeCommitter; +import org.apache.polaris.persistence.nosql.metastore.committers.ChangeCommitterWrapper; +import org.apache.polaris.persistence.nosql.metastore.committers.ChangeResult; +import org.apache.polaris.persistence.nosql.metastore.indexaccess.IndexedContainerAccess; +import org.apache.polaris.persistence.nosql.metastore.indexaccess.MemoizedIndexedAccess; +import org.apache.polaris.persistence.nosql.metastore.mutation.EntityUpdate; +import org.apache.polaris.persistence.nosql.metastore.mutation.GrantsMutation; +import org.apache.polaris.persistence.nosql.metastore.mutation.MutationAttempt; +import org.apache.polaris.persistence.nosql.metastore.mutation.MutationAttemptRoot; +import org.apache.polaris.persistence.nosql.metastore.mutation.MutationResults; +import org.apache.polaris.persistence.nosql.metastore.mutation.PolicyMutation; +import org.apache.polaris.persistence.nosql.metastore.mutation.PrincipalMutations; +import org.apache.polaris.persistence.nosql.metastore.mutation.PrincipalMutations.UpdateSecrets.SecretsUpdater; +import org.apache.polaris.persistence.nosql.metastore.mutation.UpdateKeyForCatalogAndEntityType; +import org.apache.polaris.persistence.nosql.metastore.privs.GrantTriplet; +import org.apache.polaris.persistence.nosql.metastore.privs.SecurableAndGrantee; +import org.apache.polaris.persistence.nosql.metastore.privs.SecurableGranteePrivilegeTuple; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +class NoSqlMetaStore extends NonFunctionalBasePersistence { + private static final Logger LOGGER = LoggerFactory.getLogger(NoSqlMetaStore.class); + + private final Persistence persistence; + private final Privileges privileges; + private final PolarisStorageIntegrationProvider storageIntegrationProvider; + private final MemoizedIndexedAccess memoizedIndexedAccess; + private final PolarisDiagnostics diagnostics; + + NoSqlMetaStore( + Persistence persistence, + Privileges privileges, + PolarisStorageIntegrationProvider storageIntegrationProvider, + PolarisDiagnostics diagnostics) { + this.persistence = persistence; + this.privileges = privileges; + this.storageIntegrationProvider = storageIntegrationProvider; + this.memoizedIndexedAccess = newMemoizedIndexedAccess(persistence); + this.diagnostics = diagnostics; + } + + RESULT performChange( + @Nonnull PolarisEntityType entityType, + @Nonnull Class referencedObjType, + @Nonnull Class resultType, + long catalogStableId, + @Nonnull ChangeCommitter changeCommitter) { + try { + var committer = + persistence + .createCommitter( + referenceName(entityType, catalogStableId), referencedObjType, resultType) + .synchronizingLocally(); + var commitRetryable = new ChangeCommitterWrapper<>(changeCommitter, entityType); + return committer.commitRuntimeException(commitRetryable).orElseThrow(); + } finally { + memoizedIndexedAccess.invalidateIndexedAccess(catalogStableId, entityType.getCode()); + } + } + + long generateNewId() { + return persistence.generateId(); + } + + void initializeCatalogsIfNecessary() { + memoizedIndexedAccess + .indexedAccess(0, PolarisEntityType.CATALOG.getCode()) + .nameIndex() + .ifPresent( + names -> + persistence + .bucketizedBulkFetches( + Streams.stream(names).filter(Objects::nonNull).map(Map.Entry::getValue), + CatalogObj.class) + .filter(Objects::nonNull) + .forEach( + catalogObj -> { + LOGGER.debug("Initializing catalog {} if necessary", catalogObj.name()); + initializeCatalogIfNecessary(persistence, catalogObj); + })); + } + + CreateCatalogResult createCatalog( + PolarisBaseEntity catalog, List principalRoles) { + checkArgument(catalog != null && catalog.getType() == PolarisEntityType.CATALOG); + + LOGGER.debug("create catalog #{} '{}'", catalog.getId(), catalog.getName()); + + return performChange( + PolarisEntityType.CATALOG, + CatalogsObj.class, + CreateCatalogResult.class, + 0L, + ((state, ref, byName, byId) -> { + var nameKey = IndexKey.key(catalog.getName()); + var idKey = IndexKey.key(catalog.getId()); + + // check if that catalog has already been created + var existing = byName.get(nameKey); + var persistence = state.persistence(); + var catalogObj = existing != null ? persistence.fetch(existing, CatalogObj.class) : null; + + // if found, probably a retry, simply return the previously created catalog + // TODO not sure how a "retry" could happen with the same ID though (see + // PolarisMetaStoreManagerImpl.createCatalog())... + if (catalogObj != null && catalogObj.stableId() != catalog.getId()) { + // A catalog with the same name already exists (different ID) + return new ChangeResult.NoChange<>( + new CreateCatalogResult(ENTITY_ALREADY_EXISTS, null)); + } + if (catalogObj == null) { + catalogObj = + EntityObjMappings.mapToObj( + catalog, Optional.empty()) + .id(persistence.generateId()) + .build(); + state.writeOrReplace("catalog", catalogObj); + } + + initializeCatalogIfNecessary(persistence, catalogObj); + + checkState(!byId.contains(idKey), "Catalog ID %s already used", catalog.getId()); + + // 'persistStorageIntegrationIfNeeded' is a no-op in all implementations ?!?!? + // persistStorageIntegrationIfNeeded(callCtx, catalog, integration); + + var catalogAdminRoleObj = + createCatalogRoleIdempotent( + catalogObj, + persistence.generateId(), + PolarisEntityConstants.getNameOfCatalogAdminRole()); + + var catalogAdminRole = mapToEntity(catalogAdminRoleObj, catalogObj.stableId()); + + var grants = new ArrayList(); + + // grant the catalog admin role access-management on the catalog + grants.add( + new SecurableGranteePrivilegeTuple( + catalog, catalogAdminRole, PolarisPrivilege.CATALOG_MANAGE_ACCESS)); + // grant the catalog admin role metadata-management on the catalog; this one is revocable + grants.add( + new SecurableGranteePrivilegeTuple( + catalog, catalogAdminRole, PolarisPrivilege.CATALOG_MANAGE_METADATA)); + + var effRoles = + principalRoles.isEmpty() + ? List.of( + requireNonNull( + lookupEntityByName( + 0L, + 0L, + PolarisEntityType.PRINCIPAL_ROLE.getCode(), + PolarisEntityConstants.getNameOfPrincipalServiceAdminRole()))) + : principalRoles; + + for (PolarisBaseEntity effRole : effRoles) { + grants.add( + new SecurableGranteePrivilegeTuple( + catalogAdminRole, effRole, PolarisPrivilege.CATALOG_ROLE_USAGE)); + } + + persistGrantsOrRevokes(true, grants.toArray(SecurableGranteePrivilegeTuple[]::new)); + + byName.put(nameKey, objRef(catalogObj)); + byId.put(idKey, nameKey); + + if (existing == null) { + // created + return new ChangeResult.CommitChange<>( + new CreateCatalogResult(catalog, catalogAdminRole)); + } + // retry + return new ChangeResult.NoChange<>(new CreateCatalogResult(ENTITY_ALREADY_EXISTS, null)); + })); + } + + private static void initializeCatalogIfNecessary(Persistence persistence, CatalogObj catalog) { + persistence.createReferencesSilent(catalogReferenceNames(catalog.stableId())); + } + + CatalogRoleObj createCatalogRoleIdempotent( + @Nonnull CatalogObj catalogObj, long catalogRoleStableId, @Nonnull String roleName) { + return performChange( + PolarisEntityType.CATALOG_ROLE, + CatalogRolesObj.class, + CatalogRoleObj.class, + catalogObj.stableId(), + ((state, ref, byName, byId) -> { + var nameKey = IndexKey.key(roleName); + var idKey = IndexKey.key(catalogRoleStableId); + var nameRef = byName.get(nameKey); + var persistence = state.persistence(); + + if (nameRef != null) { + var role = persistence.fetch(nameRef, CatalogRoleObj.class); + requireNonNull(role); + return new ChangeResult.NoChange<>(role); + } + + checkState(!byId.contains(idKey), "Catalog role ID %s already used", catalogRoleStableId); + + var now = persistence.currentInstant(); + var roleObj = + CatalogRoleObj.builder() + .id(persistence.generateId()) + .name(roleName) + .createTimestamp(now) + .updateTimestamp(now) + .stableId(catalogRoleStableId) + .parentStableId(catalogObj.stableId()) + .build(); + + state.writeOrReplace("role", roleObj); + + byName.put(nameKey, objRef(roleObj)); + byId.put(idKey, nameKey); + + return new ChangeResult.CommitChange<>(roleObj); + })); + } + + private String logEntityInfo(PolarisEntityCore e) { + return format("%s #%d catalog:%d '%s'", e.getType(), e.getId(), e.getCatalogId(), e.getName()); + } + + private String logEntitiesInfo(List entities) { + return entities.stream() + .map(this::logEntityInfo) + .collect(Collectors.joining(", ", "(" + entities.size() + ") ", "")); + } + + EntityResult createEntity(PolarisBaseEntity entity) { + LOGGER.atDebug().addArgument(() -> logEntityInfo(entity)).log("create entity: {}"); + + return createOrUpdateEntity(CREATE, entity); + } + + EntitiesResult createEntities(List entities) { + LOGGER.atDebug().addArgument(() -> logEntitiesInfo(entities)).log("create entities: {}"); + + return createOrUpdateEntities(entities.stream(), CREATE); + } + + EntityResult updateEntity(PolarisBaseEntity entity) { + LOGGER.atDebug().addArgument(() -> logEntityInfo(entity)).log("update entity: {}"); + return createOrUpdateEntity(UPDATE, entity); + } + + EntitiesResult updateEntities(List entities) { + LOGGER + .atDebug() + .addArgument( + () -> logEntitiesInfo(entities.stream().map(EntityWithPath::getEntity).toList())) + .log("update entities: {}"); + + return createOrUpdateEntities(entities.stream().map(EntityWithPath::getEntity), UPDATE); + } + + private EntityResult createOrUpdateEntity(EntityUpdate.Operation op, PolarisBaseEntity entity) { + var mutationResults = + performEntityMutations( + updateKeyForCatalogAndEntityType(entity), List.of(new EntityUpdate(op, entity))); + return (EntityResult) mutationResults.results().getFirst(); + } + + private EntitiesResult createOrUpdateEntities( + Stream entitiesStream, EntityUpdate.Operation op) { + var byCatalogAndEntityType = + entitiesStream + .map(e -> new EntityUpdate(op, e)) + .collect(Collectors.groupingBy(u -> updateKeyForCatalogAndEntityType(u.entity()))); + + checkArgument( + byCatalogAndEntityType.size() <= 1, + "Cannot atomically create entities against multiple targets: %s", + byCatalogAndEntityType.keySet()); + + for (var concernChanges : byCatalogAndEntityType.entrySet()) { + var results = performEntityMutations(concernChanges.getKey(), concernChanges.getValue()); + var firstFailure = results.firstFailure(); + if (firstFailure.isPresent()) { + var failure = firstFailure.get(); + return new EntitiesResult(failure.getReturnStatus(), failure.getExtraInformation()); + } + + return new EntitiesResult( + Page.fromItems( + results.results().stream() + .map(EntityResult.class::cast) + .map(EntityResult::getEntity) + .collect(Collectors.toList()))); + } + + return new EntitiesResult(Page.fromItems(List.of())); + } + + DropEntityResult dropEntity( + PolarisBaseEntity entityToDrop, Map cleanupProperties, boolean cleanup) { + requireNonNull(entityToDrop); + + LOGGER.atDebug().addArgument(() -> logEntityInfo(entityToDrop)).log("drop entity: {}"); + + var results = + performEntityMutations( + updateKeyForCatalogAndEntityType(entityToDrop), + List.of(new EntityUpdate(EntityUpdate.Operation.DELETE, entityToDrop, cleanup))); + + if (cleanup && PolarisEntityType.POLICY == entityToDrop.getType()) { + cleanup = false; + } + + var result = results.results().getFirst(); + if (result.isSuccess() && cleanup) { + // If cleanup, schedule a cleanup task for the entity. + // Do this here so that the drop operation and scheduling the cleanup task are + // transactional. + // Otherwise, we'll be unable to schedule the cleanup task + var dropped = results.droppedEntities().getFirst(); + + PolarisEntity.Builder taskEntityBuilder = + new PolarisEntity.Builder() + .setId(generateNewId()) + .setCatalogId(0L) + .setName("entityCleanup_" + entityToDrop.getId()) + .setType(PolarisEntityType.TASK) + .setSubType(PolarisEntitySubType.NULL_SUBTYPE) + .setCreateTimestamp(persistence.currentTimeMillis()); + + Map properties = new HashMap<>(); + properties.put( + PolarisTaskConstants.TASK_TYPE, + String.valueOf(AsyncTaskType.ENTITY_CLEANUP_SCHEDULER.typeCode())); + properties.put("data", PolarisObjectMapperUtil.serialize(dropped)); + taskEntityBuilder.setProperties(properties); + if (cleanupProperties != null) { + taskEntityBuilder.setInternalProperties(cleanupProperties); + } + var taskEntity = taskEntityBuilder.build(); + + try { + performEntityMutations( + new UpdateKeyForCatalogAndEntityType(PolarisEntityType.TASK, 0L, false), + List.of(new EntityUpdate(CREATE, taskEntity))); + + if (entityToDrop.getType() == PolarisEntityType.POLICY) { + detachAllPolicyMappings(true, entityToDrop.getCatalogId(), entityToDrop.getId()); + } else if (PolicyMappingUtil.isValidTargetEntityType( + entityToDrop.getType(), entityToDrop.getSubType())) { + detachAllPolicyMappings(false, entityToDrop.getCatalogId(), entityToDrop.getId()); + } + + return new DropEntityResult(taskEntity.getId()); + } catch (Exception e) { + LOGGER.warn("Failed to write cleanup task entity for dropped entity", e); + } + } + + return (DropEntityResult) result; + } + + private void detachAllPolicyMappings(boolean policyNotEntity, long catalogId, long id) { + try { + var committer = + persistence + .createCommitter(POLICY_MAPPINGS_REF_NAME, PolicyMappingsObj.class, String.class) + .synchronizingLocally(); + var ignore = + committer + .commitRuntimeException( + (state, refObjSupplier) -> { + var refObj = refObjSupplier.get(); + var index = + refObj + .map( + ref -> + ref.policyMappings() + .asUpdatableIndex( + state.persistence(), POLICY_MAPPING_SERIALIZER)) + .orElseGet( + () -> newUpdatableIndex(persistence, POLICY_MAPPING_SERIALIZER)); + var builder = PolicyMappingsObj.builder(); + refObj.ifPresent(builder::from); + + var keyBy = + policyNotEntity + ? new PolicyMappingsObj.KeyByPolicy(catalogId, id, 0, 0L, 0L) + .toPolicyPartialIndexKey() + : new PolicyMappingsObj.KeyByEntity(catalogId, id, 0, 0L, 0L) + .toEntityPartialIndexKey(); + + var keys = new ArrayList(); + for (var iter = index.iterator(keyBy, keyBy, true); iter.hasNext(); ) { + var elem = iter.next(); + keys.add(elem.getKey()); + } + + if (keys.isEmpty()) { + return state.noCommit(""); + } + + for (var key : keys) { + index.remove(key); + index.remove(fromIndexKey(key).reverse().toIndexKey()); + } + + builder.policyMappings(index.toIndexed("mappings", state::writeOrReplace)); + return state.commitResult("", builder, refObj); + }) + .orElseThrow(); + } finally { + memoizedIndexedAccess.invalidateReferenceHead(POLICY_MAPPINGS_REF_NAME); + } + } + + private MutationResults performEntityMutations( + UpdateKeyForCatalogAndEntityType updateKeyForCatalogAndEntityType, + List updates) { + LOGGER + .atDebug() + .addArgument(updates.size()) + .addArgument(updateKeyForCatalogAndEntityType.entityType()) + .addArgument(updateKeyForCatalogAndEntityType.catalogId()) + .addArgument( + updateKeyForCatalogAndEntityType.catalogContent() + ? "catalog-content" + : "non-catalog-content") + .addArgument( + () -> + updates.stream() + .map( + u -> + format( + "%s: %s #%s '%s'", + u.operation(), + u.entity().getType(), + u.entity().getId(), + u.entity().getName())) + .collect(Collectors.joining("\n ", "\n ", ""))) + .log("Applying {} updates to {} entities in catalog id {} as {} : {}"); + + if (updateKeyForCatalogAndEntityType.entityType() == PolarisEntityType.ROOT) { + checkArgument(updates.size() == 1, "Cannot write multiple root entities"); + try { + var update = updates.getFirst(); + checkArgument(update.operation() == CREATE, "Cannot update or delete the root entity"); + return persistence + .createCommitter(ROOT_REF_NAME, RootObj.class, MutationResults.class) + .synchronizingLocally() + .commitRuntimeException( + (state, refObjSupplier) -> + new MutationAttemptRoot(state, refObjSupplier, update).apply()) + .orElseThrow(); + } finally { + memoizedIndexedAccess.invalidateIndexedAccess(0L, PolarisEntityType.ROOT.getCode()); + } + } + + var mutationResults = (MutationResults) null; + + if (updateKeyForCatalogAndEntityType.catalogContent()) { + try { + var committer = + persistence + .createCommitter( + format( + CATALOG_STATE_REF_NAME_PATTERN, + updateKeyForCatalogAndEntityType.catalogId()), + CatalogStateObj.class, + MutationResults.class) + .synchronizingLocally(); + var commitRetryable = + new CatalogChangeCommitterWrapper( + ((state, ref, byName, byId, changes, locations) -> + new MutationAttempt( + updateKeyForCatalogAndEntityType, + updates, + state, + byName, + byId, + changes, + locations, + memoizedIndexedAccess) + .apply())); + mutationResults = committer.commitRuntimeException(commitRetryable).orElseThrow(); + } finally { + memoizedIndexedAccess.invalidateCatalogContent( + updateKeyForCatalogAndEntityType.catalogId()); + } + } else { + mutationResults = + performChange( + updateKeyForCatalogAndEntityType.entityType(), + containerTypeForEntityType(updateKeyForCatalogAndEntityType.entityType()), + MutationResults.class, + updateKeyForCatalogAndEntityType.catalogId(), + ((state, ref, byName, byId) -> + new MutationAttempt( + updateKeyForCatalogAndEntityType, + updates, + state, + byName, + byId, + null, + null, + memoizedIndexedAccess) + .apply())); + } + + // TODO populate MutationResults.aclsToRemove and handle those, also need a maintenance + // operation to garbage-collect ACL entries for no longer existing entities. + + // TODO handle MutationResults.policyIndexKeysToRemove(), also need a maintenance + // operation to garbage-collect stale policy entries. + + return mutationResults; + } + + Optional hasOverlappingSiblings( + T entity) { + var baseLocation = entity.getBaseLocation(); + if (baseLocation == null) { + return Optional.empty(); + } + + var checkLocation = StorageLocation.of(baseLocation).withoutScheme(); + + return hasOverlappingSiblings(entity.getCatalogId(), checkLocation); + } + + Optional hasOverlappingSiblings(long catalogId, String checkLocation) { + return memoizedIndexedAccess + .catalogContent(catalogId) + .refObj() + .flatMap( + catalogStateObj -> { + var locationsIndex = + catalogStateObj + .locations() + .map(i -> i.indexForRead(persistence, ENTITY_ID_SET_SERIALIZER)) + .orElseGet(Index::empty); + var byId = + catalogStateObj.stableIdToName().indexForRead(persistence, INDEX_KEY_SERIALIZER); + var byName = + catalogStateObj.nameToObjRef().indexForRead(persistence, OBJ_REF_SERIALIZER); + + var locationIdentifier = identifierFromLocationString(checkLocation); + var locationIndexKey = locationIdentifier.toIndexKey(); + // TODO VALIDATE THE CHECKS HERE ! + var iter = locationsIndex.iterator(locationIndexKey, null, false); + if (!iter.hasNext()) { + return Optional.empty(); + } + + var elem = iter.next(); + var elemKey = elem.getKey(); + var elemIdentifier = indexKeyToIdentifier(elemKey); + if (!elemIdentifier.startsWith(locationIdentifier)) { + return Optional.empty(); + } + + return elem.getValue().entityIds().stream() + .map(IndexKey::key) + .map(byId::get) + .filter(Objects::nonNull) + .map(byName::get) + .filter(Objects::nonNull) + .map(objRef -> persistence.fetch(objRef, ContentObj.class)) + .filter(Objects::nonNull) + .map( + contentObj -> { + // Check if conflict is the parent namespace - TODO recurse?? + var conflictingBaseLocation = + contentObj.properties().get(ENTITY_BASE_LOCATION); + return conflictingBaseLocation != null + ? conflictingBaseLocation + : String.join("/", elemIdentifier.elements()); + }) + .findFirst(); + }); + } + + @Nullable + PolarisBaseEntity lookupEntity(long catalogId, long entityId, int entityTypeCode) { + if (entityTypeCode == PolarisEntityType.ROOT.getCode()) { + return (PolarisEntityConstants.getNullId() == catalogId + && entityId == PolarisEntityConstants.getRootEntityId()) + ? lookupRoot().orElseThrow() + : null; + } + if (entityTypeCode == PolarisEntityType.NULL_TYPE.getCode()) { + return null; + } + if (entityTypeCode == PolarisEntityType.CATALOG.getCode()) { + catalogId = 0L; + } + + var access = memoizedIndexedAccess.indexedAccess(catalogId, entityTypeCode); + var resolved = access.byId(entityId); + + LOGGER.debug( + "lookupEntity result: entityTypeCode: {}, catalogId: {}, entityId: {} : {}", + entityTypeCode, + catalogId, + entityId, + resolved); + + return resolved + .flatMap(objBase -> filterIsEntityType(objBase, entityTypeCode)) + .map(objBase -> mapToEntity(objBase, access.catalogStableId())) + .orElse(null); + } + + PolarisBaseEntity lookupEntityByName( + long catalogId, long parentId, int entityTypeCode, String name) { + if (entityTypeCode == PolarisEntityType.ROOT.getCode()) { + return (PolarisEntityConstants.getNullId() == catalogId + && parentId == PolarisEntityConstants.getRootEntityId() + && PolarisEntityConstants.getRootContainerName().equals(name)) + ? lookupRoot().orElseThrow() + : null; + } + if (entityTypeCode == PolarisEntityType.NULL_TYPE.getCode()) { + return null; + } + if (entityTypeCode == PolarisEntityType.CATALOG.getCode()) { + catalogId = 0L; + } + + var rootAccess = parentId == catalogId; + var access = memoizedIndexedAccess.indexedAccess(catalogId, entityTypeCode); + var resolved = + rootAccess ? access.byNameOnRoot(name) : access.byParentIdAndName(parentId, name); + + LOGGER.debug( + "lookupEntityByName result : entityTypeCode: {}, catalogId: {}, parentId: {}, name: {} : {}", + entityTypeCode, + catalogId, + parentId, + name, + resolved); + + return resolved + .flatMap(objBase -> filterIsEntityType(objBase, entityTypeCode)) + .map(objBase -> mapToEntity(objBase, access.catalogStableId())) + .orElse(null); + } + + Optional lookupRoot() { + return memoizedIndexedAccess + .indexedAccess(0L, PolarisEntityType.ROOT.getCode()) + .byId(0L) + .map(root -> EntityObjMappings.mapToEntity(root, 0L)); + } + + Page fetchEntitiesAsPage( + long catalogStableId, + long parentId, + PolarisEntityType entityType, + PolarisEntitySubType entitySubType, + PageToken pageToken, + Function mapper, + Predicate filter, + Function transformer) { + + LOGGER.debug( + "fetchEntitiesAsPage, catalogId: {}, parentId: {}, entityType: {}, pageToken: {}", + catalogStableId, + parentId, + entityType, + pageToken); + + if (entityType == PolarisEntityType.NULL_TYPE) { + return Page.fromItems(List.of()); + } + if (entityType == PolarisEntityType.CATALOG) { + catalogStableId = 0L; + } + + var paginationToken = pageToken.valueAs(NoSqlPaginationToken.class); + var pageTokenOffset = paginationToken.map(NoSqlPaginationToken::key); + + var catalogContent = isCatalogContent(entityType); + var access = + paginationToken.isPresent() + ? memoizedIndexedAccess.indexedAccessDirect( + paginationToken.orElseThrow().containerObjRef()) + : catalogContent + ? memoizedIndexedAccess.catalogContent(catalogStableId) + : memoizedIndexedAccess.indexedAccess(catalogStableId, entityType.getCode()); + var nameIndex = access.nameIndex().orElse(null); + + if (nameIndex == null) { + return Page.fromItems(List.of()); + } + + var objRefs = Stream.>empty(); + if (catalogStableId != 0L) { + if (parentId == 0L || parentId == catalogStableId) { + // list on catalog root + var lower = pageTokenOffset.orElse(null); + objRefs = catalogRootEntriesStream(nameIndex, lower); + } else { + // list on namespace + var prefixKeyOptional = access.nameKeyById(parentId); + if (prefixKeyOptional.isPresent()) { + var prefixKey = prefixKeyOptional.get(); + var offsetKey = + pageTokenOffset.filter(pto -> pto.compareTo(prefixKey) >= 0).orElse(prefixKey); + var prefix = indexKeyToIdentifier(prefixKey); + objRefs = catalogNamespaceEntriesStream(nameIndex, offsetKey, prefix); + } + } + } else { + objRefs = Streams.stream(nameIndex.iterator(pageTokenOffset.orElse(null), null, false)); + } + + if (LOGGER.isDebugEnabled()) { + objRefs = + objRefs.peek( + o -> + LOGGER.debug( + " listEntitiesStream (before type filter): {} : {}", + o.getKey(), + o.getValue())); + } + + var filterType = objTypeForPolarisTypeForFiltering(entityType, entitySubType); + objRefs = + objRefs.filter( + o -> + filterType.isAssignableFrom( + ObjTypes.objTypeById(o.getValue().type()).targetClass())); + + return listEntitiesBuildPage(access, pageToken, mapper, filter, transformer, objRefs); + } + + Stream listChildren(Index nameIndex, ContentIdentifier parent) { + var objRefs = + (parent.isEmpty() + ? catalogRootEntriesStream(nameIndex, null) + : catalogNamespaceEntriesStream(nameIndex, parent.toIndexKey(), parent)) + .map(Map.Entry::getValue); + + return persistence.bucketizedBulkFetches(objRefs, ContentObj.class); + } + + private static Stream> catalogNamespaceEntriesStream( + Index nameIndex, IndexKey offsetKey, ContentIdentifier prefix) { + var prefixElems = prefix.elements(); + var directChildLevel = prefixElems.size() + 1; + return Streams.stream(nameIndex.iterator(offsetKey, null, false)) + .takeWhile( + e -> { + var ident = indexKeyToIdentifier(requireNonNull(e).getKey()); + var identElems = ident.elements(); + if (identElems.size() < prefixElems.size() + 1) { + return ident.equals(prefix); + } + return identElems.subList(0, prefixElems.size()).equals(prefixElems); + }) + .filter( + e -> { + var ident = indexKeyToIdentifier(requireNonNull(e).getKey()); + return ident.elements().size() == directChildLevel; + }); + } + + private static Stream> catalogRootEntriesStream( + Index nameIndex, IndexKey lower) { + return Streams.stream(nameIndex.iterator(lower, null, false)) + .filter(Objects::nonNull) + .filter( + e -> { + var ident = indexKeyToIdentifier(e.getKey()); + return ident.elements().size() == 1; + }); + } + + /** + * Number of {@link ObjBase objects} to {@link Persistence#fetchMany(Class, ObjRef...) + * bulk-fetch}. + */ + public static final int FETCH_PAGE_SIZE = 25; + + private Page listEntitiesBuildPage( + IndexedContainerAccess access, + PageToken pageToken, + Function mapper, + Predicate filter, + Function transformer, + Stream> objRefs) { + var limit = pageToken.pageSize().orElse(Integer.MAX_VALUE); + var nextToken = (NoSqlPaginationToken) null; + var result = new ArrayList(); + + var fetchBuffer = new ArrayList>(); + + for (var objRefIter = objRefs.iterator(); objRefIter.hasNext(); ) { + var keyAndRef = objRefIter.next(); + fetchBuffer.add(keyAndRef); + if (fetchBuffer.size() == FETCH_PAGE_SIZE) { + nextToken = + listEntitiesBuildPagePart( + access, fetchBuffer, mapper, filter, transformer, result, limit); + fetchBuffer.clear(); + if (nextToken != null || result.size() == limit) { + break; + } + } + } + if (!fetchBuffer.isEmpty()) { + nextToken = + listEntitiesBuildPagePart( + access, fetchBuffer, mapper, filter, transformer, result, limit); + } + + return Page.page(pageToken, result, nextToken); + } + + @Nullable + private NoSqlPaginationToken listEntitiesBuildPagePart( + IndexedContainerAccess access, + List> fetchBuffer, + Function mapper, + Predicate filter, + Function transformer, + List result, + int limit) { + var objs = + persistence.fetchMany( + ObjBase.class, fetchBuffer.stream().map(Map.Entry::getValue).toArray(ObjRef[]::new)); + for (int i = 0; i < fetchBuffer.size(); i++) { + var obj = objs[i]; + + if (obj == null) { + continue; + } + var intermediate = mapper.apply(obj); + if (intermediate == null || !filter.test(intermediate)) { + continue; + } + var transformed = transformer.apply(intermediate); + if (transformed == null) { + continue; + } + + if (result.size() == limit) { + return NoSqlPaginationToken.paginationToken( + ObjRef.objRef(access.refObj().orElseThrow()), fetchBuffer.get(i).getKey()); + } + + result.add(transformed); + } + return null; + } + + PolicyAttachmentResult attachDetachPolicyOnEntity( + long policyCatalogId, + long policyId, + @Nonnull PolicyType policyType, + long targetCatalogId, + long targetId, + boolean doAttach, + @Nonnull Map parameters) { + return new PolicyMutation( + persistence, + memoizedIndexedAccess, + policyCatalogId, + policyId, + requireNonNull(policyType), + targetCatalogId, + targetId, + doAttach, + parameters) + .apply(); + } + + // TODO remove entirely? + @SuppressWarnings("SameParameterValue") + LoadPolicyMappingsResult loadEntitiesOnPolicy( + @Nullable PolarisEntityType entityType, + long policyCatalogId, + long policyId, + Optional policyType) { + if (entityType != null && !isCatalogContent(entityType)) { + return new LoadPolicyMappingsResult(ENTITY_NOT_FOUND, null); + } + + return memoizedIndexedAccess + .referenceHead(POLICY_MAPPINGS_REF_NAME, PolicyMappingsObj.class) + .map( + policyMappingsObj -> { + var index = + policyMappingsObj + .policyMappings() + .indexForRead(persistence, POLICY_MAPPING_SERIALIZER); + + // (Partial) index-key for the lookup + var keyByPolicyTemplate = + new PolicyMappingsObj.KeyByPolicy( + policyCatalogId, + policyId, + policyType.map(PolicyType::getCode).orElse(0), + 0L, + 0L); + + // Construct the prefix-key, depending on whether to look for all attached policies or + // attached policies having the given policy-type + var prefixKey = + policyType.isPresent() + ? keyByPolicyTemplate.toPolicyWithTypePartialIndexKey() + : keyByPolicyTemplate.toPolicyPartialIndexKey(); + + Class expectedKeyType = + PolicyMappingsObj.KeyByPolicy.class; + + return loadPolicyMappings(index, prefixKey, expectedKeyType); + }) + .orElse(new LoadPolicyMappingsResult(List.of(), List.of())); + } + + LoadPolicyMappingsResult loadPoliciesOnEntity( + @Nullable PolarisEntityType entityType, + long catalogId, + long id, + Optional policyType) { + if (entityType != null + && entityType != PolarisEntityType.CATALOG + && !isCatalogContent(entityType)) { + return new LoadPolicyMappingsResult(ENTITY_NOT_FOUND, null); + } + + return memoizedIndexedAccess + .referenceHead(POLICY_MAPPINGS_REF_NAME, PolicyMappingsObj.class) + .map( + policyMappingsObj -> { + var index = + policyMappingsObj + .policyMappings() + .indexForRead(persistence, POLICY_MAPPING_SERIALIZER); + + // (Partial) index-key for the lookup + var keyByEntityTemplate = + new PolicyMappingsObj.KeyByEntity( + catalogId, id, policyType.map(PolicyType::getCode).orElse(0), 0L, 0L); + + // Construct the prefix-key, depending on whether to look for all attached policies or + // attached policies having the given policy-type + var prefixKey = + policyType.isPresent() + ? keyByEntityTemplate.toPolicyTypePartialIndexKey() + : keyByEntityTemplate.toEntityPartialIndexKey(); + + Class expectedKeyType = + PolicyMappingsObj.KeyByEntity.class; + + return loadPolicyMappings(index, prefixKey, expectedKeyType); + }) + .orElse(new LoadPolicyMappingsResult(List.of(), List.of())); + } + + private LoadPolicyMappingsResult loadPolicyMappings( + Index index, + IndexKey prefixKey, + Class expectedKeyType) { + var mappingRecords = new ArrayList(); + var policyEntities = new ArrayList(); + var seenPolicies = new HashSet(); + for (var iter = index.iterator(prefixKey, prefixKey, false); iter.hasNext(); ) { + var elem = iter.next(); + var key = fromIndexKey(elem.getKey()); + if (expectedKeyType.isInstance(key)) { + if (seenPolicies.add(key.policyId())) { + memoizedIndexedAccess + .catalogContent(key.policyCatalogId()) + .byId(key.policyId()) + .flatMap(objBase -> filterIsEntityType(objBase, PolarisEntityType.POLICY)) + .map(obj -> mapToEntity(obj, key.policyCatalogId())) + .ifPresent(policyEntities::add); + } + mappingRecords.add(key.toMappingRecord(elem.getValue())); + } else { + // `key` is not what we're looking for. + // This should actually never happen due to the prefix-key. + break; + } + } + + return new LoadPolicyMappingsResult(mappingRecords, policyEntities); + } + + // grants + + @FunctionalInterface + interface AclEntryHandler { + void handle(SecurableAndGrantee securableAndGrantee, PrivilegeSet granted); + } + + List allGrantRecords(PolarisBaseEntity entity) { + var catalogId = entity.getCatalogId(); + var aclName = GrantTriplet.forEntity(entity).toRoleName(); + + LOGGER.debug("allGrantRecords for {}", aclName); + + var grantRecords = + memoizedIndexedAccess + .grantsIndex() + .map(entries -> collectGrantRecords(catalogId, aclName, entries)) + .orElseGet(List::of); + + LOGGER + .atTrace() + .addArgument(grantRecords.size()) + .addArgument( + () -> + grantRecords.stream() + .map(PolarisGrantRecord::toString) + .collect(Collectors.joining("\n ", "\n ", ""))) + .log("Returning {} grant records: {}"); + return grantRecords; + } + + LoadGrantsResult loadGrants(long catalogId, long id, int entityTypeCode, boolean onSecurable) { + LOGGER.debug( + "loadGrants on {} for catalog:{}, id:{}, entityType:{}({})", + onSecurable ? "securable" : "grantee", + catalogId, + id, + PolarisEntityType.fromCode(entityTypeCode), + entityTypeCode); + var aclName = new GrantTriplet(true, catalogId, id, entityTypeCode).toRoleName(); + + var collector = new GrantRecordsCollector(); + var entities = new ArrayList(); + var ids = new HashSet(); + + collectGrantRecords( + catalogId, + aclName, + ((securableAndGrantee, granted) -> { + var targetCatalogId = + onSecurable + ? securableAndGrantee.granteeCatalogId() + : securableAndGrantee.securableCatalogId(); + var targetId = + onSecurable ? securableAndGrantee.granteeId() : securableAndGrantee.securableId(); + var targetTypeCode = + onSecurable + ? securableAndGrantee.granteeTypeCode() + : securableAndGrantee.securableTypeCode(); + + var indexedAccess = memoizedIndexedAccess.indexedAccess(targetCatalogId, targetTypeCode); + var entityOptional = + indexedAccess.byId(targetId).map(o -> mapToEntity(o, targetCatalogId)); + + if (entityOptional.isPresent()) { + PolarisBaseEntity entity = entityOptional.get(); + LOGGER.trace( + " Adding entity to load-grants-result: catalog:{}, id:{}, type:{}", + entity.getCatalogId(), + entity.getId(), + entity.getType()); + collector.handle(securableAndGrantee, granted); + if (ids.add(targetId)) { + entities.add(entity); + } + } else { + LOGGER.trace(" Not returning stale entity reference"); + } + })); + + LOGGER.trace( + "Returning {} grant records for loadGrants for catalog:{}, id:{}, entityType:{}({})", + collector.grantRecords.size(), + catalogId, + id, + PolarisEntityType.fromCode(entityTypeCode), + entityTypeCode); + + return new LoadGrantsResult(1, collector.grantRecords, entities); + } + + private List collectGrantRecords( + long catalogId, String aclName, Index securablesIndex) { + + var collector = new GrantRecordsCollector(); + collectGrantRecords( + catalogId, + aclName, + (securableAndGrantee, granted) -> { + var indexedAccess = + memoizedIndexedAccess.indexedAccess( + securableAndGrantee.securableCatalogId(), + securableAndGrantee.securableTypeCode()); + var existing = indexedAccess.nameKeyById(securableAndGrantee.securableId()); + if (existing.isPresent()) { + collector.handle(securableAndGrantee, granted); + } + }, + securablesIndex); + return collector.grantRecords; + } + + private void collectGrantRecords( + long catalogStableId, String aclName, AclEntryHandler aclEntryConsumer) { + LOGGER.debug("Checking ACL '{}'", aclName); + + var securablesIndex = memoizedIndexedAccess.grantsIndex(); + if (securablesIndex.isPresent()) { + collectGrantRecords(catalogStableId, aclName, aclEntryConsumer, securablesIndex.get()); + } else { + LOGGER.trace("ACL {} does not exist", aclName); + } + } + + private void collectGrantRecords( + long catalogStableId, + String aclName, + AclEntryHandler aclEntryConsumer, + Index securablesIndex) { + var securableKey = IndexKey.key(aclName); + + LOGGER.trace("Processing existing ACL {}", aclName); + Optional.ofNullable(securablesIndex.get(securableKey)) + .flatMap(aclObjRef -> Optional.ofNullable(persistence.fetch(aclObjRef, AclObj.class))) + .ifPresent( + aclObj -> { + var acl = aclObj.acl(); + + acl.forEach( + (role, entry) -> { + var triplet = GrantTriplet.fromRoleName(role); + LOGGER + .atTrace() + .setMessage(" ACL has securable {} ({}) with privileges {}") + .addArgument(role) + .addArgument(PolarisEntityType.fromCode(triplet.typeCode())) + .addArgument( + () -> + entry.granted().stream() + .map(Privilege::name) + .collect(Collectors.joining(", "))) + .log(); + + var securableAndGrantee = + SecurableAndGrantee.forTriplet(catalogStableId, aclObj, triplet); + aclEntryConsumer.handle(securableAndGrantee, entry.granted()); + }); + }); + } + + static class GrantRecordsCollector implements AclEntryHandler { + final List grantRecords = new ArrayList<>(); + + @Override + public void handle(SecurableAndGrantee securableAndGrantee, PrivilegeSet granted) { + for (var privilege : granted) { + var privilegeCode = PolarisPrivilege.valueOf(privilege.name()).getCode(); + var record = securableAndGrantee.grantRecordForPrivilege(privilegeCode); + LOGGER.trace( + " Yielding grant record: securable: catalog:{} id:{} - grantee: catalog:{} id:{} - privilege: {}", + record.getSecurableCatalogId(), + record.getSecurableId(), + record.getGranteeCatalogId(), + record.getGranteeId(), + record.getPrivilegeCode()); + grantRecords.add(record); + } + } + } + + boolean persistGrantsOrRevokes(boolean doGrant, SecurableGranteePrivilegeTuple... grants) { + LOGGER.debug("Persisting {} for '{}'", doGrant ? "grants" : "revokes", Arrays.asList(grants)); + + return new GrantsMutation(persistence, memoizedIndexedAccess, privileges, doGrant, grants) + .apply(); + } + + + PolarisStorageIntegration loadPolarisStorageIntegration( + @Nonnull PolarisBaseEntity entity) { + var storageConfig = BaseMetaStoreManager.extractStorageConfiguration(diagnostics, entity); + return storageIntegrationProvider.getStorageIntegrationForConfig(storageConfig); + } + + CreatePrincipalResult createPrincipal( + PolarisBaseEntity principal, RootCredentialsSet rootCredentialsSet) { + LOGGER.atDebug().addArgument(() -> logEntityInfo(principal)).log("createPrincipal {}"); + + return new PrincipalMutations.CreatePrincipal( + persistence, memoizedIndexedAccess, principal, rootCredentialsSet) + .apply(); + } + + static class PrincipalNotFoundException extends RuntimeException {} + + R updatePrincipalSecrets( + Class resultType, String logInfo, long principalId, SecretsUpdater updater) { + LOGGER.debug("updatePrincipalSecrets ({}), principalId: {}", logInfo, principalId); + + return new PrincipalMutations.UpdateSecrets<>( + persistence, memoizedIndexedAccess, resultType, principalId, updater) + .apply(); + } + + PolarisPrincipalSecrets loadPrincipalSecrets(@Nonnull String clientId) { + LOGGER.debug("loadPrincipalSecrets clientId: {}", clientId); + + var key = IndexKey.key(clientId); + + return memoizedIndexedAccess + .indexedAccess(0L, PolarisEntityType.PRINCIPAL.getCode()) + .refObj() + .map(PrincipalsObj.class::cast) + .map(PrincipalsObj::byClientId) + .map(c -> c.indexForRead(persistence, OBJ_REF_SERIALIZER)) + .map(i -> i.get(key)) + .map(objRef -> persistence.fetch(objRef, PrincipalObj.class)) + .map(EntityObjMappings::principalObjToPolarisPrincipalSecrets) + .orElse(null); + } +} diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NoSqlMetaStoreManager.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NoSqlMetaStoreManager.java new file mode 100644 index 0000000000..a10d960634 --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NoSqlMetaStoreManager.java @@ -0,0 +1,831 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.nosql.metastore; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static org.apache.polaris.core.persistence.dao.entity.BaseResult.ReturnStatus.ENTITY_NOT_FOUND; +import static org.apache.polaris.core.persistence.dao.entity.BaseResult.ReturnStatus.GRANT_NOT_FOUND; +import static org.apache.polaris.core.persistence.dao.entity.BaseResult.ReturnStatus.SUBSCOPE_CREDS_ERROR; +import static org.apache.polaris.persistence.nosql.coretypes.mapping.EntityObjMappings.mapToEntity; +import static org.apache.polaris.persistence.nosql.coretypes.mapping.EntityObjMappings.mapToEntityNameLookupRecord; +import static org.apache.polaris.persistence.nosql.coretypes.mapping.EntityObjMappings.principalObjToPolarisPrincipalSecrets; + +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.time.Clock; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.function.Supplier; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.entity.LocationBasedEntity; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisEntity; +import org.apache.polaris.core.entity.PolarisEntityConstants; +import org.apache.polaris.core.entity.PolarisEntityCore; +import org.apache.polaris.core.entity.PolarisEntityId; +import org.apache.polaris.core.entity.PolarisEntitySubType; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.entity.PolarisEvent; +import org.apache.polaris.core.entity.PolarisGrantRecord; +import org.apache.polaris.core.entity.PolarisPrincipalSecrets; +import org.apache.polaris.core.entity.PolarisPrivilege; +import org.apache.polaris.core.entity.PolarisTaskConstants; +import org.apache.polaris.core.entity.PrincipalEntity; +import org.apache.polaris.core.persistence.PolarisMetaStoreManager; +import org.apache.polaris.core.persistence.PolarisObjectMapperUtil; +import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; +import org.apache.polaris.core.persistence.dao.entity.BaseResult; +import org.apache.polaris.core.persistence.dao.entity.ChangeTrackingResult; +import org.apache.polaris.core.persistence.dao.entity.CreateCatalogResult; +import org.apache.polaris.core.persistence.dao.entity.CreatePrincipalResult; +import org.apache.polaris.core.persistence.dao.entity.DropEntityResult; +import org.apache.polaris.core.persistence.dao.entity.EntitiesResult; +import org.apache.polaris.core.persistence.dao.entity.EntityResult; +import org.apache.polaris.core.persistence.dao.entity.EntityWithPath; +import org.apache.polaris.core.persistence.dao.entity.GenerateEntityIdResult; +import org.apache.polaris.core.persistence.dao.entity.ListEntitiesResult; +import org.apache.polaris.core.persistence.dao.entity.LoadGrantsResult; +import org.apache.polaris.core.persistence.dao.entity.LoadPolicyMappingsResult; +import org.apache.polaris.core.persistence.dao.entity.PolicyAttachmentResult; +import org.apache.polaris.core.persistence.dao.entity.PrincipalSecretsResult; +import org.apache.polaris.core.persistence.dao.entity.PrivilegeResult; +import org.apache.polaris.core.persistence.dao.entity.ResolvedEntitiesResult; +import org.apache.polaris.core.persistence.dao.entity.ResolvedEntityResult; +import org.apache.polaris.core.persistence.dao.entity.ScopedCredentialsResult; +import org.apache.polaris.core.persistence.pagination.Page; +import org.apache.polaris.core.persistence.pagination.PageToken; +import org.apache.polaris.core.policy.PolicyEntity; +import org.apache.polaris.core.policy.PolicyType; +import org.apache.polaris.persistence.nosql.metastore.privs.SecurableGranteePrivilegeTuple; + +record NoSqlMetaStoreManager( + Supplier purgeRealm, + RootCredentialsSet rootCredentialsSet, + Supplier metaStoreSupplier, + Clock clock) + implements PolarisMetaStoreManager { + + NoSqlMetaStore ms() { + return metaStoreSupplier.get(); + } + + NoSqlMetaStore ms(PolarisCallContext callContext) { + var existing = callContext.getMetaStore(); + checkArgument(existing instanceof NoSqlMetaStore, "No meta store found in call context"); + return (NoSqlMetaStore) existing; + } + + // Realms + + @Nonnull + @Override + public BaseResult bootstrapPolarisService(@Nonnull PolarisCallContext callCtx) { + bootstrapPolarisServiceInternal(ms(callCtx)); + return new BaseResult(BaseResult.ReturnStatus.SUCCESS); + } + + Optional bootstrapPolarisServiceInternal(NoSqlMetaStore ms) { + // This function is idempotent, already existing entities will not be created again. + + // Create the root-container, if not already present + var rootContainer = + ms.lookupRoot() + .orElseGet( + () -> { + var newRoot = + new PolarisBaseEntity( + PolarisEntityConstants.getNullId(), + PolarisEntityConstants.getRootEntityId(), + PolarisEntityType.ROOT, + PolarisEntitySubType.NULL_SUBTYPE, + PolarisEntityConstants.getRootEntityId(), + PolarisEntityConstants.getRootContainerName()); + ms.createEntity(newRoot); + return newRoot; + }); + + // Create the root-principal, if not already present + var rootPrincipal = + ms.lookupEntityByName( + 0L, + 0L, + PolarisEntityType.PRINCIPAL.getCode(), + PolarisEntityConstants.getRootPrincipalName()); + var createPrincipalResult = Optional.empty(); + if (rootPrincipal == null) { + var rootPrincipalId = ms.generateNewId(); + rootPrincipal = + new PolarisBaseEntity( + PolarisEntityConstants.getNullId(), + rootPrincipalId, + PolarisEntityType.PRINCIPAL, + PolarisEntitySubType.NULL_SUBTYPE, + PolarisEntityConstants.getRootEntityId(), + PolarisEntityConstants.getRootPrincipalName()); + + createPrincipalResult = Optional.of(ms.createPrincipal(rootPrincipal, rootCredentialsSet)); + } + + // Create the service-admin principal-role, if not already present + var serviceAdminPrincipalRole = + ms.lookupEntityByName( + 0L, + 0L, + PolarisEntityType.PRINCIPAL_ROLE.getCode(), + PolarisEntityConstants.getNameOfPrincipalServiceAdminRole()); + if (serviceAdminPrincipalRole == null) { + // now create the account admin principal role + var serviceAdminPrincipalRoleId = ms.generateNewId(); + serviceAdminPrincipalRole = + new PolarisBaseEntity( + PolarisEntityConstants.getNullId(), + serviceAdminPrincipalRoleId, + PolarisEntityType.PRINCIPAL_ROLE, + PolarisEntitySubType.NULL_SUBTYPE, + PolarisEntityConstants.getRootEntityId(), + PolarisEntityConstants.getNameOfPrincipalServiceAdminRole()); + ms.createEntity(serviceAdminPrincipalRole); + } + + // Persisting already existing grants is an idempotent operation + ms.persistGrantsOrRevokes( + true, + // we also need to grant usage on the account-admin principal to the principal + new SecurableGranteePrivilegeTuple( + serviceAdminPrincipalRole, rootPrincipal, PolarisPrivilege.PRINCIPAL_ROLE_USAGE), + // grant SERVICE_MANAGE_ACCESS on the rootContainer to the serviceAdminPrincipalRole + new SecurableGranteePrivilegeTuple( + rootContainer, serviceAdminPrincipalRole, PolarisPrivilege.SERVICE_MANAGE_ACCESS)); + + return createPrincipalResult; + } + + @Nonnull + @Override + public BaseResult purge(@Nonnull PolarisCallContext callCtx) { + return purgeRealm.get(); + } + + // Catalog + + @Nonnull + @Override + public CreateCatalogResult createCatalog( + @Nonnull PolarisCallContext callCtx, + @Nonnull PolarisBaseEntity catalog, + @Nonnull List principalRoles) { + var prRoles = principalRoles.stream().map(PolarisBaseEntity.class::cast).toList(); + + return ms(callCtx).createCatalog(catalog, prRoles); + } + + // Generic entities + + @Nonnull + @Override + public EntityResult createEntityIfNotExists( + @Nonnull PolarisCallContext callCtx, + @Nullable List catalogPath, + @Nonnull PolarisBaseEntity entity) { + return ms(callCtx).createEntity(entity); + } + + @SuppressWarnings("unchecked") + @Nonnull + @Override + public EntitiesResult createEntitiesIfNotExist( + @Nonnull PolarisCallContext callCtx, + @Nullable List catalogPath, + @Nonnull List entities) { + return ms(callCtx).createEntities((List) entities); + } + + @Nonnull + @Override + public EntityResult updateEntityPropertiesIfNotChanged( + @Nonnull PolarisCallContext callCtx, + @Nullable List catalogPath, + @Nonnull PolarisBaseEntity entity) { + return ms(callCtx).updateEntity(entity); + } + + @Nonnull + @Override + public EntitiesResult updateEntitiesPropertiesIfNotChanged( + @Nonnull PolarisCallContext callCtx, @Nonnull List entities) { + return ms(callCtx).updateEntities(entities); + } + + @Nonnull + @Override + public EntityResult renameEntity( + @Nonnull PolarisCallContext callCtx, + @Nullable List catalogPath, + @Nonnull PolarisBaseEntity entityToRename, + @Nullable List newCatalogPath, + @Nonnull PolarisEntity renamedEntity) { + if (newCatalogPath != null && !newCatalogPath.isEmpty()) { + var last = newCatalogPath.getLast(); + // At least BasePolarisMetaStoreManagerTest comes with the wrong parentId in renamedEntity + if (renamedEntity.getParentId() != last.getId()) { + renamedEntity = new PolarisEntity.Builder(renamedEntity).setParentId(last.getId()).build(); + } + } + return ms(callCtx).updateEntity(renamedEntity); + } + + @Nonnull + @Override + public DropEntityResult dropEntityIfExists( + @Nonnull PolarisCallContext callCtx, + @Nullable List catalogPath, + @Nonnull PolarisBaseEntity entityToDrop, + @Nullable Map cleanupProperties, + boolean cleanup) { + return ms(callCtx).dropEntity(entityToDrop, cleanupProperties, cleanup); + } + + @Nonnull + @Override + public EntityResult readEntityByName( + @Nonnull PolarisCallContext callCtx, + @Nullable List catalogPath, + @Nonnull PolarisEntityType entityType, + @Nonnull PolarisEntitySubType entitySubType, + @Nonnull String name) { + return readEntityByName(ms(callCtx), catalogPath, entityType, name); + } + + EntityResult readEntityByName( + NoSqlMetaStore ms, + List catalogPath, + PolarisEntityType entityType, + String name) { + var catalogId = 0L; + var parentId = 0L; + if (catalogPath != null && !catalogPath.isEmpty()) { + catalogId = catalogPath.getFirst().getId(); + parentId = catalogPath.getLast().getId(); + } + var entity = ms.lookupEntityByName(catalogId, parentId, entityType.getCode(), name); + return entity != null ? new EntityResult(entity) : new EntityResult(ENTITY_NOT_FOUND, null); + } + + @Nonnull + @Override + public Page listFullEntities( + @Nonnull PolarisCallContext callCtx, + @Nullable List catalogPath, + @Nonnull PolarisEntityType entityType, + @Nonnull PolarisEntitySubType entitySubType, + @Nonnull PageToken pageToken) { + var catalogStableId = + (catalogPath != null && !catalogPath.isEmpty()) ? catalogPath.getFirst().getId() : 0L; + + var parentId = + (catalogPath != null && catalogPath.size() > 1) ? catalogPath.getLast().getId() : 0L; + + return ms(callCtx) + .fetchEntitiesAsPage( + catalogStableId, + parentId, + entityType, + entitySubType, + pageToken, + objBase -> mapToEntity(objBase, catalogStableId), + entity -> true, + Function.identity()); + } + + @Nonnull + @Override + public ListEntitiesResult listEntities( + @Nonnull PolarisCallContext callCtx, + @Nullable List catalogPath, + @Nonnull PolarisEntityType entityType, + @Nonnull PolarisEntitySubType entitySubType, + @Nonnull PageToken pageToken) { + var catalogStableId = + (catalogPath != null && !catalogPath.isEmpty()) ? catalogPath.getFirst().getId() : 0L; + + var parentId = + (catalogPath != null && catalogPath.size() > 1) ? catalogPath.getLast().getId() : 0L; + + var page = + ms(callCtx) + .fetchEntitiesAsPage( + catalogStableId, + parentId, + entityType, + entitySubType, + pageToken, + objBase -> mapToEntityNameLookupRecord(objBase, catalogStableId), + entity -> true, + Function.identity()); + + return new ListEntitiesResult(page); + } + + @Nonnull + @Override + public GenerateEntityIdResult generateNewEntityId(@Nonnull PolarisCallContext callCtx) { + return new GenerateEntityIdResult(metaStoreSupplier.get().generateNewId()); + } + + @Nonnull + @Override + public ResolvedEntitiesResult loadResolvedEntities( + @Nonnull PolarisCallContext callCtx, + @Nonnull PolarisEntityType entityType, + @Nonnull List entityIds) { + throw new UnsupportedOperationException("Not implemented"); + } + + @Nonnull + @Override + public ResolvedEntityResult loadResolvedEntityById( + @Nonnull PolarisCallContext callCtx, + long entityCatalogId, + long entityId, + PolarisEntityType entityType) { + var ms = ms(callCtx); + + // load that entity + PolarisBaseEntity entity = ms.lookupEntity(entityCatalogId, entityId, entityType.getCode()); + + // if entity not found, return null + if (entity == null) { + return new ResolvedEntityResult(ENTITY_NOT_FOUND, null); + } + + // load the grant records + var grantRecords = ms.allGrantRecords(entity); + + // return the result + return new ResolvedEntityResult(entity, entity.getGrantRecordsVersion(), grantRecords); + } + + @Nonnull + @Override + public ResolvedEntityResult loadResolvedEntityByName( + @Nonnull PolarisCallContext callCtx, + long entityCatalogId, + long parentId, + @Nonnull PolarisEntityType entityType, + @Nonnull String entityName) { + var ms = ms(callCtx); + + // load that entity + var entity = ms.lookupEntityByName(entityCatalogId, parentId, entityType.getCode(), entityName); + + // null if entity not found + if (entity == null) { + return new ResolvedEntityResult(ENTITY_NOT_FOUND, null); + } + + // load the grant records + var grantRecords = ms.allGrantRecords(entity); + + // return the result + return new ResolvedEntityResult(entity, entity.getGrantRecordsVersion(), grantRecords); + } + + @Nonnull + @Override + public ResolvedEntityResult refreshResolvedEntity( + @Nonnull PolarisCallContext callCtx, + int entityVersion, + int entityGrantRecordsVersion, + @Nonnull PolarisEntityType entityType, + long entityCatalogId, + long entityId) { + return loadResolvedEntityById(callCtx, entityCatalogId, entityId, entityType); + } + + // Principals & Polaris GrantManager + + @Nonnull + @Override + public PrivilegeResult grantUsageOnRoleToGrantee( + @Nonnull PolarisCallContext callCtx, + @Nullable PolarisEntityCore catalog, + @Nonnull PolarisEntityCore role, + @Nonnull PolarisEntityCore grantee) { + var privilege = + (grantee.getType() == PolarisEntityType.PRINCIPAL_ROLE) + ? PolarisPrivilege.CATALOG_ROLE_USAGE + : PolarisPrivilege.PRINCIPAL_ROLE_USAGE; + + return grantPrivilegeOnSecurableToRole(callCtx, grantee, null, role, privilege); + } + + @Nonnull + @Override + public PrivilegeResult revokeUsageOnRoleFromGrantee( + @Nonnull PolarisCallContext callCtx, + @Nullable PolarisEntityCore catalog, + @Nonnull PolarisEntityCore role, + @Nonnull PolarisEntityCore grantee) { + var privilege = + (grantee.getType() == PolarisEntityType.PRINCIPAL_ROLE) + ? PolarisPrivilege.CATALOG_ROLE_USAGE + : PolarisPrivilege.PRINCIPAL_ROLE_USAGE; + + return revokePrivilegeOnSecurableFromRole(callCtx, grantee, null, role, privilege); + } + + @Nonnull + @Override + public PrivilegeResult grantPrivilegeOnSecurableToRole( + @Nonnull PolarisCallContext callCtx, + @Nonnull PolarisEntityCore grantee, + @Nullable List catalogPath, + @Nonnull PolarisEntityCore securable, + @Nonnull PolarisPrivilege privilege) { + return grantOrRevokePrivilegeOnSecurableToRole(callCtx, true, grantee, securable, privilege); + } + + @Nonnull + @Override + public PrivilegeResult revokePrivilegeOnSecurableFromRole( + @Nonnull PolarisCallContext callCtx, + @Nonnull PolarisEntityCore grantee, + @Nullable List catalogPath, + @Nonnull PolarisEntityCore securable, + @Nonnull PolarisPrivilege privilege) { + return grantOrRevokePrivilegeOnSecurableToRole(callCtx, false, grantee, securable, privilege); + } + + private PrivilegeResult grantOrRevokePrivilegeOnSecurableToRole( + PolarisCallContext callCtx, + boolean grant, + PolarisEntityCore grantee, + PolarisEntityCore securable, + PolarisPrivilege privilege) { + if (!ms(callCtx) + .persistGrantsOrRevokes( + grant, new SecurableGranteePrivilegeTuple(securable, grantee, privilege)) + && !grant) { + return new PrivilegeResult(GRANT_NOT_FOUND, ""); + } + + var grantRecord = + new PolarisGrantRecord( + securable.getCatalogId(), + securable.getId(), + grantee.getCatalogId(), + grantee.getId(), + privilege.getCode()); + return new PrivilegeResult(grantRecord); + } + + @Nonnull + @Override + public LoadGrantsResult loadGrantsOnSecurable( + @Nonnull PolarisCallContext callCtx, PolarisEntityCore securable) { + return ms(callCtx) + .loadGrants(securable.getCatalogId(), securable.getId(), securable.getTypeCode(), true); + } + + @Nonnull + @Override + public LoadGrantsResult loadGrantsToGrantee( + @Nonnull PolarisCallContext callCtx, PolarisEntityCore grantee) { + return ms(callCtx) + .loadGrants(grantee.getCatalogId(), grantee.getId(), grantee.getTypeCode(), false); + } + + @Nonnull + @Override + public ChangeTrackingResult loadEntitiesChangeTracking( + @Nonnull PolarisCallContext callCtx, @Nonnull List entityIds) { + throw new UnsupportedOperationException("No change tracking - do not call this function"); + } + + @Nonnull + @Override + public EntityResult loadEntity( + @Nonnull PolarisCallContext callCtx, + long entityCatalogId, + long entityId, + @Nonnull PolarisEntityType entityType) { + var entity = ms(callCtx).lookupEntity(entityCatalogId, entityId, entityType.getCode()); + return (entity != null) ? new EntityResult(entity) : new EntityResult(ENTITY_NOT_FOUND, null); + } + + @Override + public + Optional> hasOverlappingSiblings( + @Nonnull PolarisCallContext callContext, T entity) { + return Optional.of(ms().hasOverlappingSiblings(entity)); + } + + @Nonnull + @Override + public EntitiesResult loadTasks( + @Nonnull PolarisCallContext callCtx, String executorId, PageToken pageToken) { + var ms = ms(callCtx); + + // find all available tasks + var availableTasks = + ms.fetchEntitiesAsPage( + PolarisEntityConstants.getRootEntityId(), + PolarisEntityConstants.getRootEntityId(), + PolarisEntityType.TASK, + PolarisEntitySubType.ANY_SUBTYPE, + pageToken, + objBase -> mapToEntity(objBase, PolarisEntityConstants.getRootEntityId()), + entity -> { + var taskState = PolarisObjectMapperUtil.parseTaskState(entity); + long taskAgeTimeout = + callCtx + .getRealmConfig() + .getConfig( + PolarisTaskConstants.TASK_TIMEOUT_MILLIS_CONFIG, + PolarisTaskConstants.TASK_TIMEOUT_MILLIS); + return taskState == null + || taskState.executor == null + || clock.millis() - taskState.lastAttemptStartTime > taskAgeTimeout; + }, + Function.identity()); + + // TODO the following loop is NOT a "load" - it's a mutation over all loaded tasks !! + + availableTasks + .items() + .forEach( + task -> { + var newTask = new PolarisBaseEntity.Builder(task); + var properties = PolarisObjectMapperUtil.deserializeProperties(task.getProperties()); + properties.put(PolarisTaskConstants.LAST_ATTEMPT_EXECUTOR_ID, executorId); + properties.put( + PolarisTaskConstants.LAST_ATTEMPT_START_TIME, String.valueOf(clock.millis())); + properties.put( + PolarisTaskConstants.ATTEMPT_COUNT, + String.valueOf( + Integer.parseInt( + properties.getOrDefault(PolarisTaskConstants.ATTEMPT_COUNT, "0")) + + 1)); + newTask.entityVersion(task.getEntityVersion() + 1); + newTask.properties(PolarisObjectMapperUtil.serializeProperties(properties)); + ms.updateEntity(newTask.build()); + }); + return new EntitiesResult(Page.fromItems(availableTasks.items())); + } + + // Policies + + @Nonnull + @Override + public PolicyAttachmentResult attachPolicyToEntity( + @Nonnull PolarisCallContext callCtx, + @Nonnull List targetCatalogPath, + @Nonnull PolarisEntityCore target, + @Nonnull List policyCatalogPath, + @Nonnull PolicyEntity policy, + Map parameters) { + if (parameters == null) { + parameters = Map.of(); + } + return ms(callCtx) + .attachDetachPolicyOnEntity( + policy.getCatalogId(), + policy.getId(), + policy.getPolicyType(), + target.getCatalogId(), + target.getId(), + true, + parameters); + } + + @Nonnull + @Override + public PolicyAttachmentResult detachPolicyFromEntity( + @Nonnull PolarisCallContext callCtx, + @Nonnull List catalogPath, + @Nonnull PolarisEntityCore target, + @Nonnull List policyCatalogPath, + @Nonnull PolicyEntity policy) { + return ms(callCtx) + .attachDetachPolicyOnEntity( + policy.getCatalogId(), + policy.getId(), + policy.getPolicyType(), + target.getCatalogId(), + target.getId(), + false, + Map.of()); + } + + @Nonnull + @Override + public LoadPolicyMappingsResult loadPoliciesOnEntity( + @Nonnull PolarisCallContext callCtx, @Nonnull PolarisEntityCore target) { + return ms(callCtx) + .loadPoliciesOnEntity( + target.getType(), target.getCatalogId(), target.getId(), Optional.empty()); + } + + @Nonnull + @Override + public LoadPolicyMappingsResult loadPoliciesOnEntityByType( + @Nonnull PolarisCallContext callCtx, + @Nonnull PolarisEntityCore target, + @Nonnull PolicyType policyType) { + return ms(callCtx) + .loadPoliciesOnEntity( + target.getType(), target.getCatalogId(), target.getId(), Optional.of(policyType)); + } + + // Principals & PolarisSecretsManager + + @Nonnull + @Override + public CreatePrincipalResult createPrincipal( + @Nonnull PolarisCallContext callCtx, @Nonnull PrincipalEntity principal) { + return ms(callCtx).createPrincipal(principal, rootCredentialsSet); + } + + @Nonnull + @Override + public PrincipalSecretsResult loadPrincipalSecrets( + @Nonnull PolarisCallContext callCtx, @Nonnull String clientId) { + var secrets = ms(callCtx).loadPrincipalSecrets(clientId); + + return (secrets == null) + ? new PrincipalSecretsResult(ENTITY_NOT_FOUND, null) + : new PrincipalSecretsResult(secrets); + } + + @Nonnull + @Override + public PrincipalSecretsResult rotatePrincipalSecrets( + @Nonnull PolarisCallContext callCtx, + @Nonnull String clientId, + long principalId, + boolean reset, + @Nonnull String oldSecretHash) { + return rotatePrincipalSecrets(ms(callCtx), principalId, reset, oldSecretHash); + } + + PrincipalSecretsResult rotatePrincipalSecrets( + NoSqlMetaStore ms, long principalId, boolean reset, String oldSecretHash) { + try { + return ms.updatePrincipalSecrets( + PrincipalSecretsResult.class, + "rotatePrincipalSecrets", + principalId, + (principal, updatedPrincipalBuilder) -> { + var principalSecrets = principalObjToPolarisPrincipalSecrets(principal); + + // rotate the secrets + principalSecrets.rotateSecrets(oldSecretHash); + if (reset) { + principalSecrets.rotateSecrets(principalSecrets.getMainSecretHash()); + } + + updatedPrincipalBuilder + .entityVersion(principal.entityVersion() + 1) + .credentialRotationRequired(reset && !principal.credentialRotationRequired()) + .clientId(principalSecrets.getPrincipalClientId()) + .mainSecretHash(principalSecrets.getMainSecretHash()) + .secondarySecretHash(principalSecrets.getSecondarySecretHash()) + .secretSalt(principalSecrets.getSecretSalt()); + + return new PrincipalSecretsResult(principalSecrets); + }); + } catch (NoSqlMetaStore.PrincipalNotFoundException ignore) { + return new PrincipalSecretsResult(ENTITY_NOT_FOUND, null); + } + } + + @Nonnull + @Override + public PrincipalSecretsResult resetPrincipalSecrets( + @Nonnull PolarisCallContext callCtx, + long principalId, + @Nonnull String resolvedClientId, + String customClientSecret) { + try { + return ms(callCtx) + .updatePrincipalSecrets( + PrincipalSecretsResult.class, + "resetPrincipalSecrets", + principalId, + (principal, updatedPrincipalBuilder) -> { + var principalSecrets = + new PolarisPrincipalSecrets( + principal.stableId(), resolvedClientId, customClientSecret); + updatedPrincipalBuilder + .entityVersion(principal.entityVersion() + 1) + .clientId(principalSecrets.getPrincipalClientId()) + .mainSecretHash(principalSecrets.getMainSecretHash()) + .secondarySecretHash(principalSecrets.getSecondarySecretHash()) + .secretSalt(principalSecrets.getSecretSalt()); + + return new PrincipalSecretsResult(principalSecrets); + }); + } catch (NoSqlMetaStore.PrincipalNotFoundException ignore) { + return new PrincipalSecretsResult(ENTITY_NOT_FOUND, null); + } + } + + @Override + public void deletePrincipalSecrets( + @Nonnull PolarisCallContext callCtx, @Nonnull String clientId, long principalId) { + ms(callCtx) + .updatePrincipalSecrets( + String.class, + "deletePrincipalSecrets", + principalId, + (principal, updatedPrincipalBuilder) -> { + // Do NOT update the entityVersion + updatedPrincipalBuilder + .clientId(Optional.empty()) + .secretSalt(Optional.empty()) + .mainSecretHash(Optional.empty()) + .secondarySecretHash(Optional.empty()); + return ""; // need some non-null return value + }); + } + + // PolarisCredentialVendor + + @Nonnull + @Override + public ScopedCredentialsResult getSubscopedCredsForEntity( + @Nonnull PolarisCallContext callCtx, + long catalogId, + long entityId, + @Nonnull PolarisEntityType entityType, + boolean allowListOperation, + @Nonnull Set allowedReadLocations, + @Nonnull Set allowedWriteLocations, + Optional refreshCredentialsEndpoint) { + + checkArgument( + !allowedReadLocations.isEmpty() || !allowedWriteLocations.isEmpty(), + "allowed_locations_to_subscope_is_required"); + + // reload the entity, error out if not found + var reloadedEntity = loadEntity(callCtx, catalogId, entityId, entityType); + if (reloadedEntity.getReturnStatus() != BaseResult.ReturnStatus.SUCCESS) { + return new ScopedCredentialsResult( + reloadedEntity.getReturnStatus(), reloadedEntity.getExtraInformation()); + } + + // get storage integration + var storageIntegration = ms(callCtx).loadPolarisStorageIntegration(reloadedEntity.getEntity()); + + // cannot be null + checkNotNull( + storageIntegration, + "storage_integration_not_exists, catalogId=%s, entityId=%s", + catalogId, + entityId); + + try { + var creds = + storageIntegration.getSubscopedCreds( + callCtx.getRealmConfig(), + allowListOperation, + allowedReadLocations, + allowedWriteLocations, + refreshCredentialsEndpoint); + return new ScopedCredentialsResult(creds); + } catch (Exception ex) { + return new ScopedCredentialsResult(SUBSCOPE_CREDS_ERROR, ex.getMessage()); + } + } + + @Override + public boolean requiresEntityReload() { + return false; + } + + @Override + public void writeEvents( + @Nonnull PolarisCallContext callCtx, @Nonnull List polarisEvents) { + throw new UnsupportedOperationException("Events not supported in NoSQL persistence"); + } +} diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NoSqlMetaStoreManagerFactory.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NoSqlMetaStoreManagerFactory.java new file mode 100644 index 0000000000..ad778b236b --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NoSqlMetaStoreManagerFactory.java @@ -0,0 +1,350 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.nosql.metastore; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static org.apache.polaris.persistence.nosql.coretypes.refs.References.realmReferenceNames; +import static org.apache.polaris.persistence.nosql.realms.api.RealmDefinition.RealmStatus.ACTIVE; +import static org.apache.polaris.persistence.nosql.realms.api.RealmDefinition.RealmStatus.CREATED; +import static org.apache.polaris.persistence.nosql.realms.api.RealmDefinition.RealmStatus.INACTIVE; +import static org.apache.polaris.persistence.nosql.realms.api.RealmDefinition.RealmStatus.INITIALIZING; +import static org.apache.polaris.persistence.nosql.realms.api.RealmDefinition.RealmStatus.PURGED; +import static org.apache.polaris.persistence.nosql.realms.api.RealmDefinition.RealmStatus.PURGING; + +import io.smallrye.common.annotation.Identifier; +import jakarta.annotation.PostConstruct; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import java.time.Clock; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.config.RealmConfig; +import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisEntity; +import org.apache.polaris.core.entity.PolarisEntityConstants; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.entity.PolarisPrincipalSecrets; +import org.apache.polaris.core.persistence.BasePersistence; +import org.apache.polaris.core.persistence.MetaStoreManagerFactory; +import org.apache.polaris.core.persistence.PolarisMetaStoreManager; +import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; +import org.apache.polaris.core.persistence.cache.EntityCache; +import org.apache.polaris.core.persistence.dao.entity.BaseResult; +import org.apache.polaris.core.persistence.dao.entity.CreatePrincipalResult; +import org.apache.polaris.core.persistence.dao.entity.PrincipalSecretsResult; +import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; +import org.apache.polaris.persistence.nosql.api.Persistence; +import org.apache.polaris.persistence.nosql.api.RealmPersistenceFactory; +import org.apache.polaris.persistence.nosql.authz.api.Privileges; +import org.apache.polaris.persistence.nosql.realms.api.RealmDefinition; +import org.apache.polaris.persistence.nosql.realms.api.RealmManagement; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@ApplicationScoped +@Identifier("nosql") +class NoSqlMetaStoreManagerFactory implements MetaStoreManagerFactory { + private static final Logger LOGGER = LoggerFactory.getLogger(NoSqlMetaStoreManagerFactory.class); + + private final Map realmPersistenceMap = new ConcurrentHashMap<>(); + private final RealmManagement realmManagement; + private final RealmPersistenceFactory realmPersistenceFactory; + private final Privileges privileges; + private final PolarisStorageIntegrationProvider storageIntegrationProvider; + private final Clock clock; + private final PolarisDiagnostics diagnostics; + + @SuppressWarnings("CdiInjectionPointsInspection") + @Inject + NoSqlMetaStoreManagerFactory( + RealmManagement realmManagement, + RealmPersistenceFactory realmPersistenceFactory, + Privileges privileges, + PolarisStorageIntegrationProvider storageIntegrationProvider, + Clock clock, + PolarisDiagnostics diagnostics) { + this.realmManagement = realmManagement; + this.realmPersistenceFactory = realmPersistenceFactory; + this.privileges = privileges; + this.storageIntegrationProvider = storageIntegrationProvider; + this.clock = clock; + this.diagnostics = diagnostics; + } + + @PostConstruct + void logRegisteredRealms() { + try (var realms = realmManagement.list()) { + realms.forEach(realmDefinition -> LOGGER.info("Realm registered: {}", realmDefinition)); + } + } + + @Override + public Map bootstrapRealms( + Iterable realms, RootCredentialsSet rootCredentialsSet) { + Map results = new HashMap<>(); + + for (String realmId : realms) { + var existing = realmManagement.get(realmId); + if (existing.isPresent() && !existing.get().needsBootstrap()) { + LOGGER.debug("Realm '{}' is already fully bootstrapped.", realmId); + continue; + } + + var secretsResult = bootstrapRealm(realmId, rootCredentialsSet, existing); + results.put(realmId, secretsResult); + } + + return Map.copyOf(results); + } + + @Override + public Map purgeRealms(Iterable realms) { + var results = new HashMap(); + + for (var realm : realms) { + results.put(realm, purgeRealm(realm)); + } + + return Map.copyOf(results); + } + + @Override + public EntityCache getOrCreateEntityCache(RealmContext realmContext, RealmConfig realmConfig) { + // no `EntityCache` + return null; + } + + @Override + public BasePersistence getOrCreateSession(RealmContext realmContext) { + return newPersistenceMetaStore(initializedRealmPersistence(realmContext.getRealmIdentifier())); + } + + @Override + public PolarisMetaStoreManager getOrCreateMetaStoreManager(RealmContext realmContext) { + var realmId = realmContext.getRealmIdentifier(); + var persistence = initializedRealmPersistence(realmId); + + return new NoSqlMetaStoreManager( + () -> purgeRealm(realmId), null, () -> newPersistenceMetaStore(persistence), clock); + } + + private NoSqlMetaStore newPersistenceMetaStore(Persistence persistence) { + return new NoSqlMetaStore(persistence, privileges, storageIntegrationProvider, diagnostics); + } + + private Persistence initializedRealmPersistence(String realmId) { + var persistence = realmPersistenceMap.get(realmId); + if (persistence != null) { + return persistence; + } + // This synchronization is there to prevent the CHM from locking and causing "strange" side + // effects. A naive "computeIfAbsent" with 'initializeIfNecessary' called from the mapping + // function could cause the CHM to lock for quite a long while. + synchronized (this) { + persistence = realmPersistenceMap.get(realmId); + if (persistence != null) { + return persistence; + } + + LOGGER.info("Checking realm '{}' on first use", realmId); + + var realmDesc = realmManagement.get(realmId); + checkArgument(realmDesc.isPresent(), "Realm '%s' does not exist", realmId); + checkArgument( + !realmDesc.get().needsBootstrap(), + "Realm '%s' has not been fully bootstrapped. Re-run the bootstrap admin command.", + realmId); + + persistence = buildRealmPersistence(realmId); + var ms = newPersistenceMetaStore(persistence); + ms.initializeCatalogsIfNecessary(); + realmPersistenceMap.put(realmId, persistence); + LOGGER.info("Done checking realm '{}'", realmId); + return persistence; + } + } + + private Persistence buildRealmPersistence(String realmId) { + return realmPersistenceFactory.newBuilder().realmId(realmId).build(); + } + + BaseResult purgeRealm(String realmId) { + while (true) { + var existingOptional = realmManagement.get(realmId); + if (existingOptional.isEmpty()) { + realmPersistenceMap.remove(realmId); + return new BaseResult(BaseResult.ReturnStatus.ENTITY_NOT_FOUND); + } + var existing = existingOptional.get(); + + var nextStatus = + switch (existing.status()) { + case CREATED, LOADING, INITIALIZING, INACTIVE -> PURGING; + case ACTIVE -> INACTIVE; + case PURGING -> + // TODO this state should really happen during maintenance!! + PURGED; + case PURGED -> PURGED; + }; + + var update = RealmDefinition.builder().from(existing).status(nextStatus).build(); + + var updated = realmManagement.update(existing, update); + + if (updated.status() == PURGED) { + realmManagement.delete(updated); + break; + } + } + + realmPersistenceMap.remove(realmId); + + return new BaseResult(BaseResult.ReturnStatus.SUCCESS); + } + + /** + * Bootstrap the given realm using the given root credentials. + * + *

Repeated calls to this function for the same realm are safe, the outcome is idempotent. + */ + private PrincipalSecretsResult bootstrapRealm( + String realmId, RootCredentialsSet rootCredentialsSet, Optional existing) { + LOGGER.info("Bootstrapping realm '{}' ...", realmId); + + // TODO later, update bootstrap to use RealmLifecycleCallbacks and leverage the other + // intermediate realm-states to make this function not racy + + var realmDesc = + existing.orElseGet( + () -> { + var desc = realmManagement.create(realmId); + // Move realm from CREATED to INITIALIZING state + desc = + realmManagement.update( + desc, RealmDefinition.builder().from(desc).status(INITIALIZING).build()); + return desc; + }); + + if (realmDesc.status() == CREATED) { + realmDesc = + realmManagement.update( + realmDesc, RealmDefinition.builder().from(realmDesc).status(INITIALIZING).build()); + } + + checkState( + realmDesc.status() == INITIALIZING, + "Unexpected status '%s' for realm '%s'", + realmDesc.status(), + realmId); + + var persistence = buildRealmPersistence(realmId); + persistence.createReferencesSilent(realmReferenceNames()); + var metaStore = newPersistenceMetaStore(persistence); + var metaStoreManager = + new NoSqlMetaStoreManager( + () -> { + throw new IllegalStateException("Cannot purge while bootstrapping"); + }, + rootCredentialsSet, + () -> metaStore, + clock); + + var secretsResult = + bootstrapServiceAndCreatePolarisPrincipalForRealm( + realmId, metaStoreManager, metaStore, rootCredentialsSet); + + realmManagement.update( + realmDesc, RealmDefinition.builder().from(realmDesc).status(ACTIVE).build()); + + LOGGER.info("Realm '{}' has been successfully bootstrapped.", realmId); + + return secretsResult; + } + + /** + * This method bootstraps service for a given realm: i.e., creates all the required entities in + * the metastore and creates a root service principal. After that, we rotate the root principal + * credentials and print them to stdout + */ + private PrincipalSecretsResult bootstrapServiceAndCreatePolarisPrincipalForRealm( + String realmId, + NoSqlMetaStoreManager metaStoreManager, + NoSqlMetaStore metaStore, + RootCredentialsSet rootCredentialsSet) { + var createPrincipalResult = metaStoreManager.bootstrapPolarisServiceInternal(metaStore); + + var rootPrincipal = + createPrincipalResult + .map(result -> (PolarisBaseEntity) result.getPrincipal()) + .orElseGet( + () -> + metaStoreManager + .readEntityByName( + metaStore, + null, + PolarisEntityType.PRINCIPAL, + PolarisEntityConstants.getRootPrincipalName()) + .getEntity()); + + var clientId = + PolarisEntity.of(rootPrincipal) + .getInternalPropertiesAsMap() + .get(PolarisEntityConstants.getClientIdPropertyName()); + checkState(clientId != null, "Root principal has no client-ID"); + var secrets = metaStore.loadPrincipalSecrets(clientId); + + var principalSecrets = createPrincipalResult.map(CreatePrincipalResult::getPrincipalSecrets); + if (principalSecrets.isPresent()) { + LOGGER.debug( + "Root principal created for realm '{}', directly returning credentials for client-ID '{}'", + realmId, + principalSecrets.get().getPrincipalClientId()); + return new PrincipalSecretsResult(principalSecrets.get()); + } + + var providedCredentials = rootCredentialsSet.credentials().get(realmId); + if (providedCredentials != null) { + LOGGER.debug( + "Root principal for realm '{}' already exists, credentials provided externally, returning credentials for client-ID '{}'", + realmId, + providedCredentials.clientId()); + return new PrincipalSecretsResult( + new PolarisPrincipalSecrets( + rootPrincipal.getId(), + providedCredentials.clientId(), + providedCredentials.clientSecret(), + providedCredentials.clientSecret())); + } + + // Have to rotate the secrets to retain the idempotency of this function + var result = + metaStoreManager.rotatePrincipalSecrets( + metaStore, secrets.getPrincipalId(), false, secrets.getMainSecretHash()); + LOGGER.debug( + "Rotating credentials for root principal for realm '{}', client-ID is '{}'", + realmId, + result.getPrincipalSecrets().getPrincipalClientId()); + return result; + } +} diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NoSqlPaginationToken.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NoSqlPaginationToken.java new file mode 100644 index 0000000000..e0840fef9b --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NoSqlPaginationToken.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.persistence.nosql.metastore; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import org.apache.polaris.core.persistence.pagination.Token; +import org.apache.polaris.immutables.PolarisImmutable; +import org.apache.polaris.persistence.nosql.api.index.IndexKey; +import org.apache.polaris.persistence.nosql.api.obj.ObjRef; + +/** + * Pagination token for NoSQL that refers to the next {@link IndexKey}. The next request will refer + * to the same index, for example, the same catalog state. + */ +@PolarisImmutable +@JsonSerialize(as = ImmutableNoSqlPaginationToken.class) +@JsonDeserialize(as = ImmutableNoSqlPaginationToken.class) +public interface NoSqlPaginationToken extends Token { + String ID = "n"; + + @JsonProperty("c") + ObjRef containerObjRef(); + + @JsonProperty("k") + IndexKey key(); + + @Override + default String getT() { + return ID; + } + + static NoSqlPaginationToken paginationToken(ObjRef containerObjRef, IndexKey key) { + return ImmutableNoSqlPaginationToken.builder() + .containerObjRef(containerObjRef) + .key(key) + .build(); + } + + final class NoSqlPaginationTokenType implements Token.TokenType { + @Override + public String id() { + return ID; + } + + @Override + public Class javaType() { + return NoSqlPaginationToken.class; + } + } +} diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NonFunctionalBasePersistence.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NonFunctionalBasePersistence.java new file mode 100644 index 0000000000..7fc5e413a1 --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NonFunctionalBasePersistence.java @@ -0,0 +1,368 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.persistence.nosql.metastore; + +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.util.List; +import java.util.function.Function; +import java.util.function.Predicate; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.entity.EntityNameLookupRecord; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; +import org.apache.polaris.core.entity.PolarisEntityCore; +import org.apache.polaris.core.entity.PolarisEntityId; +import org.apache.polaris.core.entity.PolarisEntitySubType; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.entity.PolarisEvent; +import org.apache.polaris.core.entity.PolarisGrantRecord; +import org.apache.polaris.core.entity.PolarisPrincipalSecrets; +import org.apache.polaris.core.persistence.BasePersistence; +import org.apache.polaris.core.persistence.IntegrationPersistence; +import org.apache.polaris.core.persistence.pagination.Page; +import org.apache.polaris.core.persistence.pagination.PageToken; +import org.apache.polaris.core.policy.PolarisPolicyMappingRecord; +import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; +import org.apache.polaris.core.storage.PolarisStorageIntegration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +abstract class NonFunctionalBasePersistence implements BasePersistence, IntegrationPersistence { + private static final Logger LOGGER = LoggerFactory.getLogger(NonFunctionalBasePersistence.class); + + @Override + public long generateNewId(@Nonnull PolarisCallContext callCtx) { + throw unimplemented(); + } + + @Override + public void writeEntity( + @Nonnull PolarisCallContext callCtx, + @Nonnull PolarisBaseEntity entity, + // nameOrParentChanged is `true` if originalEntity==null or the parentId or the name changed + boolean nameOrParentChanged, + @Nullable PolarisBaseEntity originalEntity) { + throw useMetaStoreManager("create/update/rename/delete"); + } + + @Override + public void writeEntities( + @Nonnull PolarisCallContext callCtx, + @Nonnull List entities, + @Nullable List originalEntities) { + throw useMetaStoreManager("create/update/rename/delete"); + } + + @Override + public void writeToGrantRecords( + @Nonnull PolarisCallContext callCtx, @Nonnull PolarisGrantRecord grantRec) { + throw unimplemented(); + } + + @Override + public void deleteEntity(@Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity) { + throw unimplemented(); + } + + @Override + public void deleteFromGrantRecords( + @Nonnull PolarisCallContext callCtx, @Nonnull PolarisGrantRecord grantRec) { + throw unimplemented(); + } + + @Override + public void deleteAllEntityGrantRecords( + @Nonnull PolarisCallContext callCtx, + @Nonnull PolarisEntityCore entity, + @Nonnull List grantsOnGrantee, + @Nonnull List grantsOnSecurable) { + throw unimplemented(); + } + + @Nullable + @Override + public PolarisBaseEntity lookupEntity( + @Nonnull PolarisCallContext callCtx, long catalogId, long entityId, int entityTypeCode) { + throw useMetaStoreManager("lookupEntity/loadResolvedEntityById"); + } + + @Nullable + @Override + public PolarisBaseEntity lookupEntityByName( + @Nonnull PolarisCallContext callCtx, + long catalogId, + long parentId, + int entityTypeCode, + @Nonnull String name) { + throw useMetaStoreManager("readEntityByName"); + } + + @Override + public EntityNameLookupRecord lookupEntityIdAndSubTypeByName( + @Nonnull PolarisCallContext callCtx, + long catalogId, + long parentId, + int typeCode, + @Nonnull String name) { + throw unimplemented(); + } + + @Nonnull + @Override + public List lookupEntities( + @Nonnull PolarisCallContext callCtx, List entityIds) { + throw unimplemented(); + } + + @Nonnull + @Override + public List lookupEntityVersions( + @Nonnull PolarisCallContext callCtx, List entityIds) { + throw unimplemented(); + } + + @Override + public boolean hasChildren( + @Nonnull PolarisCallContext callCtx, + @Nullable PolarisEntityType optionalEntityType, + long catalogId, + long parentId) { + throw unimplemented(); + } + + @Nonnull + @Override + public Page listFullEntities( + @Nonnull PolarisCallContext callCtx, + long catalogId, + long parentId, + @Nonnull PolarisEntityType entityType, + @Nonnull PolarisEntitySubType entitySubType, + @Nonnull Predicate entityFilter, + @Nonnull Function transformer, + PageToken pageToken) { + throw useMetaStoreManager("listFullEntities"); + } + + @Nonnull + @Override + public Page listEntities( + @Nonnull PolarisCallContext callCtx, + long catalogId, + long parentId, + @Nonnull PolarisEntityType entityType, + @Nonnull PolarisEntitySubType entitySubType, + @Nonnull PageToken pageToken) { + throw unimplemented(); + } + + @Override + public void writeToPolicyMappingRecords( + @Nonnull PolarisCallContext callCtx, @Nonnull PolarisPolicyMappingRecord record) { + throw useMetaStoreManager("attachPolicyToEntity"); + } + + @Override + public void deleteFromPolicyMappingRecords( + @Nonnull PolarisCallContext callCtx, @Nonnull PolarisPolicyMappingRecord record) { + throw useMetaStoreManager("detachPolicyFromEntity"); + } + + @Override + public void deleteAllEntityPolicyMappingRecords( + @Nonnull PolarisCallContext callCtx, + @Nonnull PolarisBaseEntity entity, + @Nonnull List mappingOnTarget, + @Nonnull List mappingOnPolicy) { + throw unimplemented(); + } + + @Nullable + @Override + public PolarisPolicyMappingRecord lookupPolicyMappingRecord( + @Nonnull PolarisCallContext callCtx, + long targetCatalogId, + long targetId, + int policyTypeCode, + long policyCatalogId, + long policyId) { + throw unimplemented(); + } + + @Nonnull + @Override + public List loadPoliciesOnTargetByType( + @Nonnull PolarisCallContext callCtx, + long targetCatalogId, + long targetId, + int policyTypeCode) { + throw useMetaStoreManager("loadPoliciesOnEntityByType"); + } + + @Nonnull + @Override + public List loadAllTargetsOnPolicy( + @Nonnull PolarisCallContext callCtx, + long policyCatalogId, + long policyId, + int policyTypeCode) { + LOGGER.warn( + "loadAllTargetsOnPolicy is not implemented and always returns empty list, at least as long as org.apache.polaris.core.persistence.PolarisTestMetaStoreManager.testPolicyMappingCleanup calls this !!"); + return List.of(); + // throw useMetaStoreManager("loadEntitiesOnPolicy"); + } + + @Nonnull + @Override + public List loadAllPoliciesOnTarget( + @Nonnull PolarisCallContext callCtx, long targetCatalogId, long targetId) { + throw useMetaStoreManager("loadPoliciesOnEntity"); + } + + @Override + public int lookupEntityGrantRecordsVersion( + @Nonnull PolarisCallContext callCtx, long catalogId, long entityId) { + throw useMetaStoreManager("loadGrantsOnSecurable"); + } + + @Nullable + @Override + public PolarisGrantRecord lookupGrantRecord( + @Nonnull PolarisCallContext callCtx, + long securableCatalogId, + long securableId, + long granteeCatalogId, + long granteeId, + int privilegeCode) { + throw useMetaStoreManager("loadGrantsOnSecurable"); + } + + @Nonnull + @Override + public List loadAllGrantRecordsOnSecurable( + @Nonnull PolarisCallContext callCtx, long securableCatalogId, long securableId) { + throw useMetaStoreManager("loadGrantsOnSecurable"); + } + + @Nonnull + @Override + public List loadAllGrantRecordsOnGrantee( + @Nonnull PolarisCallContext callCtx, long granteeCatalogId, long granteeId) { + throw useMetaStoreManager("loadGrantsToGrantee"); + } + + @Override + public void persistStorageIntegrationIfNeeded( + @Nonnull PolarisCallContext callCtx, + @Nonnull PolarisBaseEntity entity, + @Nullable PolarisStorageIntegration storageIntegration) { + throw unimplemented(); + } + + @Nullable + @Override + public + PolarisStorageIntegration createStorageIntegration( + @Nonnull PolarisCallContext callCtx, + long catalogId, + long entityId, + PolarisStorageConfigurationInfo polarisStorageConfigurationInfo) { + throw unimplemented(); + } + + @Nullable + @Override + public + PolarisStorageIntegration loadPolarisStorageIntegration( + @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity) { + throw useMetaStoreManager("getSubscopedCredsForEntity"); + } + + @Override + public void deletePrincipalSecrets( + @Nonnull PolarisCallContext callCtx, @Nonnull String clientId, long principalId) { + throw useMetaStoreManager("deletePrincipalSecrets"); + } + + @Nullable + @Override + public PolarisPrincipalSecrets rotatePrincipalSecrets( + @Nonnull PolarisCallContext callCtx, + @Nonnull String clientId, + long principalId, + boolean reset, + @Nonnull String oldSecretHash) { + throw useMetaStoreManager("rotatePrincipalSecrets"); + } + + @Nonnull + @Override + public PolarisPrincipalSecrets generateNewPrincipalSecrets( + @Nonnull PolarisCallContext callCtx, @Nonnull String principalName, long principalId) { + throw unimplemented(); + } + + @Nullable + @Override + public PolarisPrincipalSecrets loadPrincipalSecrets( + @Nonnull PolarisCallContext callCtx, @Nonnull String clientId) { + throw unimplemented(); + } + + @Nullable + @Override + public PolarisPrincipalSecrets storePrincipalSecrets( + @Nonnull PolarisCallContext callCtx, + long principalId, + @Nonnull String resolvedClientId, + String customClientSecret) { + throw useMetaStoreManager("storePrincipalSecrets"); + } + + @Override + public void writeEvents(@Nonnull List events) { + throw unimplemented(); + } + + @Override + public void deleteAll(@Nonnull PolarisCallContext callCtx) { + throw unimplemented(); + } + + @Override + public BasePersistence detach() { + throw unimplemented(); + } + + static UnsupportedOperationException unimplemented() { + var ex = new UnsupportedOperationException("Intentionally not implemented"); + LOGGER.error("Unsupported function call", ex); + return ex; + } + + static UnsupportedOperationException useMetaStoreManager(String function) { + var ex = + new UnsupportedOperationException( + "Operation not supported - use PolarisMetaStoreManager." + function + "()"); + LOGGER.error("Unsupported function call", ex); + return ex; + } +} diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/committers/CatalogChangeCommitter.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/committers/CatalogChangeCommitter.java new file mode 100644 index 0000000000..569d297f00 --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/committers/CatalogChangeCommitter.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.nosql.metastore.committers; + +import jakarta.annotation.Nonnull; +import org.apache.polaris.persistence.nosql.api.commit.CommitException; +import org.apache.polaris.persistence.nosql.api.commit.CommitterState; +import org.apache.polaris.persistence.nosql.api.index.IndexKey; +import org.apache.polaris.persistence.nosql.api.index.UpdatableIndex; +import org.apache.polaris.persistence.nosql.api.obj.ObjRef; +import org.apache.polaris.persistence.nosql.coretypes.catalog.CatalogStateObj; +import org.apache.polaris.persistence.nosql.coretypes.catalog.EntityIdSet; +import org.apache.polaris.persistence.nosql.coretypes.changes.Change; + +@FunctionalInterface +public interface CatalogChangeCommitter { + @Nonnull + ChangeResult change( + @Nonnull CommitterState state, + @Nonnull CatalogStateObj.Builder ref, + @Nonnull UpdatableIndex byName, + @Nonnull UpdatableIndex byId, + @Nonnull UpdatableIndex changes, + @Nonnull UpdatableIndex locations) + throws CommitException; +} diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/committers/CatalogChangeCommitterWrapper.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/committers/CatalogChangeCommitterWrapper.java new file mode 100644 index 0000000000..5531957d83 --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/committers/CatalogChangeCommitterWrapper.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.nosql.metastore.committers; + +import static org.apache.polaris.persistence.nosql.api.index.IndexContainer.newUpdatableIndex; +import static org.apache.polaris.persistence.nosql.api.index.IndexKey.INDEX_KEY_SERIALIZER; +import static org.apache.polaris.persistence.nosql.api.obj.ObjRef.OBJ_REF_SERIALIZER; +import static org.apache.polaris.persistence.nosql.coretypes.catalog.EntityIdSet.ENTITY_ID_SET_SERIALIZER; +import static org.apache.polaris.persistence.nosql.coretypes.changes.Change.CHANGE_SERIALIZER; + +import jakarta.annotation.Nonnull; +import java.util.Optional; +import java.util.function.Supplier; +import org.apache.polaris.persistence.nosql.api.commit.CommitException; +import org.apache.polaris.persistence.nosql.api.commit.CommitRetryable; +import org.apache.polaris.persistence.nosql.api.commit.CommitterState; +import org.apache.polaris.persistence.nosql.coretypes.ContainerObj; +import org.apache.polaris.persistence.nosql.coretypes.catalog.CatalogStateObj; + +/** + * Abstracts common {@link ContainerObj#stableIdToName()} and {@link ContainerObj#nameToObjRef()} + * handling for committing operations, for catalog related types. + * + * @param result of the commiting operation + */ +public record CatalogChangeCommitterWrapper(CatalogChangeCommitter changeCommitter) + implements CommitRetryable { + + @SuppressWarnings("DuplicatedCode") + @Nonnull + @Override + public Optional attempt( + @Nonnull CommitterState state, + @Nonnull Supplier> refObjSupplier) + throws CommitException { + var refObj = refObjSupplier.get(); + var byName = + refObj + .map(CatalogStateObj::nameToObjRef) + .map(c -> c.asUpdatableIndex(state.persistence(), OBJ_REF_SERIALIZER)) + .orElseGet(() -> newUpdatableIndex(state.persistence(), OBJ_REF_SERIALIZER)); + var byId = + refObj + .map(CatalogStateObj::stableIdToName) + .map(c -> c.asUpdatableIndex(state.persistence(), INDEX_KEY_SERIALIZER)) + .orElseGet(() -> newUpdatableIndex(state.persistence(), INDEX_KEY_SERIALIZER)); + var locations = + refObj + .flatMap(CatalogStateObj::locations) + .map(c -> c.asUpdatableIndex(state.persistence(), ENTITY_ID_SET_SERIALIZER)) + .orElseGet(() -> newUpdatableIndex(state.persistence(), ENTITY_ID_SET_SERIALIZER)); + // 'changes' contains the changes for the particular commit + var changes = newUpdatableIndex(state.persistence(), CHANGE_SERIALIZER); + + var ref = CatalogStateObj.builder(); + refObj.ifPresent(ref::from); + + var r = changeCommitter.change(state, ref, byName, byId, changes, locations); + + if (r instanceof ChangeResult.CommitChange(RESULT result)) { + ref.changes(changes.toIndexed("idx-changes-", state::writeOrReplace)) + .nameToObjRef(byName.toIndexed("idx-name-", state::writeOrReplace)) + .stableIdToName(byId.toIndexed("idx-id-", state::writeOrReplace)) + .locations(locations.toOptionalIndexed("idx-loc-", state::writeOrReplace)); + return state.commitResult(result, ref, refObj); + } + if (r instanceof ChangeResult.NoChange(RESULT result)) { + return state.noCommit(result); + } + throw new IllegalStateException("" + r); + } +} diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/committers/ChangeCommitter.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/committers/ChangeCommitter.java new file mode 100644 index 0000000000..f29db7b3bc --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/committers/ChangeCommitter.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.nosql.metastore.committers; + +import jakarta.annotation.Nonnull; +import org.apache.polaris.persistence.nosql.api.commit.CommitException; +import org.apache.polaris.persistence.nosql.api.commit.CommitterState; +import org.apache.polaris.persistence.nosql.api.index.IndexKey; +import org.apache.polaris.persistence.nosql.api.index.UpdatableIndex; +import org.apache.polaris.persistence.nosql.api.obj.ObjRef; +import org.apache.polaris.persistence.nosql.coretypes.ContainerObj; + +@FunctionalInterface +public interface ChangeCommitter { + @Nonnull + ChangeResult change( + @Nonnull CommitterState state, + @Nonnull ContainerObj.Builder ref, + @Nonnull UpdatableIndex byName, + @Nonnull UpdatableIndex byId) + throws CommitException; +} diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/committers/ChangeCommitterWrapper.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/committers/ChangeCommitterWrapper.java new file mode 100644 index 0000000000..928002cb29 --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/committers/ChangeCommitterWrapper.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.nosql.metastore.committers; + +import static org.apache.polaris.persistence.nosql.api.index.IndexContainer.newUpdatableIndex; +import static org.apache.polaris.persistence.nosql.api.index.IndexKey.INDEX_KEY_SERIALIZER; +import static org.apache.polaris.persistence.nosql.api.obj.ObjRef.OBJ_REF_SERIALIZER; + +import jakarta.annotation.Nonnull; +import java.util.Optional; +import java.util.function.Supplier; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.persistence.nosql.api.commit.CommitException; +import org.apache.polaris.persistence.nosql.api.commit.CommitRetryable; +import org.apache.polaris.persistence.nosql.api.commit.CommitterState; +import org.apache.polaris.persistence.nosql.coretypes.ContainerObj; +import org.apache.polaris.persistence.nosql.coretypes.catalog.CatalogRolesObj; +import org.apache.polaris.persistence.nosql.coretypes.catalog.CatalogStateObj; +import org.apache.polaris.persistence.nosql.coretypes.catalog.CatalogsObj; +import org.apache.polaris.persistence.nosql.coretypes.principals.PrincipalRolesObj; +import org.apache.polaris.persistence.nosql.coretypes.principals.PrincipalsObj; +import org.apache.polaris.persistence.nosql.coretypes.realm.ImmediateTasksObj; + +/** + * Abstracts common {@link ContainerObj#stableIdToName()} and {@link ContainerObj#nameToObjRef()} + * handling for committing operations, for non-catalog related types. + * + * @param commited object type + * @param builder type for {@link REF_OBJ} + * @param result of the commiting operation + */ +public record ChangeCommitterWrapper< + REF_OBJ extends ContainerObj, B extends ContainerObj.Builder, RESULT>( + ChangeCommitter changeCommitter, PolarisEntityType entityType) + implements CommitRetryable { + + @SuppressWarnings("DuplicatedCode") + @Nonnull + @Override + public Optional attempt( + @Nonnull CommitterState state, + @Nonnull Supplier> refObjSupplier) + throws CommitException { + var refObj = refObjSupplier.get(); + var byName = + refObj + .map(ContainerObj::nameToObjRef) + .map(c -> c.asUpdatableIndex(state.persistence(), OBJ_REF_SERIALIZER)) + .orElseGet(() -> newUpdatableIndex(state.persistence(), OBJ_REF_SERIALIZER)); + var byId = + refObj + .map(ContainerObj::stableIdToName) + .map(c -> c.asUpdatableIndex(state.persistence(), INDEX_KEY_SERIALIZER)) + .orElseGet(() -> newUpdatableIndex(state.persistence(), INDEX_KEY_SERIALIZER)); + @SuppressWarnings("unchecked") + var ref = (B) newContainerBuilderForEntityType(entityType); + refObj.ifPresent(ref::from); + + var r = changeCommitter.change(state, ref, byName, byId); + + if (r instanceof ChangeResult.CommitChange(RESULT result)) { + ref.nameToObjRef(byName.toIndexed("idx-name-", state::writeOrReplace)) + .stableIdToName(byId.toIndexed("idx-id-", state::writeOrReplace)); + return state.commitResult(result, ref, refObj); + } + if (r instanceof ChangeResult.NoChange(RESULT result)) { + return state.noCommit(result); + } + throw new IllegalStateException("" + r); + } + + static ContainerObj.Builder> + newContainerBuilderForEntityType(PolarisEntityType entityType) { + return switch (entityType) { + case CATALOG -> CatalogsObj.builder(); + case PRINCIPAL -> PrincipalsObj.builder(); + case PRINCIPAL_ROLE -> PrincipalRolesObj.builder(); + case TASK -> ImmediateTasksObj.builder(); + + // per catalog + case CATALOG_ROLE -> CatalogRolesObj.builder(); + case NAMESPACE, TABLE_LIKE, POLICY -> CatalogStateObj.builder(); + + default -> throw new IllegalArgumentException("Unsupported entity type: " + entityType); + }; + } +} diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/committers/ChangeResult.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/committers/ChangeResult.java new file mode 100644 index 0000000000..fe74ecd576 --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/committers/ChangeResult.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.nosql.metastore.committers; + +public interface ChangeResult { + @SuppressWarnings("unused") + RESULT result(); + + record CommitChange(RESULT result) implements ChangeResult {} + + record NoChange(RESULT result) implements ChangeResult {} +} diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/committers/PrincipalsChangeCommitter.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/committers/PrincipalsChangeCommitter.java new file mode 100644 index 0000000000..9dd01ec84d --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/committers/PrincipalsChangeCommitter.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.nosql.metastore.committers; + +import jakarta.annotation.Nonnull; +import org.apache.polaris.persistence.nosql.api.commit.CommitException; +import org.apache.polaris.persistence.nosql.api.commit.CommitterState; +import org.apache.polaris.persistence.nosql.api.index.IndexKey; +import org.apache.polaris.persistence.nosql.api.index.UpdatableIndex; +import org.apache.polaris.persistence.nosql.api.obj.ObjRef; +import org.apache.polaris.persistence.nosql.coretypes.principals.PrincipalsObj; + +@FunctionalInterface +public interface PrincipalsChangeCommitter { + @Nonnull + ChangeResult change( + @Nonnull CommitterState state, + @Nonnull PrincipalsObj.Builder ref, + @Nonnull UpdatableIndex byName, + @Nonnull UpdatableIndex byId, + @Nonnull UpdatableIndex byClientId) + throws CommitException; +} diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/committers/PrincipalsChangeCommitterWrapper.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/committers/PrincipalsChangeCommitterWrapper.java new file mode 100644 index 0000000000..cc317b10cb --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/committers/PrincipalsChangeCommitterWrapper.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.nosql.metastore.committers; + +import static org.apache.polaris.persistence.nosql.api.index.IndexContainer.newUpdatableIndex; +import static org.apache.polaris.persistence.nosql.api.index.IndexKey.INDEX_KEY_SERIALIZER; +import static org.apache.polaris.persistence.nosql.api.obj.ObjRef.OBJ_REF_SERIALIZER; + +import jakarta.annotation.Nonnull; +import java.util.Optional; +import java.util.function.Supplier; +import org.apache.polaris.persistence.nosql.api.commit.CommitException; +import org.apache.polaris.persistence.nosql.api.commit.CommitRetryable; +import org.apache.polaris.persistence.nosql.api.commit.CommitterState; +import org.apache.polaris.persistence.nosql.coretypes.ContainerObj; +import org.apache.polaris.persistence.nosql.coretypes.principals.PrincipalsObj; + +/** + * Abstracts common {@link ContainerObj#stableIdToName()} and {@link ContainerObj#nameToObjRef()} + * handling for committing operations, for principals. + * + * @param result of the commiting operation + */ +public record PrincipalsChangeCommitterWrapper( + PrincipalsChangeCommitter changeCommitter) + implements CommitRetryable { + + @SuppressWarnings("DuplicatedCode") + @Nonnull + @Override + public Optional attempt( + @Nonnull CommitterState state, + @Nonnull Supplier> refObjSupplier) + throws CommitException { + var refObj = refObjSupplier.get(); + var byName = + refObj + .map(ContainerObj::nameToObjRef) + .map(c -> c.asUpdatableIndex(state.persistence(), OBJ_REF_SERIALIZER)) + .orElseGet(() -> newUpdatableIndex(state.persistence(), OBJ_REF_SERIALIZER)); + var byId = + refObj + .map(ContainerObj::stableIdToName) + .map(c -> c.asUpdatableIndex(state.persistence(), INDEX_KEY_SERIALIZER)) + .orElseGet(() -> newUpdatableIndex(state.persistence(), INDEX_KEY_SERIALIZER)); + var byClientId = + refObj + .map(PrincipalsObj::byClientId) + .map(c -> c.asUpdatableIndex(state.persistence(), OBJ_REF_SERIALIZER)) + .orElseGet(() -> newUpdatableIndex(state.persistence(), OBJ_REF_SERIALIZER)); + + var ref = PrincipalsObj.builder(); + refObj.ifPresent(ref::from); + + var r = changeCommitter.change(state, ref, byName, byId, byClientId); + + if (r instanceof ChangeResult.CommitChange(RESULT result)) { + ref.byClientId(byClientId.toIndexed("idx-client-id-", state::writeOrReplace)) + .nameToObjRef(byName.toIndexed("idx-name-", state::writeOrReplace)) + .stableIdToName(byId.toIndexed("idx-id-", state::writeOrReplace)); + return state.commitResult(result, ref, refObj); + } + if (r instanceof ChangeResult.NoChange(RESULT result)) { + return state.noCommit(result); + } + throw new IllegalStateException("" + r); + } +} diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/indexaccess/IndexUtils.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/indexaccess/IndexUtils.java new file mode 100644 index 0000000000..c5f385d174 --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/indexaccess/IndexUtils.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.persistence.nosql.metastore.indexaccess; + +import static org.apache.polaris.persistence.nosql.metastore.ContentIdentifier.indexKeyToIdentifier; + +import org.apache.polaris.persistence.nosql.api.index.Index; +import org.apache.polaris.persistence.nosql.api.index.IndexKey; +import org.apache.polaris.persistence.nosql.api.obj.ObjRef; +import org.apache.polaris.persistence.nosql.metastore.ContentIdentifier; + +public final class IndexUtils { + private IndexUtils() {} + + public static boolean hasChildren( + long catalogId, Index nameIndex, Index stableIdIndex, long parentId) { + if (nameIndex != null) { + if (parentId == 0L || parentId == catalogId) { + return nameIndex.iterator().hasNext(); + } else { + var parentNameKey = + stableIdIndex != null ? stableIdIndex.get(IndexKey.key(parentId)) : null; + if (parentNameKey != null) { + var iter = nameIndex.iterator(parentNameKey, null, false); + // skip the parent itself + iter.next(); + if (iter.hasNext()) { + var e = iter.next(); + var nextKey = e.getKey(); + var parentIdent = indexKeyToIdentifier(parentNameKey); + var nextIdent = indexKeyToIdentifier(nextKey); + return nextIdent.parent().equals(parentIdent); + } + } + } + } + return false; + } + + public static boolean hasChildren(Index nameIndex, ContentIdentifier ident) { + if (ident.isEmpty()) { + return nameIndex.iterator().hasNext(); + } + var key = ident.toIndexKey(); + + var iter = nameIndex.iterator(key, null, false); + // skip the parent itself + iter.next(); + if (iter.hasNext()) { + var e = iter.next(); + var nextKey = e.getKey(); + var nextIdent = indexKeyToIdentifier(nextKey); + return nextIdent.parent().equals(ident); + } + return false; + } +} diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/indexaccess/IndexedContainerAccess.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/indexaccess/IndexedContainerAccess.java new file mode 100644 index 0000000000..99b0bfd073 --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/indexaccess/IndexedContainerAccess.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.nosql.metastore.indexaccess; + +import java.util.Optional; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.persistence.nosql.api.Persistence; +import org.apache.polaris.persistence.nosql.api.index.Index; +import org.apache.polaris.persistence.nosql.api.index.IndexKey; +import org.apache.polaris.persistence.nosql.api.obj.ObjRef; +import org.apache.polaris.persistence.nosql.coretypes.ContainerObj; +import org.apache.polaris.persistence.nosql.coretypes.ObjBase; +import org.apache.polaris.persistence.nosql.coretypes.mapping.EntityObjMappings; + +/** Helper class to access the indexes on a {@link ContainerObj}. */ +public abstract class IndexedContainerAccess { + + private static final int ROOT_ENTITY_TYPE_CODE = PolarisEntityType.ROOT.getCode(); + + public static IndexedContainerAccess indexedAccessDirect( + Persistence persistence, ObjRef containerObjRef) { + return new IndexedContainerAccessImpl<>(persistence, ContainerObj.class, containerObjRef); + } + + public static IndexedContainerAccess indexedAccessForEntityType( + int entityTypeCode, Persistence persistence, long catalogStableId) { + if (entityTypeCode != ROOT_ENTITY_TYPE_CODE) { + var mapping = EntityObjMappings.byEntityTypeCode(entityTypeCode); + catalogStableId = mapping.fixCatalogId(catalogStableId); + var refName = mapping.refNameForCatalog(catalogStableId); + var containerObjType = mapping.containerObjTypeClass(); + return new IndexedContainerAccessImpl<>( + persistence, refName, containerObjType, catalogStableId); + } else { + return new IndexedContainerAccessRoot<>(persistence); + } + } + + /** Checks whether the known reference head is stale. */ + public abstract boolean isStale(); + + public abstract Optional refObj(); + + public abstract Optional byId(long stableId); + + public abstract Optional nameKeyById(long stableId); + + public abstract Optional byParentIdAndName(long parentId, String name); + + public abstract Optional byNameOnRoot(String name); + + public abstract Optional> nameIndex(); + + public abstract Optional> stableIdIndex(); + + public abstract long catalogStableId(); +} diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/indexaccess/IndexedContainerAccessImpl.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/indexaccess/IndexedContainerAccessImpl.java new file mode 100644 index 0000000000..cd94102cba --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/indexaccess/IndexedContainerAccessImpl.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.persistence.nosql.metastore.indexaccess; + +import static org.apache.polaris.persistence.nosql.api.index.IndexKey.INDEX_KEY_SERIALIZER; +import static org.apache.polaris.persistence.nosql.api.obj.ObjRef.OBJ_REF_SERIALIZER; +import static org.apache.polaris.persistence.nosql.metastore.ContentIdentifier.indexKeyToIdentifierBuilder; + +import com.google.common.base.Suppliers; +import java.util.Optional; +import java.util.function.Supplier; +import org.apache.polaris.persistence.nosql.api.Persistence; +import org.apache.polaris.persistence.nosql.api.index.Index; +import org.apache.polaris.persistence.nosql.api.index.IndexKey; +import org.apache.polaris.persistence.nosql.api.obj.ObjRef; +import org.apache.polaris.persistence.nosql.coretypes.ContainerObj; +import org.apache.polaris.persistence.nosql.coretypes.ObjBase; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +final class IndexedContainerAccessImpl extends IndexedContainerAccess { + private static final Logger LOGGER = LoggerFactory.getLogger(IndexedContainerAccessImpl.class); + + private final ObjRef containerObjRef; + private final String referenceName; + private final Class referenceObjType; + private final long catalogStableId; + private Optional refObj; + private final Supplier>> nameIndexSupplier = + Suppliers.memoize(this::supplyNameIndex); + private final Supplier>> idIndexSupplier = + Suppliers.memoize(this::supplyIdIndex); + private final Persistence persistence; + + IndexedContainerAccessImpl( + Persistence persistence, + String referenceName, + Class referenceObjType, + long catalogStableId) { + this.persistence = persistence; + this.referenceName = referenceName; + this.referenceObjType = referenceObjType; + this.catalogStableId = catalogStableId; + this.containerObjRef = null; + } + + IndexedContainerAccessImpl( + Persistence persistence, Class containerObjClass, ObjRef containerObjId) { + this.persistence = persistence; + this.referenceName = null; + this.referenceObjType = containerObjClass; + this.containerObjRef = containerObjId; + this.catalogStableId = -1; + } + + @SuppressWarnings("OptionalAssignedToNull") + @Override + public boolean isStale() { + var r = refObj; + if (r == null || referenceName == null) { + return false; + } + var current = persistence.fetchReference(referenceName); + return !r.map(ContainerObj::id).equals(current.pointer().map(ObjRef::id)); + } + + @SuppressWarnings("OptionalAssignedToNull") + @Override + public Optional refObj() { + if (this.refObj == null) { + if (referenceName != null) { + this.refObj = persistence.fetchReferenceHead(referenceName, referenceObjType); + LOGGER.debug("Fetched head {} for reference '{}'", refObj, referenceName); + } else if (containerObjRef != null) { + this.refObj = Optional.ofNullable(persistence.fetch(containerObjRef, referenceObjType)); + } else { + // Should really never ever happen + throw new IllegalStateException(); + } + } + return this.refObj; + } + + @Override + public Optional byId(long stableId) { + return objRefById(stableId).flatMap(this::objByRef); + } + + @Override + public Optional nameKeyById(long stableId) { + return stableIdIndex() + .flatMap( + idIndex -> { + var nameKey = idIndex.get(IndexKey.key(stableId)); + return Optional.ofNullable(nameKey); + }); + } + + @Override + public Optional byParentIdAndName(long parentId, String name) { + return (parentId != 0L + ? nameKeyById(parentId) + .flatMap( + parentKey -> { + var fullIdentifier = + indexKeyToIdentifierBuilder(parentKey).addElements(name).build(); + return objRefByName(fullIdentifier.toIndexKey()); + }) + : objRefByName(IndexKey.key(name))) + .flatMap(this::objByRef); + } + + @Override + public Optional byNameOnRoot(String name) { + return objRefByName(IndexKey.key(name)).flatMap(this::objByRef); + } + + @Override + public Optional> nameIndex() { + return nameIndexSupplier.get(); + } + + @Override + public Optional> stableIdIndex() { + return idIndexSupplier.get(); + } + + @Override + public long catalogStableId() { + return catalogStableId; + } + + private Optional objRefById(long stableId) { + return nameKeyById(stableId).flatMap(this::objRefByName); + } + + private Optional objByRef(ObjRef objRef) { + return Optional.ofNullable(persistence.fetch(objRef, ObjBase.class)); + } + + private Optional objRefByName(IndexKey nameKey) { + return nameIndex().flatMap(nameIdx -> Optional.ofNullable(nameIdx.get(nameKey))); + } + + private Optional> supplyNameIndex() { + return refObj().map(ref -> ref.nameToObjRef().indexForRead(persistence, OBJ_REF_SERIALIZER)); + } + + private Optional> supplyIdIndex() { + return refObj() + .map(ref -> ref.stableIdToName().indexForRead(persistence, INDEX_KEY_SERIALIZER)); + } +} diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/indexaccess/IndexedContainerAccessRoot.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/indexaccess/IndexedContainerAccessRoot.java new file mode 100644 index 0000000000..b23eb5cdc6 --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/indexaccess/IndexedContainerAccessRoot.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.persistence.nosql.metastore.indexaccess; + +import static org.apache.polaris.core.entity.PolarisEntityConstants.getRootContainerName; +import static org.apache.polaris.core.entity.PolarisEntityConstants.getRootEntityId; +import static org.apache.polaris.persistence.nosql.coretypes.realm.RootObj.ROOT_REF_NAME; + +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.Optional; +import java.util.function.Supplier; +import org.apache.polaris.persistence.nosql.api.Persistence; +import org.apache.polaris.persistence.nosql.api.index.Index; +import org.apache.polaris.persistence.nosql.api.index.IndexKey; +import org.apache.polaris.persistence.nosql.api.obj.ObjRef; +import org.apache.polaris.persistence.nosql.coretypes.ContainerObj; +import org.apache.polaris.persistence.nosql.coretypes.ObjBase; + +/** + * Special implementation for the "root entity". + * + *

There is exactly one root-entity, never more. With this, the root-entity does not require + * management via a {@link ContainerObj}. This requires a custom implementation. + */ +final class IndexedContainerAccessRoot extends IndexedContainerAccess { + private static final IndexKey nameKey = IndexKey.key(getRootContainerName()); + private static final IndexKey idKey = IndexKey.key(getRootEntityId()); + + private final Persistence persistence; + + private Optional root; + + IndexedContainerAccessRoot(Persistence persistence) { + this.persistence = persistence; + } + + @SuppressWarnings("OptionalAssignedToNull") + private Optional rootLazy() { + if (root == null) { + root = persistence.fetchReferenceHead(ROOT_REF_NAME, ObjBase.class); + } + return root; + } + + @Override + @SuppressWarnings("OptionalAssignedToNull") + public boolean isStale() { + var r = root; + if (r == null) { + return false; + } + var current = persistence.fetchReference(ROOT_REF_NAME); + return !r.map(ObjBase::id).equals(current.pointer().map(ObjRef::id)); + } + + @Override + public long catalogStableId() { + return 0L; + } + + @Override + public Optional> stableIdIndex() { + return Optional.of(new SingletonIndex<>(idKey, () -> nameKey)); + } + + @Override + public Optional> nameIndex() { + return Optional.of( + new SingletonIndex<>(nameKey, () -> rootLazy().map(ObjRef::objRef).orElse(null))); + } + + @Override + public Optional byNameOnRoot(String name) { + if (name.equals(getRootContainerName())) { + return rootLazy(); + } + return Optional.empty(); + } + + @Override + public Optional byParentIdAndName(long parentId, String name) { + if (parentId == 0L) { + return byNameOnRoot(name); + } + return Optional.empty(); + } + + @Override + public Optional nameKeyById(long stableId) { + return stableId == 0L ? Optional.of(nameKey) : Optional.empty(); + } + + @Override + public Optional byId(long stableId) { + if (stableId == 0L) { + return rootLazy(); + } + return Optional.empty(); + } + + @Override + public Optional refObj() { + throw new UnsupportedOperationException(); + } + + static final class SingletonIndex implements Index { + private final IndexKey key; + private final Supplier valueSupplier; + private volatile T value; + + SingletonIndex(IndexKey key, Supplier value) { + this.key = key; + this.valueSupplier = value; + } + + @Override + public void prefetchIfNecessary(Iterable keys) {} + + @Override + public boolean contains(IndexKey key) { + return this.key.equals(key); + } + + private T value() { + var v = value; + if (v == null) { + value = v = valueSupplier.get(); + } + return v; + } + + @Nullable + @Override + public T get(@Nonnull IndexKey key) { + return this.key.equals(key) ? value() : null; + } + + @Override + @Nonnull + public Iterator> iterator() { + return Collections.singletonList(Map.entry(key, value())).iterator(); + } + + @Nonnull + @Override + public Iterator> iterator( + @Nullable IndexKey lower, @Nullable IndexKey higher, boolean prefetch) { + // this is technically incorrect, but no need to implement this for now + return iterator(); + } + + @Nonnull + @Override + public Iterator> reverseIterator() { + return iterator(); + } + + @Nonnull + @Override + public Iterator> reverseIterator( + @Nullable IndexKey lower, @Nullable IndexKey higher, boolean prefetch) { + // this is technically incorrect, but no need to implement this for now + return reverseIterator(); + } + } +} diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/indexaccess/MemoizedIndexedAccess.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/indexaccess/MemoizedIndexedAccess.java new file mode 100644 index 0000000000..d559e9360b --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/indexaccess/MemoizedIndexedAccess.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.nosql.metastore.indexaccess; + +import static org.apache.polaris.persistence.nosql.api.obj.ObjRef.OBJ_REF_SERIALIZER; +import static org.apache.polaris.persistence.nosql.coretypes.mapping.EntityObjMappings.byEntityTypeCode; +import static org.apache.polaris.persistence.nosql.coretypes.realm.RealmGrantsObj.REALM_GRANTS_REF_NAME; +import static org.apache.polaris.persistence.nosql.metastore.indexaccess.IndexedContainerAccess.indexedAccessForEntityType; + +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.persistence.nosql.api.Persistence; +import org.apache.polaris.persistence.nosql.api.index.Index; +import org.apache.polaris.persistence.nosql.api.obj.BaseCommitObj; +import org.apache.polaris.persistence.nosql.api.obj.Obj; +import org.apache.polaris.persistence.nosql.api.obj.ObjRef; +import org.apache.polaris.persistence.nosql.coretypes.ContainerObj; +import org.apache.polaris.persistence.nosql.coretypes.acl.GrantsObj; +import org.apache.polaris.persistence.nosql.coretypes.catalog.CatalogStateObj; + +/** + * Memoizes {@link IndexedContainerAccess} instances by catalog-ID and type-code, and reference + * heads by reference name for a {@code NoSqlMetaStore} instance. + * + *

Both serve different use cases. Reference head memoization is not used for index + * access. + * + *

{@link IndexedContainerAccess} instances and reference heads are memoized + * independently. Invalidating by catalog-ID and type-code does not invalidate a memoized + * reference head for the corresponding entity type. + * + *

Memoizing these instances avoids unnecessary reference-head, index lookups and index + * deserialization, even if backed by the persistence cache. Committing functions must + * always call the appropriate {@code invalidate*()} functions. + */ +public final class MemoizedIndexedAccess { + private static final int CATALOG_CONTENT_CODE = -1; + // Just need one type code for a catalog-content type. + // This is an implementation-specific choice. + // Could also be TABLE_LIKE or POLICY - it doesn't matter for this use case, the reference name + // and container-types are the same. + private static final int CATALOG_CONTENT_ENTITY_TYPE_CODE = PolarisEntityType.NAMESPACE.getCode(); + + private final Persistence persistence; + + /** + * Memoizes objects already accessed by the holding {@code PersistenceMetaStore} instance. + * + *

The {@link Index} instances held via this map are thread-safe + */ + private final Map> map = new ConcurrentHashMap<>(); + + private final Map> refHeads = new ConcurrentHashMap<>(); + + private record Key(long catalogId, int typeCode) { + private Key(long catalogId, int typeCode) { + var mapping = byEntityTypeCode(typeCode); + this.catalogId = mapping.fixCatalogId(catalogId); + this.typeCode = mapping.catalogContent() ? CATALOG_CONTENT_CODE : typeCode; + } + } + + /** + * Constructs a new {@link MemoizedIndexedAccess} instance. + * + * @param persistence persistence instance to use + */ + public static MemoizedIndexedAccess newMemoizedIndexedAccess(Persistence persistence) { + return new MemoizedIndexedAccess(persistence); + } + + private MemoizedIndexedAccess(Persistence persistence) { + this.persistence = persistence; + } + + public IndexedContainerAccess indexedAccessDirect(ObjRef containerObjRef) { + return IndexedContainerAccess.indexedAccessDirect(persistence, containerObjRef); + } + + public IndexedContainerAccess indexedAccess( + long catalogId, int entityTypeCode) { + var key = new Key(catalogId, entityTypeCode); + var access = + map.compute( + key, + (k, current) -> { + if (current == null || current.isStale()) { + return indexedAccessForEntityType(entityTypeCode, persistence, catalogId); + } + return current; + }); + @SuppressWarnings("unchecked") + var r = (IndexedContainerAccess) access; + return r; + } + + public IndexedContainerAccess catalogContent(long catalogId) { + return indexedAccess(catalogId, CATALOG_CONTENT_ENTITY_TYPE_CODE); + } + + public void invalidateCatalogContent(long catalogId) { + invalidateIndexedAccess(catalogId, CATALOG_CONTENT_ENTITY_TYPE_CODE); + } + + public void invalidateIndexedAccess(long catalogId, int entityTypeCode) { + var key = new Key(catalogId, entityTypeCode); + map.remove(key); + } + + record MemoizedGrants(long headId, Optional> securablesIndex) {} + + private volatile MemoizedGrants memoizedGrants; + + public Optional> grantsIndex() { + var current = memoizedGrants; + var head = referenceHead(REALM_GRANTS_REF_NAME, GrantsObj.class); + if (current == null || head.map(GrantsObj::id).orElse(-1L) != current.headId) { + if (head.isPresent()) { + var grantsObj = head.get(); + var securablesIndex = grantsObj.acls().indexForRead(persistence, OBJ_REF_SERIALIZER); + current = new MemoizedGrants(grantsObj.id(), Optional.of(securablesIndex)); + } else { + current = new MemoizedGrants(-1L, Optional.empty()); + } + memoizedGrants = current; + } + + return current.securablesIndex(); + } + + public void invalidateGrantsIndex() { + memoizedGrants = null; + invalidateReferenceHead(REALM_GRANTS_REF_NAME); + } + + @SuppressWarnings("OptionalAssignedToNull") + public Optional referenceHead(String refName, Class type) { + return cast( + refHeads.compute( + refName, + (r, current) -> { + if (current != null + && current + .map(Obj::id) + .equals(persistence.fetchReference(r).pointer().map(ObjRef::id))) { + return current; + } + return persistence.fetchReferenceHead(r, type); + })); + } + + public void invalidateReferenceHead(String refName) { + refHeads.remove(refName); + } + + @SuppressWarnings("unchecked") + private static R cast(Object o) { + return (R) o; + } +} diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/EntityUpdate.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/EntityUpdate.java new file mode 100644 index 0000000000..2534ee741b --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/EntityUpdate.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.nosql.metastore.mutation; + +import org.apache.polaris.core.entity.PolarisBaseEntity; + +public record EntityUpdate(Operation operation, PolarisBaseEntity entity, boolean cleanup) { + public EntityUpdate(Operation operation, PolarisBaseEntity entity) { + this(operation, entity, false); + } + + public enum Operation { + CREATE, + UPDATE, + DELETE + } +} diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/GrantsMutation.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/GrantsMutation.java new file mode 100644 index 0000000000..1174e1a7a7 --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/GrantsMutation.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.persistence.nosql.metastore.mutation; + +import static org.apache.polaris.persistence.nosql.api.index.IndexContainer.newUpdatableIndex; +import static org.apache.polaris.persistence.nosql.api.obj.ObjRef.OBJ_REF_SERIALIZER; +import static org.apache.polaris.persistence.nosql.api.obj.ObjRef.objRef; +import static org.apache.polaris.persistence.nosql.coretypes.realm.RealmGrantsObj.REALM_GRANTS_REF_NAME; + +import jakarta.annotation.Nonnull; +import java.util.Optional; +import java.util.function.Supplier; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.persistence.nosql.api.Persistence; +import org.apache.polaris.persistence.nosql.api.commit.CommitException; +import org.apache.polaris.persistence.nosql.api.commit.CommitRetryable; +import org.apache.polaris.persistence.nosql.api.commit.CommitterState; +import org.apache.polaris.persistence.nosql.api.index.IndexKey; +import org.apache.polaris.persistence.nosql.api.index.UpdatableIndex; +import org.apache.polaris.persistence.nosql.api.obj.BaseCommitObj; +import org.apache.polaris.persistence.nosql.api.obj.ObjRef; +import org.apache.polaris.persistence.nosql.authz.api.Privilege; +import org.apache.polaris.persistence.nosql.authz.api.Privileges; +import org.apache.polaris.persistence.nosql.coretypes.acl.AclObj; +import org.apache.polaris.persistence.nosql.coretypes.acl.GrantsObj; +import org.apache.polaris.persistence.nosql.coretypes.realm.RealmGrantsObj; +import org.apache.polaris.persistence.nosql.metastore.indexaccess.MemoizedIndexedAccess; +import org.apache.polaris.persistence.nosql.metastore.privs.GrantTriplet; +import org.apache.polaris.persistence.nosql.metastore.privs.SecurableGranteePrivilegeTuple; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public record GrantsMutation( + Persistence persistence, + MemoizedIndexedAccess memoizedIndexedAccess, + Privileges privileges, + boolean doGrant, + SecurableGranteePrivilegeTuple... grants) { + private static final Logger LOGGER = LoggerFactory.getLogger(GrantsMutation.class); + + public boolean apply() { + try { + return persistence + .createCommitter(REALM_GRANTS_REF_NAME, GrantsObj.class, String.class) + .synchronizingLocally() + .commitRuntimeException( + new CommitRetryable<>() { + @Nonnull + @Override + public Optional attempt( + @Nonnull CommitterState state, + @Nonnull Supplier> refObjSupplier) + throws CommitException { + var persistence = state.persistence(); + var refObj = refObjSupplier.get(); + + var ref = RealmGrantsObj.builder(); + refObj.ifPresent(ref::from); + + var securablesIndex = + refObj + .map(GrantsObj::acls) + .map(c -> c.asUpdatableIndex(persistence, OBJ_REF_SERIALIZER)) + .orElseGet(() -> newUpdatableIndex(persistence, OBJ_REF_SERIALIZER)); + + var changed = false; + for (var g : grants) { + var securable = GrantTriplet.forEntity(g.securable()); + var grantee = GrantTriplet.forEntity(g.grantee()); + var forSec = grantee.asDirected(); + var privilege = privileges.byName(g.privilege().name()); + changed |= + processGrant(state, securable, forSec, securablesIndex, privilege, doGrant); + changed |= + processGrant( + state, grantee, securable, securablesIndex, privilege, doGrant); + } + + if (!changed) { + return state.noCommit(); + } + + ref.acls(securablesIndex.toIndexed("idx-sec-", state::writeOrReplace)); + + return commitResult(state, ref, refObj); + } + + // Some fun with Java generics... + @SuppressWarnings({"unchecked", "rawtypes"}) + private static Optional commitResult( + CommitterState state, + REF_BUILDER ref, + Optional refObj) { + var cs = (CommitterState) state; + var refBuilder = (BaseCommitObj.Builder) ref; + return cs.commitResult("", refBuilder, refObj); + } + + private boolean processGrant( + CommitterState state, + GrantTriplet aclTriplet, + GrantTriplet grantee, + UpdatableIndex securablesIndex, + Privilege privilege, + boolean doGrant) { + + var aclName = aclTriplet.toRoleName(); + var granteeRoleName = grantee.toRoleName(); + + LOGGER.trace( + "{} {} {} '{}' ({}) on '{}' in ACL '{}' ({})", + doGrant ? "Granting" : "Revoking", + privilege.name(), + doGrant ? "on" : "from", + granteeRoleName, + PolarisEntityType.fromCode(grantee.typeCode()), + REALM_GRANTS_REF_NAME, + aclName, + PolarisEntityType.fromCode(aclTriplet.typeCode())); + + var aclKey = IndexKey.key(aclName); + + var aclRef = securablesIndex.get(aclKey); + var aclObjOptional = + Optional.ofNullable(aclRef) + .map(r -> state.persistence().fetch(r, AclObj.class)); + var aclObjBuilder = + aclObjOptional + .map(AclObj.builder()::from) + .orElseGet(AclObj::builder) + .id(persistence.generateId()) + .securableId(aclTriplet.id()) + .securableTypeCode(aclTriplet.typeCode()); + + var aclBuilder = + aclObjOptional + .map(o -> privileges.newAclBuilder().from(o.acl())) + .orElseGet(privileges::newAclBuilder); + + aclBuilder.modify( + granteeRoleName, + aclEntryBuilder -> { + if (doGrant) { + aclEntryBuilder.grant(privilege); + } else { + aclEntryBuilder.revoke(privilege); + } + }); + + var acl = aclBuilder.build(); + if (aclObjOptional.map(obj -> obj.acl().equals(acl)).orElseGet(() -> !doGrant)) { + // aclObj not changed + return false; + } + aclObjBuilder.acl(acl); + + var aclObj = aclObjBuilder.build(); + + state.writeOrReplace("acl-" + aclTriplet.id(), aclObj); + + securablesIndex.put(aclKey, objRef(aclObj)); + + // aclObj changed + return true; + } + }) + .isPresent(); + } finally { + memoizedIndexedAccess.invalidateGrantsIndex(); + } + } +} diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/MutationAttempt.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/MutationAttempt.java new file mode 100644 index 0000000000..a34853a017 --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/MutationAttempt.java @@ -0,0 +1,593 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.persistence.nosql.metastore.mutation; + +import static com.google.common.base.Preconditions.checkState; +import static java.util.Objects.requireNonNull; +import static org.apache.polaris.core.entity.PolarisEntityConstants.ENTITY_BASE_LOCATION; +import static org.apache.polaris.core.persistence.dao.entity.BaseResult.ReturnStatus.CATALOG_NOT_EMPTY; +import static org.apache.polaris.core.persistence.dao.entity.BaseResult.ReturnStatus.ENTITY_ALREADY_EXISTS; +import static org.apache.polaris.core.persistence.dao.entity.BaseResult.ReturnStatus.ENTITY_CANNOT_BE_RENAMED; +import static org.apache.polaris.core.persistence.dao.entity.BaseResult.ReturnStatus.ENTITY_NOT_FOUND; +import static org.apache.polaris.core.persistence.dao.entity.BaseResult.ReturnStatus.ENTITY_UNDROPPABLE; +import static org.apache.polaris.core.persistence.dao.entity.BaseResult.ReturnStatus.NAMESPACE_NOT_EMPTY; +import static org.apache.polaris.core.persistence.dao.entity.BaseResult.ReturnStatus.POLICY_HAS_MAPPINGS; +import static org.apache.polaris.core.persistence.dao.entity.BaseResult.ReturnStatus.TARGET_ENTITY_CONCURRENTLY_MODIFIED; +import static org.apache.polaris.persistence.nosql.api.obj.ObjRef.objRef; +import static org.apache.polaris.persistence.nosql.coretypes.catalog.EntityIdSet.entityIdSet; +import static org.apache.polaris.persistence.nosql.coretypes.mapping.EntityObjMappings.entitySubTypeCodeFromObjType; +import static org.apache.polaris.persistence.nosql.coretypes.mapping.EntityObjMappings.mapToEntity; +import static org.apache.polaris.persistence.nosql.coretypes.mapping.EntityObjMappings.mapToObj; +import static org.apache.polaris.persistence.nosql.coretypes.mapping.EntityObjMappings.maybeObjToPolarisPrincipalSecrets; +import static org.apache.polaris.persistence.nosql.coretypes.mapping.EntityObjMappings.objTypeForPolarisType; +import static org.apache.polaris.persistence.nosql.coretypes.realm.PolicyMapping.POLICY_MAPPING_SERIALIZER; +import static org.apache.polaris.persistence.nosql.coretypes.realm.PolicyMappingsObj.POLICY_MAPPINGS_REF_NAME; +import static org.apache.polaris.persistence.nosql.metastore.ContentIdentifier.identifierFromLocationString; +import static org.apache.polaris.persistence.nosql.metastore.ContentIdentifier.indexKeyToIdentifierBuilder; +import static org.apache.polaris.persistence.nosql.metastore.indexaccess.IndexUtils.hasChildren; +import static org.apache.polaris.persistence.nosql.metastore.mutation.MutationResults.newMutableMutationResults; + +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Consumer; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisEntityCore; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.entity.PolarisPrincipalSecrets; +import org.apache.polaris.core.persistence.dao.entity.BaseResult; +import org.apache.polaris.core.storage.StorageLocation; +import org.apache.polaris.persistence.nosql.api.commit.CommitterState; +import org.apache.polaris.persistence.nosql.api.index.IndexKey; +import org.apache.polaris.persistence.nosql.api.index.UpdatableIndex; +import org.apache.polaris.persistence.nosql.api.obj.ObjRef; +import org.apache.polaris.persistence.nosql.coretypes.ContainerObj; +import org.apache.polaris.persistence.nosql.coretypes.ObjBase; +import org.apache.polaris.persistence.nosql.coretypes.catalog.EntityIdSet; +import org.apache.polaris.persistence.nosql.coretypes.changes.Change; +import org.apache.polaris.persistence.nosql.coretypes.changes.ChangeAdd; +import org.apache.polaris.persistence.nosql.coretypes.changes.ChangeRemove; +import org.apache.polaris.persistence.nosql.coretypes.changes.ChangeRename; +import org.apache.polaris.persistence.nosql.coretypes.changes.ChangeUpdate; +import org.apache.polaris.persistence.nosql.coretypes.content.PolicyObj; +import org.apache.polaris.persistence.nosql.coretypes.realm.PolicyMappingsObj; +import org.apache.polaris.persistence.nosql.metastore.ContentIdentifier; +import org.apache.polaris.persistence.nosql.metastore.committers.ChangeResult; +import org.apache.polaris.persistence.nosql.metastore.indexaccess.MemoizedIndexedAccess; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public record MutationAttempt( + UpdateKeyForCatalogAndEntityType updateKeyForCatalogAndEntityType, + List updates, + CommitterState state, + UpdatableIndex byName, + UpdatableIndex byId, + UpdatableIndex changes, + UpdatableIndex locations, + MemoizedIndexedAccess memoizedIndexedAccess) { + + private static final Logger LOGGER = LoggerFactory.getLogger(MutationAttempt.class); + + public static ObjBase objForChangeComparison( + PolarisBaseEntity entity, + Optional currentSecrets, + ObjBase originalObj) { + return mapToObj(entity, currentSecrets) + .updateTimestamp(originalObj.createTimestamp()) + .id(originalObj.id()) + .numParts(originalObj.numParts()) + .entityVersion(originalObj.entityVersion()) + .createTimestamp(originalObj.createTimestamp()) + .build(); + } + + public ChangeResult apply() { + var mutationResults = newMutableMutationResults(); + for (var update : updates) { + LOGGER.debug("Processing update {}", update); + + switch (update.operation()) { + case CREATE -> + applyEntityCreateMutation( + updateKeyForCatalogAndEntityType, + state, + byName, + byId, + changes, + locations, + update, + mutationResults); + case UPDATE -> + applyEntityUpdateMutation( + updateKeyForCatalogAndEntityType, + state, + byName, + byId, + changes, + locations, + update, + mutationResults); + case DELETE -> + applyEntityDeleteMutation( + updateKeyForCatalogAndEntityType, + state, + byName, + byId, + changes, + locations, + update, + mutationResults, + memoizedIndexedAccess); + default -> throw new IllegalStateException("Unexpected operation " + update.operation()); + } + } + + var doCommit = mutationResults.anyChange && !mutationResults.hardFailure; + LOGGER.debug( + "{} changes (has changes: {}, failures: {})", + doCommit ? "Committing" : "Not committing", + mutationResults.anyChange, + mutationResults.failuresAsString()); + + return doCommit + ? new ChangeResult.CommitChange<>(mutationResults) + : new ChangeResult.NoChange<>(mutationResults); + } + + private static void applyEntityDeleteMutation( + UpdateKeyForCatalogAndEntityType updateKeyForCatalogAndEntityType, + CommitterState state, + UpdatableIndex byName, + UpdatableIndex byId, + UpdatableIndex changes, + UpdatableIndex locations, + EntityUpdate update, + MutationResults mutationResults, + MemoizedIndexedAccess memoizedIndexedAccess) { + var entity = update.entity(); + var entityType = entity.getType(); + var persistence = state.persistence(); + + var entityIdKey = IndexKey.key(entity.getId()); + var originalNameKey = byId.get(entityIdKey); + + if (originalNameKey == null) { + mutationResults.dropResult(ENTITY_NOT_FOUND); + return; + } + var originalRef = byName.get(originalNameKey); + if (originalRef == null) { + mutationResults.dropResult(ENTITY_NOT_FOUND); + return; + } + var originalObj = + (ObjBase) + state + .persistence() + .fetch( + originalRef, + objTypeForPolarisType(entityType, entity.getSubType()).targetClass()); + if (originalObj == null) { + mutationResults.dropResult(ENTITY_NOT_FOUND); + return; + } + if (entity.getEntityVersion() != originalObj.entityVersion()) { + mutationResults.dropResult(TARGET_ENTITY_CONCURRENTLY_MODIFIED); + return; + } + if (entity.cannotBeDroppedOrRenamed()) { + mutationResults.dropResult(ENTITY_UNDROPPABLE); + return; + } + + updateLocationsIndex(locations, originalObj, null); + + var ok = + switch (entityType) { + case NAMESPACE -> { + if (hasChildren( + updateKeyForCatalogAndEntityType.catalogId(), byName, byId, entity.getId())) { + mutationResults.dropResult(NAMESPACE_NOT_EMPTY); + yield false; + } + yield true; + } + case CATALOG -> { + var catalogState = memoizedIndexedAccess.catalogContent(entity.getId()); + + if (catalogState.nameIndex().map(idx -> idx.iterator().hasNext()).orElse(false)) { + mutationResults.dropResult(NAMESPACE_NOT_EMPTY); + yield false; + } + + // VALIDATION LOGIC COPIED + + var catalogRolesAccess = + memoizedIndexedAccess.indexedAccess( + entity.getId(), PolarisEntityType.CATALOG_ROLE.getCode()); + var numCatalogRoles = + catalogRolesAccess + .nameIndex() + .map( + idx -> { + var iter = idx.iterator(); + var cnt = 0; + if (iter.hasNext()) { + iter.next(); + cnt++; + } + if (iter.hasNext()) { + iter.next(); + cnt++; + } + return cnt; + }) + .orElse(0); + + // If we have 2, we cannot drop the catalog. + // If only one left, better be the admin role + if (numCatalogRoles > 1) { + mutationResults.dropResult(CATALOG_NOT_EMPTY); + yield false; + } + // If 1, drop the last catalog role. + // Should be the catalog admin role, but don't validate this. + // (No need to drop the catalog role here, it'll be eventually done by + // persistence-maintenance!) + + yield true; + } + case POLICY -> + memoizedIndexedAccess + .referenceHead(POLICY_MAPPINGS_REF_NAME, PolicyMappingsObj.class) + .map( + policyMappingsObj -> { + var index = + policyMappingsObj + .policyMappings() + .indexForRead(persistence, POLICY_MAPPING_SERIALIZER); + + var prefixKey = policyIndexPrefixKey((PolicyObj) originalObj, entity); + + var iter = index.iterator(prefixKey, prefixKey, false); + + if (iter.hasNext() && !update.cleanup()) { + mutationResults.dropResult(POLICY_HAS_MAPPINGS); + return false; + } + + while (iter.hasNext()) { + var elem = iter.next(); + var key = PolicyMappingsObj.PolicyMappingKey.fromIndexKey(elem.getKey()); + var reversed = key.reverse(); + + mutationResults.addPolicyIndexKeyToRemove(elem.getKey()); + mutationResults.addPolicyIndexKeyToRemove(reversed.toIndexKey()); + } + + return true; + }) + .orElse(true); + default -> true; + }; + if (ok) { + byId.remove(entityIdKey); + byName.remove(requireNonNull(originalNameKey)); + mutationResults.dropResult(entity); + + if (changes != null) { + changes.put(originalNameKey, ChangeRemove.builder().build()); + } + } + } + + private static void applyEntityUpdateMutation( + UpdateKeyForCatalogAndEntityType updateKeyForCatalogAndEntityType, + CommitterState state, + UpdatableIndex byName, + UpdatableIndex byId, + UpdatableIndex changes, + UpdatableIndex locations, + EntityUpdate update, + MutationResults mutationResults) { + var entity = update.entity(); + var entityType = entity.getType(); + var persistence = state.persistence(); + var now = persistence.currentInstant(); + + var entityIdKey = IndexKey.key(entity.getId()); + var originalNameKey = byId.get(entityIdKey); + + var entityParentId = entity.getParentId(); + + if (originalNameKey == null) { + mutationResults.entityResult(ENTITY_NOT_FOUND); + return; + } + var originalRef = byName.get(originalNameKey); + if (originalRef == null) { + mutationResults.entityResult(ENTITY_NOT_FOUND); + return; + } + var originalObj = + (ObjBase) + state + .persistence() + .fetch( + originalRef, + objTypeForPolarisType(entityType, entity.getSubType()).targetClass()); + if (originalObj == null) { + mutationResults.entityResult(ENTITY_NOT_FOUND); + return; + } + if (entity.getEntityVersion() != originalObj.entityVersion()) { + mutationResults.entityResult(TARGET_ENTITY_CONCURRENTLY_MODIFIED); + return; + } + + var currentSecrets = maybeObjToPolarisPrincipalSecrets(originalObj); + + var renameOrMove = + entityParentId != originalObj.parentStableId() + || !entity.getName().equals(originalObj.name()); + + if (renameOrMove) { + if (entity.cannotBeDroppedOrRenamed()) { + mutationResults.entityResult(ENTITY_CANNOT_BE_RENAMED); + return; + } + if (!byName.remove(originalNameKey)) { + mutationResults.entityResult(ENTITY_NOT_FOUND); + return; + } + + var newNameKey = nameKeyForEntity(entity, byId, mutationResults::entityResult); + if (newNameKey == null) { + return; + } + + var existingRef = byName.get(newNameKey); + if (existingRef != null) { + mutationResults.entityResult( + ENTITY_ALREADY_EXISTS, entitySubTypeCodeFromObjType(existingRef)); + return; + } + + var entityObj = + mapToObj(entity, currentSecrets) + .id(persistence.generateId()) + .updateTimestamp(now) + .entityVersion(originalObj.entityVersion() + 1) + .build(); + + updateLocationsIndex(locations, originalObj, entityObj); + + state.writeOrReplace("entity-" + entityObj.stableId(), entityObj); + + byName.put(newNameKey, objRef(entityObj)); + byId.put(entityIdKey, newNameKey); + if (changes != null) { + checkState( + changes.put(newNameKey, ChangeRename.builder().renameFrom(originalNameKey).build()), + "Entity '%s' updated more than once", + newNameKey); + } + mutationResults.entityResult( + mapToEntity(entityObj, updateKeyForCatalogAndEntityType.catalogId())); + + LOGGER.debug( + "Renamed {} '{}' with ID {} to '{}'...", + entityType, + originalNameKey, + entity.getId(), + newNameKey); + } else { + // no rename/move + + var unchangedCompareObj = objForChangeComparison(entity, currentSecrets, originalObj); + if (!unchangedCompareObj.equals(originalObj)) { + var entityObj = + mapToObj(entity, currentSecrets) + .id(persistence.generateId()) + .updateTimestamp(now) + .entityVersion(originalObj.entityVersion() + 1) + .build(); + + updateLocationsIndex(locations, originalObj, entityObj); + + state.writeOrReplace("entity-" + entityObj.stableId(), entityObj); + byName.put(originalNameKey, objRef(entityObj)); + if (changes != null) { + checkState( + changes.put(originalNameKey, ChangeUpdate.builder().build()), + "Entity '%s' updated more than once", + originalNameKey); + } + mutationResults.entityResult( + mapToEntity(entityObj, updateKeyForCatalogAndEntityType.catalogId())); + + LOGGER.debug("Updated {} '{}' with ID {}...", entityType, originalNameKey, entity.getId()); + } else { + mutationResults.unchangedEntityResult(entity); + + LOGGER.debug( + "Not updating {} '{}' with ID {} (no change)...", + entityType, + originalNameKey, + entity.getId()); + } + } + } + + private static void applyEntityCreateMutation( + UpdateKeyForCatalogAndEntityType updateKeyForCatalogAndEntityType, + CommitterState state, + UpdatableIndex byName, + UpdatableIndex byId, + UpdatableIndex changes, + UpdatableIndex locations, + EntityUpdate update, + MutationResults mutationResults) { + var entity = update.entity(); + var entityType = entity.getType(); + var persistence = state.persistence(); + var now = persistence.currentInstant(); + + var entityIdKey = IndexKey.key(entity.getId()); + var originalNameKey = byId.get(entityIdKey); + + if (entityType == PolarisEntityType.PRINCIPAL) { + throw new IllegalArgumentException("Use createPrincipal function instead of writeEntity"); + } + + var entityObjBuilder = + mapToObj(entity, Optional.empty()) + .id(persistence.generateId()) + .createTimestamp(now) + .updateTimestamp(now) + .entityVersion(1); + + var nameKey = nameKeyForEntity(entity, byId, mutationResults::entityResult); + if (nameKey == null) { + return; + } + + var entityObj = entityObjBuilder.build(); + var existingRef = byName.get(nameKey); + if (existingRef != null || originalNameKey != null) { + // PolarisMetaStoreManager.createEntityIfNotExists: if the entity already exists, + // return it. + if (existingRef == null) { + existingRef = byName.get(originalNameKey); + } + if (existingRef != null) { + var originalObj = + (ObjBase) + state + .persistence() + .fetch( + existingRef, + objTypeForPolarisType(entityType, entity.getSubType()).targetClass()); + if (originalObj != null) { + var unchangedCompareObj = objForChangeComparison(entity, Optional.empty(), originalObj); + if (unchangedCompareObj.equals(originalObj)) { + mutationResults.entityResultNoChange(entity); + return; + } + } + } + + mutationResults.entityResult( + ENTITY_ALREADY_EXISTS, entitySubTypeCodeFromObjType(existingRef)); + return; + } + + updateLocationsIndex(locations, null, entityObj); + + mutationResults.entityResult( + mapToEntity(entityObj, updateKeyForCatalogAndEntityType.catalogId())); + state.writeOrReplace("entity-" + entityObj.stableId(), entityObj); + + byName.put(nameKey, objRef(entityObj)); + byId.put(entityIdKey, nameKey); + + if (changes != null) { + checkState( + changes.put(nameKey, ChangeAdd.builder().build()), + "Entity '%s' updated more than once", + nameKey); + } + + LOGGER.debug( + "Added {} '{}' with ID {}...", entityObj.type().name(), nameKey, entityObj.stableId()); + } + + private static IndexKey policyIndexPrefixKey(PolicyObj policyObj, PolarisBaseEntity entity) { + // (Partial) index-key for the lookup + var keyByPolicyTemplate = + new PolicyMappingsObj.KeyByPolicy( + entity.getCatalogId(), entity.getId(), policyObj.policyType().getCode(), 0L, 0L); + + // Construct the prefix-key + return keyByPolicyTemplate.toPolicyWithTypePartialIndexKey(); + } + + public static void updateLocationsIndex( + UpdatableIndex locations, ObjBase originalObj, ObjBase entityObj) { + var previousBaseLocation = + originalObj != null ? originalObj.properties().get(ENTITY_BASE_LOCATION) : null; + var entityBaseLocation = + entityObj != null ? entityObj.properties().get(ENTITY_BASE_LOCATION) : null; + + if (Objects.equals(previousBaseLocation, entityBaseLocation)) { + return; + } + + if (previousBaseLocation != null) { + var locationIdentifier = + identifierFromLocationString(StorageLocation.of(previousBaseLocation).withoutScheme()); + var locationKey = locationIdentifier.toIndexKey(); + var currentEntityIds = locations.get(locationKey); + var newIds = new HashSet(); + if (currentEntityIds != null) { + newIds.addAll(currentEntityIds.entityIds()); + } + newIds.remove(originalObj.stableId()); + if (newIds.isEmpty()) { + locations.remove(locationKey); + } else { + locations.put(locationKey, entityIdSet(newIds)); + } + } + if (entityBaseLocation != null) { + var locationWithoutScheme = StorageLocation.of(entityBaseLocation).withoutScheme(); + var locationIdentifier = identifierFromLocationString(locationWithoutScheme); + var locationKey = locationIdentifier.toIndexKey(); + var currentEntityIds = locations.get(locationKey); + var newIds = new HashSet(); + if (currentEntityIds != null) { + newIds.addAll(currentEntityIds.entityIds()); + } + newIds.add(entityObj.stableId()); + locations.put(locationKey, entityIdSet(newIds)); + } + } + + private static IndexKey nameKeyForEntity( + PolarisEntityCore entity, + UpdatableIndex byId, + Consumer errorHandler) { + var identifierBuilder = ContentIdentifier.builder(); + var entityParentId = entity.getParentId(); + if (entityParentId != 0L && entityParentId != entity.getCatalogId()) { + var parentNameKey = byId.get(IndexKey.key(entityParentId)); + if (parentNameKey == null) { + errorHandler.accept(ENTITY_NOT_FOUND); + return null; + } + indexKeyToIdentifierBuilder(parentNameKey, identifierBuilder); + } + identifierBuilder.addElements(entity.getName()); + var identifier = identifierBuilder.build(); + return identifier.toIndexKey(); + } +} diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/MutationAttemptRoot.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/MutationAttemptRoot.java new file mode 100644 index 0000000000..cfea8baa51 --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/MutationAttemptRoot.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.persistence.nosql.metastore.mutation; + +import static org.apache.polaris.core.persistence.dao.entity.BaseResult.ReturnStatus.ENTITY_ALREADY_EXISTS; +import static org.apache.polaris.core.persistence.dao.entity.BaseResult.ReturnStatus.TARGET_ENTITY_CONCURRENTLY_MODIFIED; +import static org.apache.polaris.persistence.nosql.metastore.mutation.MutationResults.singleEntityResult; + +import java.util.Optional; +import java.util.function.Supplier; +import org.apache.polaris.persistence.nosql.api.commit.CommitterState; +import org.apache.polaris.persistence.nosql.coretypes.mapping.EntityObjMappings; +import org.apache.polaris.persistence.nosql.coretypes.realm.RootObj; + +public record MutationAttemptRoot( + CommitterState state, + Supplier> refObjSupplier, + EntityUpdate update) { + public Optional apply() { + var entity = update.entity(); + var ref = EntityObjMappings.mapToObj(entity, Optional.empty()); + var refObj = refObjSupplier.get(); + return switch (update.operation()) { + case CREATE -> { + if (refObj.isPresent()) { + yield state.noCommit(singleEntityResult(ENTITY_ALREADY_EXISTS)); + } + yield state.commitResult(singleEntityResult(entity), ref, refObj); + } + case UPDATE -> { + if (refObj.isPresent()) { + var rootObj = refObj.get(); + if (entity.getEntityVersion() != rootObj.entityVersion()) { + yield state.noCommit(singleEntityResult(TARGET_ENTITY_CONCURRENTLY_MODIFIED)); + } + } + yield state.commitResult(singleEntityResult(entity), ref, refObj); + } + default -> throw new IllegalStateException("Unexpected operation " + update.operation()); + }; + } +} diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/MutationResults.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/MutationResults.java new file mode 100644 index 0000000000..9b2b1d9745 --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/MutationResults.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.nosql.metastore.mutation; + +import static org.apache.polaris.core.persistence.dao.entity.BaseResult.ReturnStatus.SUCCESS; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.persistence.dao.entity.BaseResult; +import org.apache.polaris.core.persistence.dao.entity.DropEntityResult; +import org.apache.polaris.core.persistence.dao.entity.EntityResult; +import org.apache.polaris.persistence.nosql.api.index.IndexKey; +import org.apache.polaris.persistence.nosql.metastore.privs.GrantTriplet; + +public final class MutationResults { + private final List results; + // TODO populate and process 'aclsToRemove' + private final List aclsToRemove; + private final List droppedEntities; + private final List policyIndexKeysToRemove = new ArrayList<>(); + + boolean anyChange; + boolean hardFailure; + + private MutationResults( + List results, + List aclsToRemove, + List droppedEntities) { + this.results = results; + this.aclsToRemove = aclsToRemove; + this.droppedEntities = droppedEntities; + } + + MutationResults(BaseResult single) { + this(List.of(single), List.of(), List.of()); + } + + static MutationResults singleEntityResult(BaseResult.ReturnStatus returnStatus) { + return new MutationResults(new EntityResult(returnStatus, null)); + } + + static MutationResults singleEntityResult(PolarisBaseEntity entity) { + return new MutationResults(new EntityResult(entity)); + } + + static MutationResults newMutableMutationResults() { + return new MutationResults(new ArrayList<>(), new ArrayList<>(), new ArrayList<>()); + } + + public List results() { + return results; + } + + public List aclsToRemove() { + return aclsToRemove; + } + + public List droppedEntities() { + return droppedEntities; + } + + public List policyIndexKeysToRemove() { + return policyIndexKeysToRemove; + } + + void addPolicyIndexKeyToRemove(IndexKey indexKey) { + policyIndexKeysToRemove.add(indexKey); + } + + void entityResult(PolarisBaseEntity entity) { + add(new EntityResult(entity)); + anyChange = true; + } + + void entityResultNoChange(PolarisBaseEntity entity) { + add(new EntityResult(entity)); + } + + void unchangedEntityResult(PolarisBaseEntity entity) { + add(new EntityResult(entity)); + } + + void entityResult(BaseResult.ReturnStatus returnStatus) { + entityResult(returnStatus, null); + } + + void entityResult(BaseResult.ReturnStatus returnStatus, String extraInformation) { + add(new EntityResult(returnStatus, extraInformation)); + hardFailure |= returnStatus != SUCCESS; + } + + void dropResult(PolarisBaseEntity entity) { + add(new DropEntityResult()); + droppedEntities.add(entity); + anyChange = true; + } + + void dropResult(BaseResult.ReturnStatus returnStatus) { + dropResult(returnStatus, null); + } + + void dropResult(BaseResult.ReturnStatus returnStatus, String extraInformation) { + add(new DropEntityResult(returnStatus, extraInformation)); + hardFailure |= returnStatus != SUCCESS; + } + + private void add(BaseResult result) { + results.add(result); + hardFailure |= result.getReturnStatus() != SUCCESS; + } + + String failuresAsString() { + if (!hardFailure) { + return "(none)"; + } + + return results.stream() + .filter(result -> result.getReturnStatus() != SUCCESS) + .map(result -> result.getReturnStatus().name()) + .collect(Collectors.joining(", ")); + } + + public Optional firstFailure() { + if (!hardFailure) { + return Optional.empty(); + } + return results.stream().filter(result -> result.getReturnStatus() != SUCCESS).findFirst(); + } +} diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/PolicyMutation.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/PolicyMutation.java new file mode 100644 index 0000000000..c3123620c9 --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/PolicyMutation.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.persistence.nosql.metastore.mutation; + +import static org.apache.polaris.core.persistence.dao.entity.BaseResult.ReturnStatus.ENTITY_NOT_FOUND; +import static org.apache.polaris.core.persistence.dao.entity.BaseResult.ReturnStatus.POLICY_MAPPING_OF_SAME_TYPE_ALREADY_EXISTS; +import static org.apache.polaris.persistence.nosql.coretypes.realm.PolicyMapping.POLICY_MAPPING_SERIALIZER; +import static org.apache.polaris.persistence.nosql.coretypes.realm.PolicyMappingsObj.POLICY_MAPPINGS_REF_NAME; +import static org.apache.polaris.persistence.nosql.coretypes.realm.PolicyMappingsObj.PolicyMappingKey.fromIndexKey; + +import jakarta.annotation.Nonnull; +import java.util.Map; +import org.apache.polaris.core.persistence.dao.entity.BaseResult; +import org.apache.polaris.core.persistence.dao.entity.PolicyAttachmentResult; +import org.apache.polaris.core.policy.PolarisPolicyMappingRecord; +import org.apache.polaris.core.policy.PolicyType; +import org.apache.polaris.persistence.nosql.api.Persistence; +import org.apache.polaris.persistence.nosql.api.index.IndexContainer; +import org.apache.polaris.persistence.nosql.coretypes.realm.PolicyMapping; +import org.apache.polaris.persistence.nosql.coretypes.realm.PolicyMappingsObj; +import org.apache.polaris.persistence.nosql.metastore.indexaccess.MemoizedIndexedAccess; + +public record PolicyMutation( + Persistence persistence, + MemoizedIndexedAccess memoizedIndexedAccess, + long policyCatalogId, + long policyId, + @Nonnull PolicyType policyType, + long targetCatalogId, + long targetId, + boolean doAttach, + @Nonnull Map parameters) { + public PolicyAttachmentResult apply() { + try { + var committer = + persistence + .createCommitter( + POLICY_MAPPINGS_REF_NAME, PolicyMappingsObj.class, PolicyAttachmentResult.class) + .synchronizingLocally(); + return committer + .commitRuntimeException( + (state, refObjSupplier) -> { + var refObj = refObjSupplier.get(); + var index = + refObj + .map( + ref -> + ref.policyMappings() + .asUpdatableIndex( + state.persistence(), POLICY_MAPPING_SERIALIZER)) + .orElseGet( + () -> + IndexContainer.newUpdatableIndex( + persistence, POLICY_MAPPING_SERIALIZER)); + var builder = PolicyMappingsObj.builder(); + refObj.ifPresent(builder::from); + + var policyCatalogAccess = memoizedIndexedAccess.catalogContent(policyCatalogId); + + var policyOptional = policyCatalogAccess.byId(policyId); + if (policyOptional.isEmpty()) { + return state.noCommit( + new PolicyAttachmentResult( + BaseResult.ReturnStatus.POLICY_MAPPING_NOT_FOUND, null)); + } + if (targetCatalogId != 0L && targetCatalogId != targetId) { + // catalog content, check whether the entity exists + var targetCatalogAccess = + targetCatalogId == policyCatalogId + ? policyCatalogAccess + : memoizedIndexedAccess.catalogContent(targetCatalogId); + var targetOptional = targetCatalogAccess.byId(targetId); + if (targetOptional.isEmpty()) { + return state.noCommit(new PolicyAttachmentResult(ENTITY_NOT_FOUND, null)); + } + } + // else: against catalog, assume that it exists + + var result = + new PolicyAttachmentResult( + new PolarisPolicyMappingRecord( + targetCatalogId, + targetId, + policyCatalogId, + policyId, + policyType.getCode(), + parameters)); + + var keyByPolicy = + new PolicyMappingsObj.KeyByPolicy( + policyCatalogId, policyId, policyType.getCode(), targetCatalogId, targetId); + var keyByEntity = + new PolicyMappingsObj.KeyByEntity( + targetCatalogId, targetId, policyType.getCode(), policyCatalogId, policyId); + + var changed = false; + if (doAttach) { + if (policyType.isInheritable()) { + // The contract says that at max one policy of the same inheritable policy type + // must + // be attached to a single entity. + var policyPrefixKey = keyByEntity.toPolicyTypePartialIndexKey(); + var iter = index.iterator(policyPrefixKey, policyPrefixKey, false); + if (iter.hasNext()) { + var key = fromIndexKey(iter.next().getKey()); + if (!(key instanceof PolicyMappingsObj.KeyByEntity existing + && existing.policyCatalogId() == policyCatalogId + && existing.policyId() == policyId)) { + // same policy-type attached, error-out + return state.noCommit( + new PolicyAttachmentResult( + POLICY_MAPPING_OF_SAME_TYPE_ALREADY_EXISTS, null)); + } + } + } + + // note: parameters are only added to the "by entity" entry + index.put(keyByPolicy.toIndexKey(), PolicyMapping.EMPTY); + index.put( + keyByEntity.toIndexKey(), + PolicyMapping.builder().parameters(parameters).build()); + changed = true; + } else { + changed |= index.remove(keyByPolicy.toIndexKey()); + changed |= index.remove(keyByEntity.toIndexKey()); + } + + if (changed) { + builder.policyMappings(index.toIndexed("mappings", state::writeOrReplace)); + return state.commitResult(result, builder, refObj); + } + return state.noCommit(result); + }) + .orElseThrow(); + } finally { + memoizedIndexedAccess.invalidateReferenceHead(POLICY_MAPPINGS_REF_NAME); + } + } +} diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/PrincipalMutations.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/PrincipalMutations.java new file mode 100644 index 0000000000..3fe89c10d5 --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/PrincipalMutations.java @@ -0,0 +1,348 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.persistence.nosql.metastore.mutation; + +import static com.google.common.base.Preconditions.checkState; +import static org.apache.polaris.core.persistence.dao.entity.BaseResult.ReturnStatus.ENTITY_ALREADY_EXISTS; +import static org.apache.polaris.persistence.nosql.api.obj.ObjRef.objRef; +import static org.apache.polaris.persistence.nosql.coretypes.mapping.EntityObjMappings.mapToEntity; +import static org.apache.polaris.persistence.nosql.coretypes.mapping.EntityObjMappings.mapToObj; +import static org.apache.polaris.persistence.nosql.coretypes.mapping.EntityObjMappings.principalObjToPolarisPrincipalSecrets; +import static org.apache.polaris.persistence.nosql.coretypes.principals.PrincipalsObj.PRINCIPALS_REF_NAME; + +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.util.Optional; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.entity.PolarisPrincipalSecrets; +import org.apache.polaris.core.persistence.PrincipalSecretsGenerator; +import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; +import org.apache.polaris.core.persistence.dao.entity.CreatePrincipalResult; +import org.apache.polaris.persistence.nosql.api.Persistence; +import org.apache.polaris.persistence.nosql.api.commit.CommitException; +import org.apache.polaris.persistence.nosql.api.commit.CommitterState; +import org.apache.polaris.persistence.nosql.api.index.IndexKey; +import org.apache.polaris.persistence.nosql.api.index.UpdatableIndex; +import org.apache.polaris.persistence.nosql.api.obj.ObjRef; +import org.apache.polaris.persistence.nosql.coretypes.principals.PrincipalObj; +import org.apache.polaris.persistence.nosql.coretypes.principals.PrincipalsObj; +import org.apache.polaris.persistence.nosql.metastore.committers.ChangeResult; +import org.apache.polaris.persistence.nosql.metastore.committers.PrincipalsChangeCommitter; +import org.apache.polaris.persistence.nosql.metastore.committers.PrincipalsChangeCommitterWrapper; +import org.apache.polaris.persistence.nosql.metastore.indexaccess.MemoizedIndexedAccess; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class PrincipalMutations implements PrincipalsChangeCommitter { + private final Persistence persistence; + private final MemoizedIndexedAccess memoizedIndexedAccess; + + protected PrincipalMutations( + Persistence persistence, MemoizedIndexedAccess memoizedIndexedAccess) { + this.persistence = persistence; + this.memoizedIndexedAccess = memoizedIndexedAccess; + } + + PrincipalSecretsGenerator secretsGenerator(@Nullable RootCredentialsSet rootCredentialsSet) { + if (rootCredentialsSet != null) { + var realmId = this.persistence.realmId(); + return PrincipalSecretsGenerator.bootstrap(realmId, rootCredentialsSet); + } else { + return PrincipalSecretsGenerator.RANDOM_SECRETS; + } + } + + abstract Class resultType(); + + public RESULT apply() { + try { + return persistence + .createCommitter(PRINCIPALS_REF_NAME, PrincipalsObj.class, resultType()) + .synchronizingLocally() + .commitRuntimeException(new PrincipalsChangeCommitterWrapper<>(this)) + .orElseThrow(); + } finally { + memoizedIndexedAccess.invalidateIndexedAccess(0L, PolarisEntityType.PRINCIPAL.getCode()); + } + } + + public static final class UpdateSecrets extends PrincipalMutations { + + @FunctionalInterface + public interface SecretsUpdater { + R update(PrincipalObj principalObj, PrincipalObj.Builder principalObjBuilder); + } + + static class PrincipalNotFoundException extends RuntimeException {} + + private final Class resultType; + private final long principalId; + private final SecretsUpdater updater; + + public UpdateSecrets( + Persistence persistence, + MemoizedIndexedAccess memoizedIndexedAccess, + Class resultType, + long principalId, + SecretsUpdater updater) { + super(persistence, memoizedIndexedAccess); + this.resultType = resultType; + this.principalId = principalId; + this.updater = updater; + } + + @Override + Class resultType() { + return resultType; + } + + @Nonnull + @Override + public ChangeResult change( + @Nonnull CommitterState state, + @Nonnull PrincipalsObj.Builder ref, + @Nonnull UpdatableIndex byName, + @Nonnull UpdatableIndex byId, + @Nonnull UpdatableIndex byClientId) + throws CommitException { + var principalIdName = byId.get(IndexKey.key(principalId)); + if (principalIdName == null) { + throw new PrincipalNotFoundException(); + } + var principalObjRef = byName.get(principalIdName); + if (principalObjRef == null) { + throw new PrincipalNotFoundException(); + } + + var persistence = state.persistence(); + var principal = persistence.fetch(principalObjRef, PrincipalObj.class); + if (principal == null) { + throw new PrincipalNotFoundException(); + } + + var updatedPrincipalBuilder = + PrincipalObj.builder() + .from(principal) + .id(persistence.generateId()) + .updateTimestamp(persistence.currentInstant()); + + var apiResult = updater.update(principal, updatedPrincipalBuilder); + + var updatedPrincipal = updatedPrincipalBuilder.build(); + + ObjRef updatedPrincipalObjRef = objRef(updatedPrincipal); + byName.put(IndexKey.key(updatedPrincipal.name()), updatedPrincipalObjRef); + + principal.clientId().map(IndexKey::key).ifPresent(byClientId::remove); + updatedPrincipal + .clientId() + .ifPresent( + clientId -> { + var clientIdKey = IndexKey.key(clientId); + byClientId.put(clientIdKey, updatedPrincipalObjRef); + }); + + state.writeOrReplace("principal", updatedPrincipal); + + return new ChangeResult.CommitChange<>(apiResult); + } + } + + public static final class CreatePrincipal extends PrincipalMutations { + private static final Logger LOGGER = LoggerFactory.getLogger(CreatePrincipal.class); + private final PolarisBaseEntity principal; + private final RootCredentialsSet rootCredentialsSet; + + public CreatePrincipal( + Persistence persistence, + MemoizedIndexedAccess memoizedIndexedAccess, + PolarisBaseEntity principal, + RootCredentialsSet rootCredentialsSet) { + super(persistence, memoizedIndexedAccess); + this.principal = principal; + this.rootCredentialsSet = rootCredentialsSet; + } + + @Override + Class resultType() { + return CreatePrincipalResult.class; + } + + @Nonnull + @Override + public ChangeResult change( + @Nonnull CommitterState state, + @Nonnull PrincipalsObj.Builder ref, + @Nonnull UpdatableIndex byName, + @Nonnull UpdatableIndex byId, + @Nonnull UpdatableIndex byClientId) + throws CommitException { + var principalName = principal.getName(); + var principalId = principal.getId(); + var nameKey = IndexKey.key(principalName); + var persistence = state.persistence(); + + var existingPrincipal = + Optional.ofNullable(byName.get(nameKey)) + .map(objRef -> persistence.fetch(objRef, PrincipalObj.class)); + if (existingPrincipal.isPresent()) { + var existing = existingPrincipal.get(); + var secrets = principalObjToPolarisPrincipalSecrets(existing); + var forComparison = + MutationAttempt.objForChangeComparison(principal, Optional.of(secrets), existing); + return new ChangeResult.NoChange<>( + existing.equals(forComparison) + ? new CreatePrincipalResult(principal, secrets) + : new CreatePrincipalResult(ENTITY_ALREADY_EXISTS, null)); + } + + LOGGER.debug("Creating principal '{}' ...", principalName); + + PolarisPrincipalSecrets newPrincipalSecrets; + while (true) { + newPrincipalSecrets = + secretsGenerator(rootCredentialsSet).produceSecrets(principalName, principalId); + var newClientId = newPrincipalSecrets.getPrincipalClientId(); + if (byClientId.get(IndexKey.key(newClientId)) == null) { + LOGGER.debug("Generated secrets for principal '{}' ...", principalName); + break; + } + } + + var now = persistence.currentInstant(); + // Map from the given entity to retain both the properties and internal-properties bags + // (for example, PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_STATE) + var updatedPrincipalBuilder = + mapToObj(principal, Optional.of(newPrincipalSecrets)) + .name(principalName) + .stableId(principalId) + .entityVersion(1) + .createTimestamp(now) + .updateTimestamp(now) + .id(persistence.generateId()); + var updatedPrincipal = updatedPrincipalBuilder.build(); + + var updatedPrincipalObjRef = objRef(updatedPrincipal); + byClientId.put( + IndexKey.key(newPrincipalSecrets.getPrincipalClientId()), updatedPrincipalObjRef); + byName.put(nameKey, updatedPrincipalObjRef); + byId.put(IndexKey.key(principalId), nameKey); + + state.writeOrReplace("principal", updatedPrincipal); + + // return those + return new ChangeResult.CommitChange<>( + new CreatePrincipalResult( + mapToEntity(updatedPrincipal, 0L), + principalObjToPolarisPrincipalSecrets( + (PrincipalObj) updatedPrincipal, newPrincipalSecrets))); + } + } + + // TODO remove this? + // The code is not accessible via PolarisMetaStoreManager, would only be via BasePersistence. + public static final class GenerateNewSecrets extends PrincipalMutations { + private final String principalName; + private final long principalId; + + public GenerateNewSecrets( + Persistence persistence, + MemoizedIndexedAccess memoizedIndexedAccess, + String principalName, + long principalId) { + super(persistence, memoizedIndexedAccess); + this.principalName = principalName; + this.principalId = principalId; + } + + @Override + Class resultType() { + return PolarisPrincipalSecrets.class; + } + + @Nonnull + @Override + public ChangeResult change( + @Nonnull CommitterState state, + @Nonnull PrincipalsObj.Builder ref, + @Nonnull UpdatableIndex byName, + @Nonnull UpdatableIndex byId, + @Nonnull UpdatableIndex byClientId) + throws CommitException { + var nameKey = IndexKey.key(principalName); + var principalObjRef = byName.get(nameKey); + + var pers = state.persistence(); + var existingPrincipal = + Optional.ofNullable(principalObjRef) + .map(objRef -> pers.fetch(objRef, PrincipalObj.class)); + + checkState( + existingPrincipal.isEmpty() || principalId == existingPrincipal.get().stableId(), + "Principal id mismatch"); + + // generate new secrets + PolarisPrincipalSecrets newPrincipalSecrets; + while (true) { + newPrincipalSecrets = secretsGenerator(null).produceSecrets(principalName, principalId); + var newClientId = newPrincipalSecrets.getPrincipalClientId(); + if (byClientId.get(IndexKey.key(newClientId)) == null) { + break; + } + } + + var updatedPrincipalBuilder = PrincipalObj.builder(); + existingPrincipal.ifPresent(updatedPrincipalBuilder::from); + var now = pers.currentInstant(); + if (existingPrincipal.isEmpty()) { + updatedPrincipalBuilder + .name(principalName) + .stableId(principalId) + .entityVersion(1) + .createTimestamp(now); + } + updatedPrincipalBuilder + .id(pers.generateId()) + .updateTimestamp(now) + .clientId(newPrincipalSecrets.getPrincipalClientId()) + .mainSecretHash(newPrincipalSecrets.getMainSecretHash()) + .secondarySecretHash(newPrincipalSecrets.getSecondarySecretHash()) + .secretSalt(newPrincipalSecrets.getSecretSalt()); + var updatedPrincipal = updatedPrincipalBuilder.build(); + + existingPrincipal + .flatMap(PrincipalObj::clientId) + .map(IndexKey::key) + .ifPresent(byClientId::remove); + var updatedPrincipalObjRef = objRef(updatedPrincipal); + updatedPrincipal + .clientId() + .ifPresent(c -> byClientId.put(IndexKey.key(c), updatedPrincipalObjRef)); + byName.put(nameKey, updatedPrincipalObjRef); + byId.put(IndexKey.key(principalId), nameKey); + + state.writeOrReplace("principal", updatedPrincipal); + + // return those + return new ChangeResult.CommitChange<>(newPrincipalSecrets); + } + } +} diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/UpdateKeyForCatalogAndEntityType.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/UpdateKeyForCatalogAndEntityType.java new file mode 100644 index 0000000000..74f21776e8 --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/UpdateKeyForCatalogAndEntityType.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.nosql.metastore.mutation; + +import static org.apache.polaris.persistence.nosql.coretypes.mapping.EntityObjMappings.byEntityType; + +import jakarta.annotation.Nonnull; +import org.apache.polaris.core.entity.PolarisEntityCore; +import org.apache.polaris.core.entity.PolarisEntityType; + +public record UpdateKeyForCatalogAndEntityType( + @Nonnull PolarisEntityType entityType, long catalogId, boolean catalogContent) { + public static UpdateKeyForCatalogAndEntityType updateKeyForCatalogAndEntityType( + PolarisEntityCore entity) { + var type = entity.getType(); + var catalogId = entity.getCatalogId(); + var mapping = byEntityType(type); + mapping.checkCatalogId(catalogId); + var catalogContent = mapping.catalogContent(); + return new UpdateKeyForCatalogAndEntityType(type, catalogId, catalogContent); + } +} diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/privs/GrantTriplet.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/privs/GrantTriplet.java new file mode 100644 index 0000000000..aa1ed10dba --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/privs/GrantTriplet.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.nosql.metastore.privs; + +import static com.google.common.base.Preconditions.checkArgument; + +import org.apache.polaris.core.entity.PolarisEntityCore; + +/** + * Represents the triplet of catalog-ID, entity-ID and type-code plus a reverse-or-key marker. + * String representations of this type are used as ACL names and "role" names. + */ +public record GrantTriplet(boolean reverseOrKey, long catalogId, long id, int typeCode) { + public static GrantTriplet fromRoleName(String roleName) { + var c0 = roleName.charAt(0); + checkArgument(roleName.charAt(1) == '/' && (c0 == 'r' || c0 == 'd')); + + var idx2 = roleName.indexOf('/', 2); + var idx3 = roleName.indexOf('/', idx2 + 1); + + var catalogId = Long.parseLong(roleName.substring(2, idx2)); + var id = Long.parseLong(roleName.substring(idx2 + 1, idx3)); + var typeCode = Integer.parseInt(roleName.substring(idx3 + 1)); + + var reversed = c0 == 'r'; + + return new GrantTriplet(reversed, catalogId, id, typeCode); + } + + public static GrantTriplet forEntity(PolarisEntityCore entity) { + return new GrantTriplet(true, entity.getCatalogId(), entity.getId(), entity.getTypeCode()); + } + + public GrantTriplet asDirected() { + return new GrantTriplet(false, catalogId, id, typeCode); + } + + public String toRoleName() { + return (reverseOrKey ? "r/" : "d/") + catalogId + "/" + id + "/" + typeCode; + } +} diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/privs/PolarisPrivilegesProvider.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/privs/PolarisPrivilegesProvider.java new file mode 100644 index 0000000000..bf1f1ade3c --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/privs/PolarisPrivilegesProvider.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.nosql.metastore.privs; + +import static org.apache.polaris.persistence.nosql.authz.api.Privilege.InheritablePrivilege.inheritablePrivilege; + +import jakarta.enterprise.context.ApplicationScoped; +import java.util.Arrays; +import java.util.stream.Stream; +import org.apache.polaris.core.entity.PolarisPrivilege; +import org.apache.polaris.persistence.nosql.authz.spi.PrivilegeDefinition; +import org.apache.polaris.persistence.nosql.authz.spi.PrivilegesProvider; + +@ApplicationScoped +class PolarisPrivilegesProvider implements PrivilegesProvider { + @Override + public String name() { + return "Polaris privileges provider"; + } + + @Override + public Stream privilegeDefinitions() { + return Arrays.stream(PolarisPrivilege.values()) + .map(p -> PrivilegeDefinition.builder().privilege(inheritablePrivilege(p.name())).build()); + } +} diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/privs/SecurableAndGrantee.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/privs/SecurableAndGrantee.java new file mode 100644 index 0000000000..db6c4773a7 --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/privs/SecurableAndGrantee.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.nosql.metastore.privs; + +import org.apache.polaris.core.entity.PolarisGrantRecord; +import org.apache.polaris.persistence.nosql.coretypes.acl.AclObj; + +/** + * Holds IDs needed during ACL/grants loading, before those are exploded into many {@link + * PolarisGrantRecord}s. + */ +public record SecurableAndGrantee( + long securableCatalogId, + long securableId, + int securableTypeCode, + long granteeCatalogId, + long granteeId, + int granteeTypeCode) { + + public static SecurableAndGrantee forTriplet( + long catalogId, AclObj aclObj, GrantTriplet triplet) { + var reverseOrKey = triplet.reverseOrKey(); + var securableCatalogId = reverseOrKey ? triplet.catalogId() : catalogId; + var securableId = reverseOrKey ? triplet.id() : aclObj.securableId(); + var securableTypeCode = + reverseOrKey ? triplet.typeCode() : aclObj.securableTypeCode().orElseThrow(); + var granteeCatalogId = reverseOrKey ? catalogId : triplet.catalogId(); + var granteeId = reverseOrKey ? aclObj.securableId() : triplet.id(); + var granteeTypeCode = + reverseOrKey ? aclObj.securableTypeCode().orElseThrow() : triplet.typeCode(); + return new SecurableAndGrantee( + securableCatalogId, + securableId, + securableTypeCode, + granteeCatalogId, + granteeId, + granteeTypeCode); + } + + public PolarisGrantRecord grantRecordForPrivilege(int privilegeCode) { + return new PolarisGrantRecord( + securableCatalogId, securableId, granteeCatalogId, granteeId, privilegeCode); + } +} diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/privs/SecurableGranteePrivilegeTuple.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/privs/SecurableGranteePrivilegeTuple.java new file mode 100644 index 0000000000..74ed7a08cf --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/privs/SecurableGranteePrivilegeTuple.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.nosql.metastore.privs; + +import jakarta.annotation.Nonnull; +import org.apache.polaris.core.entity.PolarisEntityCore; +import org.apache.polaris.core.entity.PolarisPrivilege; + +public record SecurableGranteePrivilegeTuple( + @Nonnull PolarisEntityCore securable, + @Nonnull PolarisEntityCore grantee, + @Nonnull PolarisPrivilege privilege) {} diff --git a/persistence/nosql/persistence/metastore/src/main/resources/META-INF/beans.xml b/persistence/nosql/persistence/metastore/src/main/resources/META-INF/beans.xml new file mode 100644 index 0000000000..a297f1aa53 --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/main/resources/META-INF/beans.xml @@ -0,0 +1,24 @@ + + + + + \ No newline at end of file diff --git a/persistence/nosql/persistence/metastore/src/main/resources/META-INF/services/org.apache.polaris.core.persistence.pagination.Token$TokenType b/persistence/nosql/persistence/metastore/src/main/resources/META-INF/services/org.apache.polaris.core.persistence.pagination.Token$TokenType new file mode 100644 index 0000000000..2f2982584f --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/main/resources/META-INF/services/org.apache.polaris.core.persistence.pagination.Token$TokenType @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +org.apache.polaris.persistence.nosql.metastore.NoSqlPaginationToken$NoSqlPaginationTokenType diff --git a/persistence/nosql/persistence/metastore/src/test/java/org/apache/polaris/persistence/nosql/metastore/TestContentIdentifier.java b/persistence/nosql/persistence/metastore/src/test/java/org/apache/polaris/persistence/nosql/metastore/TestContentIdentifier.java new file mode 100644 index 0000000000..7e5def75a5 --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/test/java/org/apache/polaris/persistence/nosql/metastore/TestContentIdentifier.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.persistence.nosql.metastore; + +import static org.junit.jupiter.params.provider.Arguments.arguments; + +import java.util.List; +import java.util.stream.Stream; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +@ExtendWith(SoftAssertionsExtension.class) +public class TestContentIdentifier { + @InjectSoftAssertions SoftAssertions soft; + + @ParameterizedTest + @MethodSource + public void storageLocation(String input, List elements) { + var identifier = ContentIdentifier.identifierFromLocationString(input); + soft.assertThat(identifier.elements()).containsExactlyElementsOf(elements); + var inKey = identifier.toIndexKey(); + var identifierFromKey = ContentIdentifier.indexKeyToIdentifier(inKey); + soft.assertThat(identifierFromKey).isEqualTo(identifierFromKey); + } + + static Stream storageLocation() { + return Stream.of( + arguments("", List.of()), + arguments("foo", List.of("foo")), + arguments("//foo", List.of("foo")), + arguments("//foo/\\/bar", List.of("foo", "bar")), + arguments("\\foo/\\/bar", List.of("foo", "bar")), + arguments("\\/foo/\\/bar", List.of("foo", "bar")), + arguments("\\/\\foo/\\/bar", List.of("foo", "bar")), + arguments("foo/\\/bar", List.of("foo", "bar")), + arguments("foo/\\/bar/", List.of("foo", "bar")), + arguments("foo/\\/bar\\", List.of("foo", "bar")), + arguments("foo/\\/bar/\\/", List.of("foo", "bar"))); + } +} diff --git a/persistence/nosql/persistence/metastore/src/test/java/org/apache/polaris/persistence/nosql/metastore/TestNoSqlMetaStoreManager.java b/persistence/nosql/persistence/metastore/src/test/java/org/apache/polaris/persistence/nosql/metastore/TestNoSqlMetaStoreManager.java new file mode 100644 index 0000000000..9dcb12cf71 --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/test/java/org/apache/polaris/persistence/nosql/metastore/TestNoSqlMetaStoreManager.java @@ -0,0 +1,259 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.nosql.metastore; + +import static org.apache.polaris.core.entity.PolarisEntityConstants.ENTITY_BASE_LOCATION; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.InstanceOfAssertFactories.BOOLEAN; + +import io.smallrye.common.annotation.Identifier; +import jakarta.inject.Inject; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import org.apache.iceberg.catalog.Namespace; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.config.PolarisConfigurationStore; +import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.entity.NamespaceEntity; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisEntityConstants; +import org.apache.polaris.core.entity.PolarisEntityCore; +import org.apache.polaris.core.entity.PolarisEntitySubType; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.persistence.BasePolarisMetaStoreManagerTest; +import org.apache.polaris.core.persistence.MetaStoreManagerFactory; +import org.apache.polaris.core.persistence.PolarisMetaStoreManager; +import org.apache.polaris.core.persistence.PolarisTestMetaStoreManager; +import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; +import org.apache.polaris.core.persistence.dao.entity.CreateCatalogResult; +import org.apache.polaris.core.persistence.dao.entity.EntityResult; +import org.apache.polaris.ids.api.MonotonicClock; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; +import org.jboss.weld.junit5.EnableWeld; +import org.jboss.weld.junit5.WeldInitiator; +import org.jboss.weld.junit5.WeldSetup; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +@SuppressWarnings("CdiInjectionPointsInspection") +@EnableWeld +@ExtendWith(SoftAssertionsExtension.class) +public class TestNoSqlMetaStoreManager extends BasePolarisMetaStoreManagerTest { + @WeldSetup WeldInitiator weld = WeldInitiator.performDefaultDiscovery(); + + @InjectSoftAssertions SoftAssertions soft; + + @Inject + @Identifier("nosql") + MetaStoreManagerFactory metaStoreManagerFactory; + + @Inject PolarisConfigurationStore configurationStore; + @Inject MonotonicClock monotonicClock; + + String realmId; + RealmContext realmContext; + + PolarisMetaStoreManager metaStore; + PolarisCallContext callContext; + + @Override + protected PolarisTestMetaStoreManager createPolarisTestMetaStoreManager() { + realmId = UUID.randomUUID().toString(); + realmContext = () -> realmId; + + var startTime = monotonicClock.currentTimeMillis(); + + metaStoreManagerFactory.bootstrapRealms(List.of(realmId), RootCredentialsSet.fromEnvironment()); + + var manager = metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext); + var session = metaStoreManagerFactory.getOrCreateSession(realmContext); + + var callCtx = new PolarisCallContext(realmContext, session, configurationStore); + + return new PolarisTestMetaStoreManager(manager, callCtx, startTime, false); + } + + @Override + @Disabled("No entity cache, no need to test it") + protected void testEntityCache() {} + + @Override + @Disabled( + "Nothing in the code base calls 'loadTasks', the contract of that function is not what the test exercises") + protected void testLoadTasksInParallel() {} + + @Override + @Disabled( + "Nothing in the code base calls 'loadTasks', the contract of that function is not what the test exercises") + protected void testLoadTasks() {} + + @Override + protected void testLoadResolvedEntitiesById() { + assertThatCode(super::testLoadResolvedEntitiesById) + .isInstanceOf(UnsupportedOperationException.class); + } + + @BeforeEach + void setup() { + this.metaStore = polarisTestMetaStoreManager.polarisMetaStoreManager(); + this.callContext = polarisTestMetaStoreManager.polarisCallContext(); + } + + @Test + public void overlappingLocations() { + PolarisBaseEntity catalog = + new PolarisBaseEntity( + PolarisEntityConstants.getNullId(), + metaStore.generateNewEntityId(callContext).getId(), + PolarisEntityType.CATALOG, + PolarisEntitySubType.NULL_SUBTYPE, + PolarisEntityConstants.getRootEntityId(), + "overlappingLocations"); + CreateCatalogResult catalogCreated = metaStore.createCatalog(callContext, catalog, List.of()); + assertThat(catalogCreated).isNotNull(); + catalog = catalogCreated.getCatalog(); + + var nsFoo = + createEntity( + List.of(catalog), + PolarisEntityType.NAMESPACE, + PolarisEntitySubType.NULL_SUBTYPE, + "ns2", + Map.of(ENTITY_BASE_LOCATION, "s3://bucket/foo/")); + assertThat(nsFoo).extracting(EntityResult::isSuccess, BOOLEAN).isTrue(); + var nsBar = + createEntity( + List.of(catalog), + PolarisEntityType.NAMESPACE, + PolarisEntitySubType.NULL_SUBTYPE, + "ns3", + Map.of(ENTITY_BASE_LOCATION, "s3://bucket/bar/")); + assertThat(nsBar).extracting(EntityResult::isSuccess, BOOLEAN).isTrue(); + var nsFoobar = + createEntity( + List.of(catalog), + PolarisEntityType.NAMESPACE, + PolarisEntitySubType.NULL_SUBTYPE, + "foobar", + Map.of(ENTITY_BASE_LOCATION, "s3://bucket/foo/bar/")); + assertThat(nsFoobar).extracting(EntityResult::isSuccess, BOOLEAN).isTrue(); + var nsFoobar2 = + createEntity( + List.of(catalog), + PolarisEntityType.NAMESPACE, + PolarisEntitySubType.NULL_SUBTYPE, + "foobar2", + // Same location again + Map.of(ENTITY_BASE_LOCATION, "s3://bucket/foo/bar/")); + assertThat(nsFoobar2).extracting(EntityResult::isSuccess, BOOLEAN).isTrue(); + + soft.assertThat( + metaStore.hasOverlappingSiblings( + callContext, + new NamespaceEntity.Builder(Namespace.of("x")) + .setCatalogId(catalog.getId()) + .setBaseLocation("s3://bucket/foo/") + .build())) + .isPresent() + .contains(Optional.of("s3://bucket/foo/")); + + for (var check : + List.of( + "s3://bucket/foo/bar", + "s3://bucket/foo/bar/", + "s3a://bucket/foo/bar/", + "gs://bucket/foo/bar/")) { + soft.assertThat( + metaStore.hasOverlappingSiblings( + callContext, + new NamespaceEntity.Builder(Namespace.of("x")) + .setCatalogId(catalog.getId()) + .setBaseLocation(check) + .build())) + .isPresent() + .contains(Optional.of("s3://bucket/foo/bar/")); + } + + soft.assertThat( + metaStore.hasOverlappingSiblings( + callContext, + new NamespaceEntity.Builder(Namespace.of("x")) + .setCatalogId(catalog.getId()) + .setBaseLocation("s3://other/data/stuff/") + .build())) + .isPresent() + .contains(Optional.empty()); + + // Drop one of the entities with the duplicate base location + metaStore.dropEntityIfExists(callContext, List.of(catalog), nsFoobar.getEntity(), null, false); + // Must still report an overlap + soft.assertThat( + metaStore.hasOverlappingSiblings( + callContext, + new NamespaceEntity.Builder(Namespace.of("x")) + .setCatalogId(catalog.getId()) + .setBaseLocation("s3://bucket/foo/bar") + .build())) + .isPresent() + .contains(Optional.of("s3://bucket/foo/bar/")); + + // Drop one of the entities with the duplicate base location + metaStore.dropEntityIfExists(callContext, List.of(catalog), nsFoobar2.getEntity(), null, false); + // No more overlaps + soft.assertThat( + metaStore.hasOverlappingSiblings( + callContext, + new NamespaceEntity.Builder(Namespace.of("x")) + .setCatalogId(catalog.getId()) + .setBaseLocation("s3://bucket/foo/bar") + .build())) + .isPresent() + .contains(Optional.empty()); + } + + @SuppressWarnings("SameParameterValue") + EntityResult createEntity( + List catalogPath, + PolarisEntityType entityType, + PolarisEntitySubType entitySubType, + String name, + Map properties) { + var entityId = metaStore.generateNewEntityId(callContext).getId(); + PolarisBaseEntity newEntity = + new PolarisBaseEntity.Builder() + .catalogId(catalogPath.getFirst().getId()) + .id(entityId) + .typeCode(entityType.getCode()) + .subTypeCode(entitySubType.getCode()) + .parentId(catalogPath.getLast().getId()) + .name(name) + .propertiesAsMap(properties) + .build(); + @SuppressWarnings({"unchecked", "rawtypes"}) + var path = (List) (List) catalogPath; + return metaStore.createEntityIfNotExists(callContext, path, newEntity); + } +} diff --git a/persistence/nosql/persistence/metastore/src/test/java/org/apache/polaris/persistence/nosql/metastore/TestNoSqlResolver.java b/persistence/nosql/persistence/metastore/src/test/java/org/apache/polaris/persistence/nosql/metastore/TestNoSqlResolver.java new file mode 100644 index 0000000000..464af9ddbd --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/test/java/org/apache/polaris/persistence/nosql/metastore/TestNoSqlResolver.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.nosql.metastore; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.smallrye.common.annotation.Identifier; +import jakarta.inject.Inject; +import java.util.List; +import java.util.UUID; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.config.PolarisConfigurationStore; +import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.entity.PolarisGrantRecord; +import org.apache.polaris.core.persistence.BaseResolverTest; +import org.apache.polaris.core.persistence.MetaStoreManagerFactory; +import org.apache.polaris.core.persistence.PolarisMetaStoreManager; +import org.apache.polaris.core.persistence.PolarisTestMetaStoreManager; +import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; +import org.apache.polaris.ids.api.MonotonicClock; +import org.jboss.weld.junit5.EnableWeld; +import org.jboss.weld.junit5.WeldInitiator; +import org.jboss.weld.junit5.WeldSetup; +import org.junit.jupiter.api.BeforeAll; + +@SuppressWarnings("CdiInjectionPointsInspection") +@EnableWeld +public class TestNoSqlResolver extends BaseResolverTest { + @WeldSetup WeldInitiator weld = WeldInitiator.performDefaultDiscovery(); + + @Inject + @Identifier("nosql") + MetaStoreManagerFactory metaStoreManagerFactory; + + @Inject PolarisConfigurationStore configurationStore; + @Inject MonotonicClock monotonicClock; + + PolarisMetaStoreManager metaStoreManager; + + PolarisCallContext callCtx; + PolarisTestMetaStoreManager tm; + + String realmId; + RealmContext realmContext; + + @Override + protected PolarisCallContext callCtx() { + if (callCtx == null) { + realmId = UUID.randomUUID().toString(); + realmContext = () -> realmId; + + var startTime = monotonicClock.currentTimeMillis(); + + metaStoreManagerFactory.bootstrapRealms( + List.of(realmId), RootCredentialsSet.fromEnvironment()); + + metaStoreManager = metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext); + var session = metaStoreManagerFactory.getOrCreateSession(realmContext); + + callCtx = new PolarisCallContext(realmContext, session, configurationStore); + + tm = new PolarisTestMetaStoreManager(metaStoreManager, callCtx, startTime, false); + } + return callCtx; + } + + @Override + protected PolarisTestMetaStoreManager tm() { + callCtx(); + return tm; + } + + @Override + protected PolarisMetaStoreManager metaStoreManager() { + callCtx(); + return metaStoreManager; + } + + @Override + protected void checkRefGrantRecords( + List grantRecords, List refGrantRecords) { + assertThat(grantRecords).containsExactlyInAnyOrderElementsOf(refGrantRecords); + } + + @BeforeAll + public static void beforeAll() { + supportEntityCache = false; + } +} diff --git a/persistence/nosql/persistence/metastore/src/test/java/org/apache/polaris/persistence/nosql/metastore/indexaccess/TestIndexedContainerAccess.java b/persistence/nosql/persistence/metastore/src/test/java/org/apache/polaris/persistence/nosql/metastore/indexaccess/TestIndexedContainerAccess.java new file mode 100644 index 0000000000..1a780d476c --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/test/java/org/apache/polaris/persistence/nosql/metastore/indexaccess/TestIndexedContainerAccess.java @@ -0,0 +1,312 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.persistence.nosql.metastore.indexaccess; + +import static org.apache.polaris.core.entity.PolarisEntityConstants.getNullId; +import static org.apache.polaris.core.entity.PolarisEntityConstants.getRootContainerName; +import static org.apache.polaris.core.entity.PolarisEntityConstants.getRootEntityId; +import static org.apache.polaris.core.entity.PolarisEntitySubType.NULL_SUBTYPE; +import static org.apache.polaris.core.entity.PolarisEntityType.CATALOG; +import static org.apache.polaris.core.entity.PolarisEntityType.CATALOG_ROLE; +import static org.apache.polaris.core.entity.PolarisEntityType.NAMESPACE; +import static org.apache.polaris.core.entity.PolarisEntityType.POLICY; +import static org.apache.polaris.core.entity.PolarisEntityType.PRINCIPAL; +import static org.apache.polaris.core.entity.PolarisEntityType.PRINCIPAL_ROLE; +import static org.apache.polaris.core.entity.PolarisEntityType.ROOT; +import static org.apache.polaris.core.entity.PolarisEntityType.TABLE_LIKE; +import static org.apache.polaris.core.entity.PolarisEntityType.TASK; +import static org.apache.polaris.persistence.nosql.api.obj.ObjRef.objRef; +import static org.apache.polaris.persistence.nosql.coretypes.mapping.EntityObjMappings.byEntityType; +import static org.apache.polaris.persistence.nosql.metastore.indexaccess.IndexedContainerAccess.indexedAccessDirect; +import static org.apache.polaris.persistence.nosql.metastore.indexaccess.IndexedContainerAccess.indexedAccessForEntityType; + +import io.smallrye.common.annotation.Identifier; +import jakarta.inject.Inject; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Stream; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisEntitySubType; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.entity.PrincipalEntity; +import org.apache.polaris.core.persistence.MetaStoreManagerFactory; +import org.apache.polaris.core.persistence.PolarisMetaStoreManager; +import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; +import org.apache.polaris.persistence.nosql.api.Persistence; +import org.apache.polaris.persistence.nosql.api.RealmPersistenceFactory; +import org.apache.polaris.persistence.nosql.coretypes.realm.RootObj; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; +import org.jboss.weld.junit5.EnableWeld; +import org.jboss.weld.junit5.WeldInitiator; +import org.jboss.weld.junit5.WeldSetup; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +@SuppressWarnings("CdiInjectionPointsInspection") +@ExtendWith(SoftAssertionsExtension.class) +@EnableWeld +public class TestIndexedContainerAccess { + @InjectSoftAssertions public SoftAssertions soft; + + @WeldSetup WeldInitiator weld = WeldInitiator.performDefaultDiscovery(); + + @Inject RealmPersistenceFactory realmPersistenceFactory; + + @Inject + @Identifier("nosql") + MetaStoreManagerFactory metaStoreManagerFactory; + + PolarisCallContext callContext; + PolarisMetaStoreManager metaStoreManager; + + Persistence persistence; + + @BeforeEach + public void setup(TestInfo testInfo) { + var realmId = testInfo.getTestMethod().orElseThrow().getName(); + persistence = realmPersistenceFactory.newBuilder().realmId(realmId).build(); + + metaStoreManagerFactory.bootstrapRealms(List.of(realmId), RootCredentialsSet.EMPTY); + + var realmContext = (RealmContext) () -> realmId; + callContext = + new PolarisCallContext( + realmContext, metaStoreManagerFactory.getOrCreateSession(realmContext)); + metaStoreManager = metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext); + } + + @Test + public void indexedAccessForRoot() { + var ia = indexedAccessForEntityType(ROOT.getCode(), persistence, 0L); + + soft.assertThat(ia).isInstanceOf(IndexedContainerAccessRoot.class); + soft.assertThat(ia.catalogStableId()).isEqualTo(0L); + soft.assertThat(ia.nameIndex()).isNotEmpty(); + soft.assertThat(ia.stableIdIndex()).isNotEmpty(); + soft.assertThat(ia.byNameOnRoot(getRootContainerName())).get().isInstanceOf(RootObj.class); + soft.assertThat(ia.byNameOnRoot("foo")).isEmpty(); + soft.assertThat(ia.byParentIdAndName(0L, getRootContainerName())) + .get() + .isInstanceOf(RootObj.class); + soft.assertThat(ia.byParentIdAndName(42L, getRootContainerName())).isEmpty(); + soft.assertThat(ia.byParentIdAndName(0L, "foo")).isEmpty(); + soft.assertThat(ia.byId(0L)).get().isInstanceOf(RootObj.class); + soft.assertThat(ia.byId(42L)).isEmpty(); + soft.assertThatThrownBy(ia::refObj).isInstanceOf(UnsupportedOperationException.class); + } + + @ParameterizedTest + @MethodSource("realmScopedEntityTypes") + public void indexedAccessForRealmScopedTypes( + PolarisEntityType entityType, PolarisEntitySubType subType) { + // Accessing a catalog SHOULD lead to an empty index. + // See org.apache.polaris.persistence.nosql.coretypes.mapping.BaseMapping.fixCatalogId(): + // ... some tests use a non-0 catalog ID for non-catalog entity types. + // soft.assertThatIllegalArgumentException() + // .isThrownBy(() -> + // IndexedContainerAccess.indexedAccessForEntityType( + // entityType.getCode(), persistence, catalogId)); + + // Accessing no-catalog gives the implementation. + var ia = indexedAccessForEntityType(entityType.getCode(), persistence, 0L); + soft.assertThat(ia).isInstanceOf(IndexedContainerAccessImpl.class); + soft.assertThat(ia.catalogStableId()).isEqualTo(0L); + + var entityName = "entity-for-" + entityType.name(); + long entityId = persistence.generateId(); + var entityForType = + new PolarisBaseEntity( + getNullId(), entityId, entityType, subType, getRootEntityId(), entityName); + if (entityType == PRINCIPAL) { + metaStoreManager.createPrincipal(callContext, new PrincipalEntity(entityForType)); + } else { + metaStoreManager.createEntityIfNotExists(callContext, List.of(), entityForType); + } + + var mapping = byEntityType(entityType); + var typeClass = mapping.objTypeForSubType(subType).targetClass(); + + soft.assertThat(ia).isInstanceOf(IndexedContainerAccessImpl.class); + soft.assertThat(ia.catalogStableId()).isEqualTo(0L); + soft.assertThat(ia.nameIndex()).isNotEmpty(); + soft.assertThat(ia.stableIdIndex()).isNotEmpty(); + soft.assertThat(ia.byNameOnRoot(entityName)).get().isInstanceOf(typeClass); + soft.assertThat(ia.byNameOnRoot("foo")).isEmpty(); + soft.assertThat(ia.byParentIdAndName(0L, entityName)).get().isInstanceOf(typeClass); + soft.assertThat(ia.byParentIdAndName(42L, entityName)).isEmpty(); + soft.assertThat(ia.byParentIdAndName(0L, "foo")).isEmpty(); + soft.assertThat(ia.byId(entityId)).get().isInstanceOf(typeClass); + soft.assertThat(ia.byId(42L)).isEmpty(); + soft.assertThat(ia.refObj()).get().isInstanceOf(mapping.containerObjTypeClass()); + + // verify via indexedAccessDirect() + ia = indexedAccessDirect(persistence, objRef(ia.refObj().orElseThrow())); + soft.assertThat(ia).isInstanceOf(IndexedContainerAccessImpl.class); + soft.assertThat(ia.catalogStableId()).isEqualTo(-1L); + soft.assertThat(ia.nameIndex()).isNotEmpty(); + soft.assertThat(ia.stableIdIndex()).isNotEmpty(); + soft.assertThat(ia.byNameOnRoot(entityName)).get().isInstanceOf(typeClass); + soft.assertThat(ia.byNameOnRoot("foo")).isEmpty(); + soft.assertThat(ia.byParentIdAndName(0L, entityName)).get().isInstanceOf(typeClass); + soft.assertThat(ia.byParentIdAndName(42L, entityName)).isEmpty(); + soft.assertThat(ia.byParentIdAndName(0L, "foo")).isEmpty(); + soft.assertThat(ia.byId(entityId)).get().isInstanceOf(typeClass); + soft.assertThat(ia.byId(42L)).isEmpty(); + soft.assertThat(ia.refObj()).get().isInstanceOf(mapping.containerObjTypeClass()); + } + + @ParameterizedTest + @MethodSource("catalogScopedEntityTypes") + public void indexedAccessForCatalogScopedTypes( + PolarisEntityType entityType, PolarisEntitySubType subType) { + + // Accessing no-catalog must lead to an exception. + soft.assertThatIllegalArgumentException() + .isThrownBy(() -> indexedAccessForEntityType(entityType.getCode(), persistence, 0L)); + + var catalogId = persistence.generateId(); + var catalogEntity = + new PolarisBaseEntity( + getNullId(), catalogId, CATALOG, NULL_SUBTYPE, getRootEntityId(), "my-catalog"); + metaStoreManager.createCatalog(callContext, catalogEntity, List.of()); + + var entityName = "entity-for-" + entityType.name(); + long entityId = persistence.generateId(); + var entityForType = + new PolarisBaseEntity( + catalogId, entityId, entityType, subType, getRootEntityId(), entityName); + metaStoreManager.createEntityIfNotExists(callContext, List.of(catalogEntity), entityForType); + + var mapping = byEntityType(entityType); + var typeClass = mapping.objTypeForSubType(subType).targetClass(); + + // Accessing a catalog gives the implementation. + var ia = indexedAccessForEntityType(entityType.getCode(), persistence, catalogId); + soft.assertThat(ia).isInstanceOf(IndexedContainerAccessImpl.class); + soft.assertThat(ia.catalogStableId()).isEqualTo(catalogId); + soft.assertThat(ia.nameIndex()).isNotEmpty(); + soft.assertThat(ia.stableIdIndex()).isNotEmpty(); + soft.assertThat(ia.byNameOnRoot(entityName)).get().isInstanceOf(typeClass); + soft.assertThat(ia.byNameOnRoot("foo")).isEmpty(); + soft.assertThat(ia.byParentIdAndName(0L, entityName)).get().isInstanceOf(typeClass); + soft.assertThat(ia.byParentIdAndName(42L, entityName)).isEmpty(); + soft.assertThat(ia.byParentIdAndName(0L, "foo")).isEmpty(); + soft.assertThat(ia.byId(entityId)).get().isInstanceOf(typeClass); + soft.assertThat(ia.byId(42L)).isEmpty(); + soft.assertThat(ia.refObj()).get().isInstanceOf(mapping.containerObjTypeClass()); + + // verify via indexedAccessDirect() + ia = indexedAccessDirect(persistence, objRef(ia.refObj().orElseThrow())); + soft.assertThat(ia).isInstanceOf(IndexedContainerAccessImpl.class); + soft.assertThat(ia.catalogStableId()).isEqualTo(-1L); + soft.assertThat(ia.nameIndex()).isNotEmpty(); + soft.assertThat(ia.stableIdIndex()).isNotEmpty(); + soft.assertThat(ia.byNameOnRoot(entityName)).get().isInstanceOf(typeClass); + soft.assertThat(ia.byNameOnRoot("foo")).isEmpty(); + soft.assertThat(ia.byParentIdAndName(0L, entityName)).get().isInstanceOf(typeClass); + soft.assertThat(ia.byParentIdAndName(42L, entityName)).isEmpty(); + soft.assertThat(ia.byParentIdAndName(0L, "foo")).isEmpty(); + soft.assertThat(ia.byId(entityId)).get().isInstanceOf(typeClass); + soft.assertThat(ia.byId(42L)).isEmpty(); + soft.assertThat(ia.refObj()).get().isInstanceOf(mapping.containerObjTypeClass()); + + if (mapping.catalogContent()) { + + var namespaceName = "namespace-for-" + entityType.name(); + long namespaceId = persistence.generateId(); + var namespace = + new PolarisBaseEntity( + catalogId, namespaceId, NAMESPACE, NULL_SUBTYPE, getRootEntityId(), namespaceName); + metaStoreManager.createEntityIfNotExists(callContext, List.of(catalogEntity), namespace); + + var subName = "sub-for-" + entityType.name(); + long subId = persistence.generateId(); + var sub = new PolarisBaseEntity(catalogId, subId, entityType, subType, namespaceId, subName); + metaStoreManager.createEntityIfNotExists(callContext, List.of(catalogEntity, namespace), sub); + + // Get a new ICA as the previous one has cached the index. + ia = indexedAccessForEntityType(entityType.getCode(), persistence, catalogId); + soft.assertThat(ia).isInstanceOf(IndexedContainerAccessImpl.class); + soft.assertThat(ia.catalogStableId()).isEqualTo(catalogId); + soft.assertThat(ia.nameIndex()).isNotEmpty(); + soft.assertThat(ia.stableIdIndex()).isNotEmpty(); + soft.assertThat(ia.byNameOnRoot(subName)).isEmpty(); + soft.assertThat(ia.byParentIdAndName(namespaceId, subName)).get().isInstanceOf(typeClass); + soft.assertThat(ia.byParentIdAndName(42L, subName)).isEmpty(); + soft.assertThat(ia.byParentIdAndName(0L, subName)).isEmpty(); + soft.assertThat(ia.byId(subId)).get().isInstanceOf(typeClass); + soft.assertThat(ia.byId(42L)).isEmpty(); + soft.assertThat(ia.refObj()).get().isInstanceOf(mapping.containerObjTypeClass()); + + // verify via indexedAccessDirect() + ia = indexedAccessDirect(persistence, objRef(ia.refObj().orElseThrow())); + soft.assertThat(ia).isInstanceOf(IndexedContainerAccessImpl.class); + soft.assertThat(ia.catalogStableId()).isEqualTo(-1L); + soft.assertThat(ia.nameIndex()).isNotEmpty(); + soft.assertThat(ia.stableIdIndex()).isNotEmpty(); + soft.assertThat(ia.byNameOnRoot(subName)).isEmpty(); + soft.assertThat(ia.byParentIdAndName(namespaceId, subName)).get().isInstanceOf(typeClass); + soft.assertThat(ia.byParentIdAndName(42L, subName)).isEmpty(); + soft.assertThat(ia.byParentIdAndName(0L, subName)).isEmpty(); + soft.assertThat(ia.byId(subId)).get().isInstanceOf(typeClass); + soft.assertThat(ia.byId(42L)).isEmpty(); + soft.assertThat(ia.refObj()).get().isInstanceOf(mapping.containerObjTypeClass()); + } + } + + static Stream realmScopedEntityTypes() { + return Stream.of(CATALOG, PRINCIPAL, PRINCIPAL_ROLE, TASK) + .flatMap( + t -> { + var subTypes = + Arrays.stream(PolarisEntitySubType.values()) + .filter(s -> s.getParentType() == t) + .toList(); + + return subTypes.isEmpty() + ? Stream.of(Arguments.of(t, NULL_SUBTYPE)) + : subTypes.stream().map(s -> Arguments.of(t, s)); + }); + } + + static Stream catalogScopedEntityTypes() { + return Stream.of(CATALOG_ROLE, NAMESPACE, TABLE_LIKE, POLICY) + .flatMap( + t -> { + var subTypes = + Arrays.stream(PolarisEntitySubType.values()) + .filter(s -> s.getParentType() == t) + .toList(); + + return subTypes.isEmpty() + ? Stream.of(Arguments.of(t, NULL_SUBTYPE)) + : subTypes.stream().map(s -> Arguments.of(t, s)); + }); + } +} diff --git a/persistence/nosql/persistence/metastore/src/test/java/org/apache/polaris/persistence/nosql/metastore/indexaccess/TestMemoizedIndexAccess.java b/persistence/nosql/persistence/metastore/src/test/java/org/apache/polaris/persistence/nosql/metastore/indexaccess/TestMemoizedIndexAccess.java new file mode 100644 index 0000000000..fc8319b32b --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/test/java/org/apache/polaris/persistence/nosql/metastore/indexaccess/TestMemoizedIndexAccess.java @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.persistence.nosql.metastore.indexaccess; + +import static org.apache.polaris.core.entity.PolarisEntityConstants.getNullId; +import static org.apache.polaris.core.entity.PolarisEntityConstants.getRootEntityId; +import static org.apache.polaris.core.entity.PolarisEntitySubType.NULL_SUBTYPE; +import static org.apache.polaris.core.entity.PolarisEntityType.CATALOG; +import static org.apache.polaris.core.entity.PolarisEntityType.CATALOG_ROLE; +import static org.apache.polaris.core.entity.PolarisEntityType.NAMESPACE; +import static org.apache.polaris.core.entity.PolarisEntityType.POLICY; +import static org.apache.polaris.core.entity.PolarisEntityType.PRINCIPAL; +import static org.apache.polaris.core.entity.PolarisEntityType.PRINCIPAL_ROLE; +import static org.apache.polaris.core.entity.PolarisEntityType.TABLE_LIKE; +import static org.apache.polaris.core.entity.PolarisEntityType.TASK; +import static org.apache.polaris.persistence.nosql.coretypes.mapping.EntityObjMappings.byEntityType; + +import io.smallrye.common.annotation.Identifier; +import jakarta.inject.Inject; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Stream; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisEntityCore; +import org.apache.polaris.core.entity.PolarisEntitySubType; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.entity.PrincipalEntity; +import org.apache.polaris.core.persistence.MetaStoreManagerFactory; +import org.apache.polaris.core.persistence.PolarisMetaStoreManager; +import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; +import org.apache.polaris.persistence.nosql.api.Persistence; +import org.apache.polaris.persistence.nosql.api.RealmPersistenceFactory; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; +import org.jboss.weld.junit5.EnableWeld; +import org.jboss.weld.junit5.WeldInitiator; +import org.jboss.weld.junit5.WeldSetup; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +@SuppressWarnings("CdiInjectionPointsInspection") +@ExtendWith(SoftAssertionsExtension.class) +@EnableWeld +public class TestMemoizedIndexAccess { + @InjectSoftAssertions public SoftAssertions soft; + + @WeldSetup WeldInitiator weld = WeldInitiator.performDefaultDiscovery(); + + @Inject RealmPersistenceFactory realmPersistenceFactory; + + @Inject + @Identifier("nosql") + MetaStoreManagerFactory metaStoreManagerFactory; + + PolarisCallContext callContext; + PolarisMetaStoreManager metaStoreManager; + + Persistence persistence; + MemoizedIndexedAccess memoized; + + @BeforeEach + public void setup(TestInfo testInfo) { + var realmId = testInfo.getTestMethod().orElseThrow().getName(); + persistence = realmPersistenceFactory.newBuilder().realmId(realmId).build(); + + metaStoreManagerFactory.bootstrapRealms(List.of(realmId), RootCredentialsSet.EMPTY); + + var realmContext = (RealmContext) () -> realmId; + callContext = + new PolarisCallContext( + realmContext, metaStoreManagerFactory.getOrCreateSession(realmContext)); + metaStoreManager = metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext); + + memoized = MemoizedIndexedAccess.newMemoizedIndexedAccess(persistence); + } + + @ParameterizedTest + @MethodSource("realmScopedEntityTypes") + public void realmScoped(PolarisEntityType entityType, PolarisEntitySubType subType) { + checkType(0L, List.of(), entityType, subType); + } + + @ParameterizedTest + @MethodSource("catalogScopedEntityTypes") + public void catalogScoped(PolarisEntityType entityType, PolarisEntitySubType subType) { + var catalogId = persistence.generateId(); + var catalogEntity = + new PolarisBaseEntity( + getNullId(), catalogId, CATALOG, NULL_SUBTYPE, getRootEntityId(), "my-catalog"); + metaStoreManager.createCatalog(callContext, catalogEntity, List.of()); + + checkType(catalogId, List.of(catalogEntity), entityType, subType); + } + + private void checkType( + long catalogId, + List catalogPath, + PolarisEntityType entityType, + PolarisEntitySubType subType) { + var entityName = "entity-for-" + entityType.name(); + long entityId = persistence.generateId(); + var entityForType = + new PolarisBaseEntity( + catalogId, entityId, entityType, subType, getRootEntityId(), entityName); + + if (entityType == PRINCIPAL) { + metaStoreManager.createPrincipal(callContext, new PrincipalEntity(entityForType)); + } else { + metaStoreManager.createEntityIfNotExists(callContext, catalogPath, entityForType); + } + + var ia = memoized.indexedAccess(catalogId, entityType.getCode()); + soft.assertThat(ia).isInstanceOf(IndexedContainerAccessImpl.class); + soft.assertThat(memoized.indexedAccess(catalogId, entityType.getCode())).isSameAs(ia); + + var mapping = byEntityType(entityType); + var refName = mapping.refNameForCatalog(catalogId); + var referenceHead = memoized.referenceHead(refName, mapping.containerObjTypeClass()); + soft.assertThat(referenceHead).isNotEmpty().get().isInstanceOf(mapping.containerObjTypeClass()); + + // "populate" the ICA's data + soft.assertThat(ia.byNameOnRoot(entityName)).isNotEmpty(); + + var entityName2 = "entity2-for-" + entityType.name(); + long entityId2 = persistence.generateId(); + var entityForType2 = + new PolarisBaseEntity( + catalogId, entityId2, entityType, subType, getRootEntityId(), entityName2); + if (entityType == PRINCIPAL) { + metaStoreManager.createPrincipal(callContext, new PrincipalEntity(entityForType2)); + } else { + metaStoreManager.createEntityIfNotExists(callContext, catalogPath, entityForType2); + } + + soft.assertThat(ia.byNameOnRoot(entityName)).isNotEmpty(); + soft.assertThat(ia.byNameOnRoot(entityName2)).isEmpty(); + + var ia2 = memoized.indexedAccess(catalogId, entityType.getCode()); + // Memoized index detects reference-pointer bump. + soft.assertThat(ia2).isNotSameAs(ia); + soft.assertThat(memoized.indexedAccess(catalogId, entityType.getCode())).isSameAs(ia2); + + memoized.invalidateIndexedAccess(catalogId, entityType.getCode()); + + var ia3 = memoized.indexedAccess(catalogId, entityType.getCode()); + soft.assertThat(ia3.byNameOnRoot(entityName)).isNotEmpty(); + soft.assertThat(ia3.byNameOnRoot(entityName2)).isNotEmpty(); + + // reference head changed + soft.assertThat(persistence.fetchReferenceHead(refName, mapping.containerObjTypeClass())) + .isNotEqualTo(referenceHead); + // memoized value should have refreshed + soft.assertThat(memoized.referenceHead(refName, mapping.containerObjTypeClass())) + .isNotEqualTo(referenceHead); + memoized.invalidateReferenceHead(refName); + soft.assertThat(memoized.referenceHead(refName, mapping.containerObjTypeClass())) + .get() + .isNotEqualTo(referenceHead); + } + + static Stream realmScopedEntityTypes() { + return Stream.of(CATALOG, PRINCIPAL, PRINCIPAL_ROLE, TASK) + .flatMap( + t -> { + var subTypes = + Arrays.stream(PolarisEntitySubType.values()) + .filter(s -> s.getParentType() == t) + .toList(); + + return subTypes.isEmpty() + ? Stream.of(Arguments.of(t, NULL_SUBTYPE)) + : subTypes.stream().map(s -> Arguments.of(t, s)); + }); + } + + static Stream catalogScopedEntityTypes() { + return Stream.of(CATALOG_ROLE, NAMESPACE, TABLE_LIKE, POLICY) + .flatMap( + t -> { + var subTypes = + Arrays.stream(PolarisEntitySubType.values()) + .filter(s -> s.getParentType() == t) + .toList(); + + return subTypes.isEmpty() + ? Stream.of(Arguments.of(t, NULL_SUBTYPE)) + : subTypes.stream().map(s -> Arguments.of(t, s)); + }); + } +} diff --git a/persistence/nosql/persistence/metastore/src/test/resources/logback-test.xml b/persistence/nosql/persistence/metastore/src/test/resources/logback-test.xml new file mode 100644 index 0000000000..b0249025e4 --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/test/resources/logback-test.xml @@ -0,0 +1,35 @@ + + + + + + + %date{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + diff --git a/persistence/nosql/persistence/metastore/src/test/resources/weld.properties b/persistence/nosql/persistence/metastore/src/test/resources/weld.properties new file mode 100644 index 0000000000..c26169e0e1 --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/test/resources/weld.properties @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# See https://bugs.openjdk.org/browse/JDK-8349545 +org.jboss.weld.bootstrap.concurrentDeployment=false diff --git a/persistence/nosql/persistence/metastore/src/testFixtures/java/org/apache/polaris/persistence/nosql/metastore/CdiProducers.java b/persistence/nosql/persistence/metastore/src/testFixtures/java/org/apache/polaris/persistence/nosql/metastore/CdiProducers.java new file mode 100644 index 0000000000..79de0d6ae1 --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/testFixtures/java/org/apache/polaris/persistence/nosql/metastore/CdiProducers.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.nosql.metastore; + +import jakarta.annotation.Nullable; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.inject.Produces; +import java.time.Clock; +import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; +import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.config.PolarisConfigurationStore; +import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; +import org.apache.polaris.core.storage.PolarisStorageIntegration; +import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; + +@ApplicationScoped +public class CdiProducers { + @Produces + PolarisStorageIntegrationProvider producePolarisStorageIntegrationProvider() { + return new PolarisStorageIntegrationProvider() { + @Override + public @Nullable + PolarisStorageIntegration getStorageIntegrationForConfig( + PolarisStorageConfigurationInfo polarisStorageConfigurationInfo) { + throw new UnsupportedOperationException(); + } + }; + } + + @Produces + PolarisConfigurationStore producePolarisConfigurationStore() { + return new PolarisConfigurationStore() {}; + } + + @Produces + PolarisDiagnostics producePolarisDiagnostics() { + return new PolarisDefaultDiagServiceImpl(); + } + + @Produces + Clock produceClock() { + return Clock.systemUTC(); + } +} diff --git a/persistence/nosql/persistence/metastore/src/testFixtures/resources/META-INF/beans.xml b/persistence/nosql/persistence/metastore/src/testFixtures/resources/META-INF/beans.xml new file mode 100644 index 0000000000..a297f1aa53 --- /dev/null +++ b/persistence/nosql/persistence/metastore/src/testFixtures/resources/META-INF/beans.xml @@ -0,0 +1,24 @@ + + + + + \ No newline at end of file From 3bfd728ba344e93f661501239d08eba36417d080 Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Tue, 9 Dec 2025 08:56:52 +0100 Subject: [PATCH 2/9] Remove a actually unnecessary parameter --- .../nosql/metastore/NoSqlMetaStoreManager.java | 15 ++++----------- .../metastore/NoSqlMetaStoreManagerFactory.java | 5 +---- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NoSqlMetaStoreManager.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NoSqlMetaStoreManager.java index a10d960634..91fa72cea2 100644 --- a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NoSqlMetaStoreManager.java +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NoSqlMetaStoreManager.java @@ -79,16 +79,9 @@ import org.apache.polaris.persistence.nosql.metastore.privs.SecurableGranteePrivilegeTuple; record NoSqlMetaStoreManager( - Supplier purgeRealm, - RootCredentialsSet rootCredentialsSet, - Supplier metaStoreSupplier, - Clock clock) + Supplier purgeRealm, RootCredentialsSet rootCredentialsSet, Clock clock) implements PolarisMetaStoreManager { - NoSqlMetaStore ms() { - return metaStoreSupplier.get(); - } - NoSqlMetaStore ms(PolarisCallContext callContext) { var existing = callContext.getMetaStore(); checkArgument(existing instanceof NoSqlMetaStore, "No meta store found in call context"); @@ -349,7 +342,7 @@ public ListEntitiesResult listEntities( @Nonnull @Override public GenerateEntityIdResult generateNewEntityId(@Nonnull PolarisCallContext callCtx) { - return new GenerateEntityIdResult(metaStoreSupplier.get().generateNewId()); + return new GenerateEntityIdResult(ms(callCtx).generateNewId()); } @Nonnull @@ -537,7 +530,7 @@ public EntityResult loadEntity( public Optional> hasOverlappingSiblings( @Nonnull PolarisCallContext callContext, T entity) { - return Optional.of(ms().hasOverlappingSiblings(entity)); + return Optional.of(ms(callContext).hasOverlappingSiblings(entity)); } @Nonnull @@ -787,7 +780,7 @@ public ScopedCredentialsResult getSubscopedCredsForEntity( !allowedReadLocations.isEmpty() || !allowedWriteLocations.isEmpty(), "allowed_locations_to_subscope_is_required"); - // reload the entity, error out if not found + // reload the entity or error out if not found var reloadedEntity = loadEntity(callCtx, catalogId, entityId, entityType); if (reloadedEntity.getReturnStatus() != BaseResult.ReturnStatus.SUCCESS) { return new ScopedCredentialsResult( diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NoSqlMetaStoreManagerFactory.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NoSqlMetaStoreManagerFactory.java index ad778b236b..749e0f7dcc 100644 --- a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NoSqlMetaStoreManagerFactory.java +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NoSqlMetaStoreManagerFactory.java @@ -143,10 +143,8 @@ public BasePersistence getOrCreateSession(RealmContext realmContext) { @Override public PolarisMetaStoreManager getOrCreateMetaStoreManager(RealmContext realmContext) { var realmId = realmContext.getRealmIdentifier(); - var persistence = initializedRealmPersistence(realmId); - return new NoSqlMetaStoreManager( - () -> purgeRealm(realmId), null, () -> newPersistenceMetaStore(persistence), clock); + return new NoSqlMetaStoreManager(() -> purgeRealm(realmId), null, clock); } private NoSqlMetaStore newPersistenceMetaStore(Persistence persistence) { @@ -267,7 +265,6 @@ private PrincipalSecretsResult bootstrapRealm( throw new IllegalStateException("Cannot purge while bootstrapping"); }, rootCredentialsSet, - () -> metaStore, clock); var secretsResult = From cad46920127caa11ce5f31f3f9cb75806b3195e7 Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Tue, 9 Dec 2025 14:38:12 +0100 Subject: [PATCH 3/9] adapt test --- .../nosql/metastore/TestNoSqlMetaStoreManager.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/persistence/nosql/persistence/metastore/src/test/java/org/apache/polaris/persistence/nosql/metastore/TestNoSqlMetaStoreManager.java b/persistence/nosql/persistence/metastore/src/test/java/org/apache/polaris/persistence/nosql/metastore/TestNoSqlMetaStoreManager.java index 9dcb12cf71..e6a15e96ab 100644 --- a/persistence/nosql/persistence/metastore/src/test/java/org/apache/polaris/persistence/nosql/metastore/TestNoSqlMetaStoreManager.java +++ b/persistence/nosql/persistence/metastore/src/test/java/org/apache/polaris/persistence/nosql/metastore/TestNoSqlMetaStoreManager.java @@ -256,4 +256,12 @@ EntityResult createEntity( var path = (List) (List) catalogPath; return metaStore.createEntityIfNotExists(callContext, path, newEntity); } + + @Override + @Test + @Disabled( + "The test is not applicable to the NoSQL metastore implementation, " + + "because both the creation and modification timestamps are enforced by the implementation and " + + "cannot be tweaked by call sites") + protected void testCreatePrincipalReturnedEntitySameAsPersisted() {} } From c62981050cb0a0279943c0f21afc246f93d0d897 Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Thu, 11 Dec 2025 10:54:32 +0100 Subject: [PATCH 4/9] ci javadoc --- .../nosql/metastore/ContentIdentifier.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/ContentIdentifier.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/ContentIdentifier.java index 6f11f71e4c..1680f7e13b 100644 --- a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/ContentIdentifier.java +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/ContentIdentifier.java @@ -27,6 +27,7 @@ import java.util.List; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.polaris.core.storage.StorageLocation; import org.apache.polaris.immutables.PolarisImmutable; import org.apache.polaris.persistence.nosql.api.index.IndexKey; import org.apache.polaris.service.types.PolicyIdentifier; @@ -63,13 +64,22 @@ static ContentIdentifier identifierFor(Namespace namespace) { return identifier(namespace.levels()); } + /** + * Constructs a {@link ContentIdentifier} from a storage location without the scheme. + * + *

This is used for the locations index for location-overlap checks. + * + *

Valid inputs follow the values returned by {@link StorageLocation#withoutScheme() + * StorageLocation.of(previousBaseLocation).withoutScheme()}, which can, in case of "file" + * locations, contain system-specific file separators. + */ static ContentIdentifier identifierFromLocationString(String locationString) { var builder = builder(); var len = locationString.length(); var off = -1; for (var i = 0; i < len; i++) { var c = locationString.charAt(i); - checkArgument(c >= ' ', "Control characters are forbidden in locations"); + checkArgument(c >= ' ' && c != 127, "Control characters are forbidden in locations"); if (c == '/' || c == '\\') { if (off != -1) { builder.addElements(locationString.substring(off, i)); From 2b6f81edab0753ee10eeb7051cf672c213897ee9 Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Thu, 11 Dec 2025 10:54:42 +0100 Subject: [PATCH 5/9] rm unnecessary fallthrough --- .../persistence/nosql/metastore/NoSqlMetaStore.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NoSqlMetaStore.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NoSqlMetaStore.java index dee536976b..566eb4b4f4 100644 --- a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NoSqlMetaStore.java +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NoSqlMetaStore.java @@ -289,13 +289,9 @@ CreateCatalogResult createCatalog( byName.put(nameKey, objRef(catalogObj)); byId.put(idKey, nameKey); - if (existing == null) { - // created - return new ChangeResult.CommitChange<>( - new CreateCatalogResult(catalog, catalogAdminRole)); - } - // retry - return new ChangeResult.NoChange<>(new CreateCatalogResult(ENTITY_ALREADY_EXISTS, null)); + // created + return new ChangeResult.CommitChange<>( + new CreateCatalogResult(catalog, catalogAdminRole)); })); } From 489e30969357ed6245bba8bae9487f3b92992e8c Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Thu, 11 Dec 2025 10:56:44 +0100 Subject: [PATCH 6/9] rm unnecessary duplication --- .../persistence/nosql/metastore/mutation/MutationAttempt.java | 2 +- .../persistence/nosql/metastore/mutation/MutationResults.java | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/MutationAttempt.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/MutationAttempt.java index a34853a017..b0497f6707 100644 --- a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/MutationAttempt.java +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/MutationAttempt.java @@ -490,7 +490,7 @@ private static void applyEntityCreateMutation( if (originalObj != null) { var unchangedCompareObj = objForChangeComparison(entity, Optional.empty(), originalObj); if (unchangedCompareObj.equals(originalObj)) { - mutationResults.entityResultNoChange(entity); + mutationResults.unchangedEntityResult(entity); return; } } diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/MutationResults.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/MutationResults.java index 9b2b1d9745..9d6e14b57a 100644 --- a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/MutationResults.java +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/MutationResults.java @@ -91,10 +91,6 @@ void entityResult(PolarisBaseEntity entity) { anyChange = true; } - void entityResultNoChange(PolarisBaseEntity entity) { - add(new EntityResult(entity)); - } - void unchangedEntityResult(PolarisBaseEntity entity) { add(new EntityResult(entity)); } From 8e65b35b041bdd5408bfd5308079fe7ec09e740e Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Thu, 11 Dec 2025 11:11:07 +0100 Subject: [PATCH 7/9] gt javadoc --- .../nosql/metastore/privs/GrantTriplet.java | 60 +++++++++++++++---- 1 file changed, 47 insertions(+), 13 deletions(-) diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/privs/GrantTriplet.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/privs/GrantTriplet.java index aa1ed10dba..07b9d22e76 100644 --- a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/privs/GrantTriplet.java +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/privs/GrantTriplet.java @@ -21,12 +21,58 @@ import static com.google.common.base.Preconditions.checkArgument; import org.apache.polaris.core.entity.PolarisEntityCore; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.persistence.nosql.api.index.IndexKey; +import org.apache.polaris.persistence.nosql.coretypes.ObjBase; +import org.apache.polaris.persistence.nosql.coretypes.acl.GrantsObj; /** * Represents the triplet of catalog-ID, entity-ID and type-code plus a reverse-or-key marker. - * String representations of this type are used as ACL names and "role" names. + * + *

This is intended to construct {@link IndexKey}s for {@link GrantsObj#acls}, which contain the + * "reversed" and "directed" mappings. + * + *

    + *
  • "Reversed" ({@code reverseOrKey == true}, role encoding {@code 'r'}) means from a + * grantee or securable. + *
  • "Directed" ({@code reverseOrKey == false}, role encoding {@code 'd'}) means to a + * grantee (no use case for to securable). + *
+ * + *

String representations of this type are used as ACL names and "role" names. + * + * @param reverseOrKey + * @param catalogId catalog id + * @param id entity id (aka {@link ObjBase#stableId()} + * @param typeCode {@link PolarisEntityType#getCode() entity type code} */ public record GrantTriplet(boolean reverseOrKey, long catalogId, long id, int typeCode) { + + /** + * Constructs a new {@link GrantTriplet} instance for the given entity, with {@link + * #reverseOrKey()} set to {@code true}. + */ + public static GrantTriplet forEntity(PolarisEntityCore entity) { + return new GrantTriplet(true, entity.getCatalogId(), entity.getId(), entity.getTypeCode()); + } + + /** Convert to a "directed" grant-triplet, having {@link #reverseOrKey()} set to {@code false}. */ + public GrantTriplet asDirected() { + return new GrantTriplet(false, catalogId, id, typeCode); + } + + /** + * Constructs the role name, the encoded string representation, for this triplet in the pattern + * {@code [r|d] "/" catalogId "/" id "/" typeCode} + */ + public String toRoleName() { + return (reverseOrKey ? "r/" : "d/") + catalogId + "/" + id + "/" + typeCode; + } + + /** + * Parses a {@link GrantTriplet#toRoleName()}, expecting exactly the pattern {@code [r|d] "/" + * catalogId "/" id "/" typeCode}. + */ public static GrantTriplet fromRoleName(String roleName) { var c0 = roleName.charAt(0); checkArgument(roleName.charAt(1) == '/' && (c0 == 'r' || c0 == 'd')); @@ -42,16 +88,4 @@ public static GrantTriplet fromRoleName(String roleName) { return new GrantTriplet(reversed, catalogId, id, typeCode); } - - public static GrantTriplet forEntity(PolarisEntityCore entity) { - return new GrantTriplet(true, entity.getCatalogId(), entity.getId(), entity.getTypeCode()); - } - - public GrantTriplet asDirected() { - return new GrantTriplet(false, catalogId, id, typeCode); - } - - public String toRoleName() { - return (reverseOrKey ? "r/" : "d/") + catalogId + "/" + id + "/" + typeCode; - } } From e73f87a81fc6dca688b076faa74d8b2502888f90 Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Fri, 12 Dec 2025 13:13:23 +0100 Subject: [PATCH 8/9] review --- .../nosql/metastore/mutation/MutationAttempt.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/MutationAttempt.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/MutationAttempt.java index b0497f6707..312e699a94 100644 --- a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/MutationAttempt.java +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/mutation/MutationAttempt.java @@ -87,12 +87,17 @@ public record MutationAttempt( private static final Logger LOGGER = LoggerFactory.getLogger(MutationAttempt.class); + /** + * Produces an {@linkplain ObjBase obj instance} that can be used to detect whether an update + * operation results in an actual change or a noop. For this, the entity is mapped to an object, + * but the technical attributes are taken from the existing object. + */ public static ObjBase objForChangeComparison( PolarisBaseEntity entity, Optional currentSecrets, ObjBase originalObj) { return mapToObj(entity, currentSecrets) - .updateTimestamp(originalObj.createTimestamp()) + .updateTimestamp(originalObj.updateTimestamp()) .id(originalObj.id()) .numParts(originalObj.numParts()) .entityVersion(originalObj.entityVersion()) @@ -355,10 +360,7 @@ private static void applyEntityUpdateMutation( mutationResults.entityResult(ENTITY_CANNOT_BE_RENAMED); return; } - if (!byName.remove(originalNameKey)) { - mutationResults.entityResult(ENTITY_NOT_FOUND); - return; - } + checkState(byName.remove(originalNameKey)); var newNameKey = nameKeyForEntity(entity, byId, mutationResults::entityResult); if (newNameKey == null) { From 4d07106f4c15bae7c665e63305c2f7bec510e0a2 Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Fri, 12 Dec 2025 16:50:32 +0100 Subject: [PATCH 9/9] rebase-fix --- .../persistence/nosql/metastore/NoSqlMetaStoreManager.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NoSqlMetaStoreManager.java b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NoSqlMetaStoreManager.java index 91fa72cea2..9ecf0737c0 100644 --- a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NoSqlMetaStoreManager.java +++ b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NoSqlMetaStoreManager.java @@ -37,6 +37,7 @@ import java.util.function.Function; import java.util.function.Supplier; import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.auth.PolarisPrincipal; import org.apache.polaris.core.entity.LocationBasedEntity; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisEntity; @@ -774,6 +775,7 @@ public ScopedCredentialsResult getSubscopedCredsForEntity( boolean allowListOperation, @Nonnull Set allowedReadLocations, @Nonnull Set allowedWriteLocations, + @Nonnull PolarisPrincipal polarisPrincipal, Optional refreshCredentialsEndpoint) { checkArgument( @@ -804,6 +806,7 @@ public ScopedCredentialsResult getSubscopedCredsForEntity( allowListOperation, allowedReadLocations, allowedWriteLocations, + polarisPrincipal, refreshCredentialsEndpoint); return new ScopedCredentialsResult(creds); } catch (Exception ex) {