-
Notifications
You must be signed in to change notification settings - Fork 3.7k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
chore: Cherry pick indexes for
policyMap
to improve query response …
…time chore: Cherry pick indexes for `policyMap` to improve query response time
- Loading branch information
Showing
4 changed files
with
156 additions
and
1 deletion.
There are no files selected for viewing
3 changes: 2 additions & 1 deletion
3
...th-server/src/main/java/com/appsmith/server/migrations/constants/DeprecatedFieldName.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,5 +1,6 @@ | ||
package com.appsmith.server.migrations.constants; | ||
|
||
public class DeprecatedFieldName { | ||
public static String POLICIES = "policies"; | ||
public static final String POLICIES = "policies"; | ||
public static final String DELETED = "deleted"; | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
93 changes: 93 additions & 0 deletions
93
...rc/main/java/com/appsmith/server/migrations/db/ce/Migration060AddIndexesForPolicyMap.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,93 @@ | ||
package com.appsmith.server.migrations.db.ce; | ||
|
||
import com.appsmith.server.domains.Workspace; | ||
import io.mongock.api.annotations.ChangeUnit; | ||
import io.mongock.api.annotations.Execution; | ||
import io.mongock.api.annotations.RollbackExecution; | ||
import lombok.RequiredArgsConstructor; | ||
import lombok.extern.slf4j.Slf4j; | ||
import org.springframework.data.mongodb.UncategorizedMongoDbException; | ||
import org.springframework.data.mongodb.core.MongoTemplate; | ||
import org.springframework.data.mongodb.core.index.Index; | ||
import reactor.core.publisher.Mono; | ||
import reactor.core.scheduler.Schedulers; | ||
|
||
import static com.appsmith.external.helpers.StringUtils.dotted; | ||
import static com.appsmith.server.migrations.DatabaseChangelog1.dropIndexIfExists; | ||
import static com.appsmith.server.migrations.DatabaseChangelog1.ensureIndexes; | ||
import static com.appsmith.server.migrations.DatabaseChangelog1.makeIndex; | ||
import static com.appsmith.server.migrations.constants.DeprecatedFieldName.DELETED; | ||
import static com.appsmith.server.migrations.constants.FieldName.DELETED_AT; | ||
import static com.appsmith.server.migrations.constants.FieldName.PERMISSION_GROUPS; | ||
import static com.appsmith.server.migrations.constants.FieldName.POLICY_MAP; | ||
import static com.appsmith.server.migrations.constants.FieldName.TENANT_ID; | ||
|
||
/** | ||
* This migration adds indexes to the policyMap fields within application and workspace collection to speed up queries. | ||
* This migration also adds a compound index on the deleted and deletedAt fields to speed up queries that filter on | ||
* these fields. | ||
* Ideally we should rely on @see <a href="https://www.mongodb.com/docs/v5.0/core/index-wildcard/#wildcard-indexes">Wildcard Indexes</a>, | ||
* but this may end up hogging a lot of memory as it recursively create index on policyMap, hence we are just creating | ||
* the compound index which have the most impact. | ||
*/ | ||
@RequiredArgsConstructor | ||
@Slf4j | ||
@ChangeUnit(order = "060", id = "add-idx-policy-map", author = " ") | ||
public class Migration060AddIndexesForPolicyMap { | ||
private final MongoTemplate mongoTemplate; | ||
|
||
@RollbackExecution | ||
public void rollbackExecution() {} | ||
|
||
@Execution | ||
public void executeMigration() { | ||
String readWorkspaceTanantIdCompoundIndex = "policy_read_workspace_tanantId_compound_index"; | ||
String policyMapReadWorkspacePath = dotted(POLICY_MAP, "read:workspaces", PERMISSION_GROUPS); | ||
|
||
String manageWorkspaceTanantIdCompoundIndex = "policy_manage_workspace_tanantId_compound_index"; | ||
String policyMapManageWorkspacePath = dotted(POLICY_MAP, "manage:workspaces", PERMISSION_GROUPS); | ||
|
||
Mono<Boolean> readWorkspaceMono = Mono.fromCallable(() -> { | ||
log.debug("Adding read workspace policy map indices"); | ||
createAndApplyIndex( | ||
readWorkspaceTanantIdCompoundIndex, | ||
Workspace.class, | ||
policyMapReadWorkspacePath, | ||
TENANT_ID, | ||
DELETED, | ||
DELETED_AT); | ||
return true; | ||
}) | ||
.subscribeOn(Schedulers.boundedElastic()); | ||
|
||
Mono<Boolean> manageWorkspaceMono = Mono.fromCallable(() -> { | ||
log.debug("Adding manage workspace policy map indices"); | ||
createAndApplyIndex( | ||
manageWorkspaceTanantIdCompoundIndex, | ||
Workspace.class, | ||
policyMapManageWorkspacePath, | ||
TENANT_ID, | ||
DELETED, | ||
DELETED_AT); | ||
return true; | ||
}) | ||
.subscribeOn(Schedulers.boundedElastic()); | ||
|
||
Mono.zip(readWorkspaceMono, manageWorkspaceMono).block(); | ||
} | ||
|
||
private void createAndApplyIndex(String indexName, Class<?> clazz, String... fields) { | ||
try { | ||
Index index = makeIndex(fields).named(indexName); | ||
dropIndexIfExists(mongoTemplate, clazz, indexName); | ||
ensureIndexes(mongoTemplate, clazz, index); | ||
} catch (UncategorizedMongoDbException exception) { | ||
log.error( | ||
"An error occurred while creating the index : {}, skipping the addition of index because of {}.", | ||
indexName, | ||
exception.getMessage()); | ||
} catch (Exception e) { | ||
log.error("An error occurred while creating the index : {}", indexName, e); | ||
} | ||
} | ||
} |
58 changes: 58 additions & 0 deletions
58
...ain/java/com/appsmith/server/migrations/db/ce/Migration061TenantPolicySetToPolicyMap.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,58 @@ | ||
package com.appsmith.server.migrations.db.ce; | ||
|
||
import com.appsmith.external.models.Policy; | ||
import com.appsmith.server.domains.Tenant; | ||
import com.appsmith.server.helpers.CollectionUtils; | ||
import com.appsmith.server.repositories.CacheableRepositoryHelper; | ||
import io.mongock.api.annotations.ChangeUnit; | ||
import io.mongock.api.annotations.Execution; | ||
import io.mongock.api.annotations.RollbackExecution; | ||
import lombok.extern.slf4j.Slf4j; | ||
import org.springframework.data.mongodb.core.MongoTemplate; | ||
import org.springframework.data.mongodb.core.query.Query; | ||
|
||
import java.util.HashMap; | ||
import java.util.Map; | ||
|
||
import static com.appsmith.server.constants.ce.FieldNameCE.DEFAULT; | ||
import static org.springframework.data.mongodb.core.query.Criteria.where; | ||
|
||
@Slf4j | ||
@ChangeUnit(order = "061", id = "migrate-policy-set-to-map-tenant", author = " ") | ||
public class Migration061TenantPolicySetToPolicyMap { | ||
private final MongoTemplate mongoTemplate; | ||
private final CacheableRepositoryHelper cacheableRepositoryHelper; | ||
|
||
public Migration061TenantPolicySetToPolicyMap( | ||
MongoTemplate mongoTemplate, CacheableRepositoryHelper cacheableRepositoryHelper) { | ||
this.mongoTemplate = mongoTemplate; | ||
this.cacheableRepositoryHelper = cacheableRepositoryHelper; | ||
} | ||
|
||
@RollbackExecution | ||
public void rollbackExecution() {} | ||
|
||
@Execution | ||
public void executeMigration() { | ||
// Fetch default tenant and verify earlier migration has updated the policyMap field | ||
Query tenantQuery = new Query(); | ||
tenantQuery.addCriteria(where(Tenant.Fields.slug).is(DEFAULT)); | ||
Tenant defaultTenant = mongoTemplate.findOne(tenantQuery, Tenant.class); | ||
if (defaultTenant == null) { | ||
log.error( | ||
"No default tenant found. Aborting migration to update policy set to map in tenant Migration061TenantPolicySetToPolicyMap."); | ||
return; | ||
} | ||
// Evict the tenant to avoid any cache inconsistencies | ||
if (CollectionUtils.isNullOrEmpty(defaultTenant.getPolicyMap())) { | ||
Map<String, Policy> policyMap = new HashMap<>(); | ||
defaultTenant.getPolicies().forEach(policy -> policyMap.put(policy.getPermission(), policy)); | ||
defaultTenant.setPolicyMap(policyMap); | ||
mongoTemplate.save(defaultTenant); | ||
cacheableRepositoryHelper.evictCachedTenant(defaultTenant.getId()).block(); | ||
} else { | ||
log.info( | ||
"Tenant already has policyMap set. Skipping migration to update policy set to map in tenant Migration061TenantPolicySetToPolicyMap."); | ||
} | ||
} | ||
} |