From 2f5bd68bf12da5a170df1279a809cf4673445a3c Mon Sep 17 00:00:00 2001
From: Okke Harsta <oharsta@zilverline.com>
Date: Wed, 15 Jan 2025 15:42:22 +0100
Subject: [PATCH] New 9.0.0-SNAPSHOT version with PdP datasource removed

---
 manage-gui/pom.xml                            |   2 +-
 manage-gui/src/api/index.js                   |  16 --
 manage-gui/src/pages/Policies.jsx             | 114 +----------
 manage-server/pom.xml                         |  16 +-
 .../manage/control/MetaDataController.java    |  42 +++--
 .../java/manage/control/PdPController.java    | 178 +-----------------
 .../manage/policies/PolicyRepository.java     |  33 ----
 .../src/main/resources/application.yml        |  10 -
 .../control/MetaDataControllerTest.java       |  23 ++-
 pom.xml                                       |   2 +-
 10 files changed, 69 insertions(+), 367 deletions(-)
 delete mode 100644 manage-server/src/main/java/manage/policies/PolicyRepository.java

diff --git a/manage-gui/pom.xml b/manage-gui/pom.xml
index b99dccbe..589b7733 100644
--- a/manage-gui/pom.xml
+++ b/manage-gui/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>org.openconext</groupId>
         <artifactId>manage</artifactId>
-        <version>8.0.4-SNAPSHOT</version>
+        <version>9.0.0-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 
diff --git a/manage-gui/src/api/index.js b/manage-gui/src/api/index.js
index 15804c96..becbd9a3 100644
--- a/manage-gui/src/api/index.js
+++ b/manage-gui/src/api/index.js
@@ -309,22 +309,6 @@ export function getAllowedLoas() {
     return fetchJson("loas");
 }
 
-export function getPdPPolicies() {
-    return fetchJson("pdp/policies");
-}
-
-export function getMigratedPdPPolicies() {
-    return fetchJson("pdp/migrated_policies");
-}
-
-export function getPolicyPushAnalysis() {
-    return fetchJson("/pdp/push_analysis");
-}
-
-export function importPdPPolicies() {
-    return postPutJson("pdp/import_policies", {}, "PUT")
-}
-
 export function getPlaygroundPolicies() {
     return search({ALL_ATTRIBUTES: true}, "policy")
 }
diff --git a/manage-gui/src/pages/Policies.jsx b/manage-gui/src/pages/Policies.jsx
index 8e7ab231..50702c86 100644
--- a/manage-gui/src/pages/Policies.jsx
+++ b/manage-gui/src/pages/Policies.jsx
@@ -1,8 +1,7 @@
 import React from "react";
 import I18n from "i18n-js";
-import {isEmpty, stop} from "../utils/Utils";
+import {stop} from "../utils/Utils";
 import "./Policies.scss";
-import {getPolicyPushAnalysis, importPdPPolicies} from "../api";
 import PolicyPlayground from "../components/PolicyPlaygound";
 import withRouterHooks from "../utils/RouterBackwardCompatability";
 import PolicyMissingEnforcements from "../components/PolicyMissingEnforcements";
@@ -11,14 +10,14 @@ class Policies extends React.PureComponent {
 
     constructor(props) {
         super(props);
-        const tabs = ["import", "push", "playground", "missing_enforcements"];
-        const {tab = "import"} = props.params || {};
+        const tabs = ["playground", "missing_enforcements"];
+        const {tab = "playground"} = props.params || {};
         this.state = {
             tabs: tabs,
             selectedTab: tab,
             importResults: {},
             showMoreImported: false,
-            policyPushAnalysis: {differences:[], missing_policies:[]},
+            policyPushAnalysis: {differences: [], missing_policies: []},
             loading: false,
             copiedToClipboardClassName: "",
         };
@@ -34,12 +33,7 @@ class Policies extends React.PureComponent {
 
     initialState = e => {
         stop(e);
-        this.setState({loading: true});
-        getPolicyPushAnalysis()
-            .then(res => this.setState({
-                policyPushAnalysis: res,
-                loading: false
-            }));
+        this.setState({loading: false});
     }
 
     switchTab = tab => e => {
@@ -54,22 +48,9 @@ class Policies extends React.PureComponent {
                 pdpMigratedPolicies: []
             });
         }
-        if (tab === "push") {
-            this.setState({loading: true});
-            getPolicyPushAnalysis()
-                .then(res => this.setState({
-                    policyPushAnalysis: res,
-                    loading: false
-                }));
-        }
         this.props.navigate(`/policies/${tab}`);
     };
 
-    toggleShowMore = e => {
-        stop(e);
-        this.setState({showMoreImported: !this.state.showMoreImported})
-    }
-
     renderTab = (tab, selectedTab) =>
         <span key={tab}
               className={tab === selectedTab ? "active" : ""}
@@ -77,86 +58,6 @@ class Policies extends React.PureComponent {
             {I18n.t(`policies.${tab}`)}
         </span>;
 
-    runImport = () => {
-        this.setState({loading: true});
-        importPdPPolicies()
-            .then(res => this.setState({importResults: res, loading: false}))
-    }
-
-    renderImport = () => {
-        const {importResults, showMoreImported, loading} = this.state;
-        return (
-            <section className="import">
-                <p>Import the current PdP policies into Manage. Once imported they can be pushed.</p>
-                <a className={`button ${loading ? "grey disabled" : "green"}`}
-                   onClick={this.runImport}>
-                    {I18n.t("policies.runImport")}
-                </a>
-                {!isEmpty(importResults) &&
-                    <section className="results">
-                        <h2>Not imported policies</h2>
-                        <ul className="policies">
-                            {importResults.errors.map((data, index) =>
-                                <li key={index}>
-                                    <span>{data.name}</span>
-                                    <span>{data.error}</span>
-                                </li>)}
-                        </ul>
-                        <h2>Imported policies</h2>
-                        <a href={"/#show"}
-                           onClick={this.toggleShowMore}>
-                            {!showMoreImported ? "Show all" : "Hide"}
-                        </a>
-                        {showMoreImported && <ul className="policies">
-                            {importResults.imported.map((metaData, index) =>
-                                <li key={index}>
-                                    <span>{metaData.data.name}</span>
-                                    <span>{metaData.data.description}</span>
-                                </li>)}
-                        </ul>}
-                    </section>}
-            </section>
-        );
-    };
-
-    renderPush = () => {
-        const {policyPushAnalysis, loading} = this.state;
-        return (
-            <section className="import">
-                <p>After importing the current PdP policies into Manage and subsequently pushing those Manage policies
-                    to PdP, we now compare the original PdP policies with the pushed ones.</p>
-                <a className={`button ${loading ? "grey disabled" : "green"}`}
-                   onClick={e => this.initialState(e)}>
-                    {I18n.t("policies.reload")}
-                </a>
-                <section className="results">
-                    <h2># Total PDP policies </h2>
-                    <p>{policyPushAnalysis.policy_count}</p>
-                    <h2># Total active PDP policies </h2>
-                    <p>{policyPushAnalysis.active_policy_count}</p>
-                    <h2># Pushed policies</h2>
-                    <p>{policyPushAnalysis.migrated_policy_count}</p>
-                    <h2>Missing policies</h2>
-                    {policyPushAnalysis.missing_policies.length === 0 && <p>None missing</p>}
-                    <ul className="policies">
-                        {policyPushAnalysis.missing_policies.map((policy, index) => <li key={index}>
-                            <span>{policy.name}</span>
-                            <span>{policy.description}</span>
-                        </li>)}
-                    </ul>
-                    <h2>Diffs between policies</h2>
-                    {policyPushAnalysis.differences.length === 0 && <p>No diffs</p>}
-                    <ul className="policies">
-                        {policyPushAnalysis.differences.map((diff, index) => <li key={index}>
-                            <span>{Object.keys(diff)[0]}</span>
-                            <span>{Object.values(diff)[0]}</span>
-                        </li>)}
-                    </ul>
-                </section>
-            </section>
-        );
-    };
-
     renderPlayground = () => {
         return (
             <PolicyPlayground/>
@@ -171,10 +72,6 @@ class Policies extends React.PureComponent {
 
     renderCurrentTab = selectedTab => {
         switch (selectedTab) {
-            case "import" :
-                return this.renderImport();
-            case "push" :
-                return this.renderPush();
             case "playground" :
                 return this.renderPlayground();
             case "missing_enforcements" :
@@ -199,4 +96,5 @@ class Policies extends React.PureComponent {
         );
     }
 }
+
 export default withRouterHooks(Policies);
diff --git a/manage-server/pom.xml b/manage-server/pom.xml
index dec662ad..bd6969f9 100644
--- a/manage-server/pom.xml
+++ b/manage-server/pom.xml
@@ -6,7 +6,7 @@
     <parent>
         <groupId>org.openconext</groupId>
         <artifactId>manage</artifactId>
-        <version>8.0.4-SNAPSHOT</version>
+        <version>9.0.0-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 
@@ -62,20 +62,6 @@
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-data-mongodb</artifactId>
         </dependency>
-        <!-- https://mvnrepository.com/artifact/org.mariadb.jdbc/mariadb-java-client -->
-        <dependency>
-            <groupId>org.mariadb.jdbc</groupId>
-            <artifactId>mariadb-java-client</artifactId>
-            <version>3.3.2</version>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-boot-starter-jdbc</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-boot-starter-data-jpa</artifactId>
-        </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-validation</artifactId>
diff --git a/manage-server/src/main/java/manage/control/MetaDataController.java b/manage-server/src/main/java/manage/control/MetaDataController.java
index 7c472a85..4d345eeb 100644
--- a/manage-server/src/main/java/manage/control/MetaDataController.java
+++ b/manage-server/src/main/java/manage/control/MetaDataController.java
@@ -26,10 +26,7 @@
 import javax.xml.stream.XMLStreamException;
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
+import java.util.*;
 
 import static manage.api.Scope.TEST;
 import static manage.mongo.MongoChangelog.CHANGE_REQUEST_POSTFIX;
@@ -115,7 +112,7 @@ public List<StatsEntry> stats(FederatedUser user) {
     @PreAuthorize("hasAnyRole('WRITE_SP', 'WRITE_IDP', 'SYSTEM')")
     @PostMapping("/internal/metadata")
     public MetaData postInternal(@Validated @RequestBody MetaData metaData, APIUser apiUser) {
-        ScopeEnforcer.enforceWriteScope(apiUser, EntityType.fromType(metaData.getType()) );
+        ScopeEnforcer.enforceWriteScope(apiUser, EntityType.fromType(metaData.getType()));
         return metaDataService.doPost(metaData, apiUser, !apiUser.getScopes().contains(TEST));
     }
 
@@ -233,16 +230,39 @@ public MetaData put(@Validated @RequestBody MetaData metaData, FederatedUser use
     @Transactional
     public MetaData putInternal(@Validated @RequestBody MetaData metaData, APIUser apiUser)
             throws JsonProcessingException {
-        ScopeEnforcer.enforceWriteScope(apiUser, EntityType.fromType(metaData.getType()) );
+        ScopeEnforcer.enforceWriteScope(apiUser, EntityType.fromType(metaData.getType()));
         return metaDataService.doPut(metaData, apiUser, !apiUser.getScopes().contains(TEST));
     }
 
+    @PreAuthorize("hasAnyRole('SYSTEM')")
+    @PutMapping("/internal/removeExtraneousKeys/{type}")
+    @Transactional
+    public ResponseEntity<List<String>> removeExtraneousKeys(@PathVariable("type") String type, @RequestBody List<String> extraneousKeys, APIUser apiUser) {
+        LOG.info("RemoveExtraneousKeys called by {}", apiUser.getName());
+
+        List<String> results = new ArrayList<>();
+        List<MetaData> metaDataEntries = metaDataRepository.findAllByType(type);
+        metaDataEntries.forEach(metaData -> {
+            Map<String, Object> metaDataFields = metaData.metaDataFields();
+            Set<String> keySet = metaDataFields.keySet();
+            if (keySet.stream().anyMatch(key -> extraneousKeys.contains(key))) {
+                keySet.removeIf(key -> extraneousKeys.contains(key));
+
+                LOG.info(String.format("Saving %s metadata where extraneousKeys are removed", metaData.getData().get("entityid")));
+
+                metaDataRepository.update(metaData);
+                results.add((String) metaData.getData().get("entityid"));
+            }
+        });
+        return ResponseEntity.ok(results);
+    }
+
     @PreAuthorize("hasAnyRole('WRITE_SP', 'WRITE_IDP', 'SYSTEM')")
     @PutMapping("/internal/delete-metadata-key")
     @Transactional
     public List<String> deleteMetaDataKey(@Validated @RequestBody MetaDataKeyDelete metaDataKeyDelete,
                                           APIUser apiUser) throws IOException {
-        ScopeEnforcer.enforceWriteScope(apiUser, EntityType.fromType(metaDataKeyDelete.getType()) );
+        ScopeEnforcer.enforceWriteScope(apiUser, EntityType.fromType(metaDataKeyDelete.getType()));
         return metaDataService.deleteMetaDataKey(metaDataKeyDelete, apiUser);
     }
 
@@ -251,7 +271,7 @@ public List<String> deleteMetaDataKey(@Validated @RequestBody MetaDataKeyDelete
     @Transactional
     public MetaData update(@Validated @RequestBody MetaDataUpdate metaDataUpdate, APIUser apiUser)
             throws JsonProcessingException {
-        ScopeEnforcer.enforceWriteScope(apiUser, EntityType.fromType(metaDataUpdate.getType()) );
+        ScopeEnforcer.enforceWriteScope(apiUser, EntityType.fromType(metaDataUpdate.getType()));
         return metaDataService
                 .doMergeUpdate(metaDataUpdate, apiUser, "Internal API merge", true)
                 .get();
@@ -269,7 +289,7 @@ public List<MetaDataChangeRequest> changeRequests(@PathVariable("type") String t
     public List<MetaDataChangeRequest> internalChangeRequests(@PathVariable("type") String type,
                                                               @PathVariable("metaDataId") String metaDataId,
                                                               APIUser apiUser) {
-        ScopeEnforcer.enforceChangeRequestScope(apiUser, EntityType.fromType(type) );
+        ScopeEnforcer.enforceChangeRequestScope(apiUser, EntityType.fromType(type));
         return metaDataRepository.changeRequests(metaDataId, type.concat(CHANGE_REQUEST_POSTFIX));
     }
 
@@ -290,7 +310,7 @@ public long openChangeRequests() {
     @PostMapping("internal/change-requests")
     @Transactional
     public MetaDataChangeRequest changeRequestInternal(@Validated @RequestBody MetaDataChangeRequest metaDataChangeRequest, APIUser apiUser) throws JsonProcessingException {
-        ScopeEnforcer.enforceChangeRequestScope(apiUser, EntityType.fromType(metaDataChangeRequest.getType()) );
+        ScopeEnforcer.enforceChangeRequestScope(apiUser, EntityType.fromType(metaDataChangeRequest.getType()));
         return metaDataService.doChangeRequest(metaDataChangeRequest, apiUser);
     }
 
@@ -335,7 +355,7 @@ public DeleteResult removeChangeRequest(@PathVariable("type") String type,
     @PutMapping("/internal/change-requests/reject")
     @Transactional
     public MetaData internalRejectChangeRequest(@RequestBody @Validated ChangeRequest changeRequest, APIUser apiUser) {
-        ScopeEnforcer.enforceChangeRequestScope(apiUser, EntityType.fromType(changeRequest.getType()) );
+        ScopeEnforcer.enforceChangeRequestScope(apiUser, EntityType.fromType(changeRequest.getType()));
         return metaDataService.doRejectChangeRequest(changeRequest, apiUser);
     }
 
diff --git a/manage-server/src/main/java/manage/control/PdPController.java b/manage-server/src/main/java/manage/control/PdPController.java
index eb195435..8e7e4124 100644
--- a/manage-server/src/main/java/manage/control/PdPController.java
+++ b/manage-server/src/main/java/manage/control/PdPController.java
@@ -1,17 +1,7 @@
 package manage.control;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import manage.api.APIUser;
-import manage.api.Scope;
-import manage.model.EntityType;
 import manage.model.MetaData;
-import manage.policies.PdpPolicyDefinition;
-import manage.policies.PolicyRepository;
-import manage.policies.PolicySummary;
 import manage.repository.MetaDataRepository;
-import manage.service.MetaDataService;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.http.HttpEntity;
 import org.springframework.http.HttpHeaders;
@@ -19,50 +9,28 @@
 import org.springframework.http.MediaType;
 import org.springframework.http.client.support.BasicAuthenticationInterceptor;
 import org.springframework.security.access.prepost.PreAuthorize;
-import org.springframework.util.CollectionUtils;
-import org.springframework.web.bind.annotation.*;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RestController;
 import org.springframework.web.client.RestTemplate;
-import org.xmlunit.builder.DiffBuilder;
-import org.xmlunit.builder.Input;
 
-import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import static java.util.Collections.singletonList;
-import static java.util.stream.Collectors.toList;
-import static manage.model.EntityType.*;
 
 @RestController
 @SuppressWarnings("unchecked")
 public class PdPController {
 
-    private final PolicyRepository policyRepository;
-    private final String policyUrl;
     private final String decideUrl;
     private final RestTemplate pdpRestTemplate;
-    private final ObjectMapper objectMapper;
-    private final MetaDataService metaDataService;
     private final MetaDataRepository metaDataRepository;
     private final HttpHeaders headers;
-    private final Pattern descriptionPattern = Pattern.compile("<Description>(.+?)</Description>", Pattern.DOTALL);
 
-    public PdPController(PolicyRepository policyRepository,
-                         @Value("${push.pdp.policy_url}") String policyUrl,
-                         @Value("${push.pdp.decide_url}") String decideUrl,
+    public PdPController(@Value("${push.pdp.decide_url}") String decideUrl,
                          @Value("${push.pdp.user}") String pdpUser,
                          @Value("${push.pdp.password}") String pdpPassword,
-                         ObjectMapper objectMapper,
-                         MetaDataService metaDataService,
                          MetaDataRepository metaDataRepository) {
-        this.policyRepository = policyRepository;
-        this.policyUrl = policyUrl;
         this.decideUrl = decideUrl;
-        this.objectMapper = objectMapper;
-        this.metaDataService = metaDataService;
         this.metaDataRepository = metaDataRepository;
         this.pdpRestTemplate = new RestTemplate();
         this.pdpRestTemplate.getInterceptors().add(new BasicAuthenticationInterceptor(pdpUser, pdpPassword));
@@ -71,148 +39,16 @@ public PdPController(PolicyRepository policyRepository,
     }
 
     @PreAuthorize("hasRole('ADMIN')")
-    @GetMapping("/client/pdp/push_analysis")
-    public Map<String, Object> pushAnalysis() {
-        List<PolicySummary> policies = policyRepository.policies();
-        List<PolicySummary> migratedPolicies = policyRepository.migratedPolicies();
-        List<PolicySummary> missingPolicies = policies.stream()
-                .filter(policy -> policy.isActive() && migratedPolicies.stream()
-                        .noneMatch(migratedPolicy -> this.nameEquality(policy, migratedPolicy)))
-                .collect(toList());
-        this.addDescription(missingPolicies);
-        List<Map<String, String>> differences = migratedPolicies.stream()
-                .map(migratedPolicy -> policies.stream()
-                        .filter(policy -> this.nameEquality(policy, migratedPolicy))
-                        .findFirst()
-                        .map(policy -> Map.of(
-                                policy.getName(),
-                                DiffBuilder
-                                        .compare(Input.fromString(policy.getXml()))
-                                        .withTest(Input.fromString(migratedPolicy.getXml()))
-                                        .ignoreWhitespace()
-                                        .normalizeWhitespace()
-                                        .ignoreElementContentWhitespace()
-                                        .build()
-                                        .toString())))
-                .filter(optionalMap -> optionalMap.isPresent())
-                .map(optionalMap -> optionalMap.get())
-                .filter(map -> !map.containsValue("[identical]"))
-                .collect(toList());
-        return Map.of(
-                "policy_count", policies.size(),
-                "active_policy_count", policies.stream().filter(policy -> policy.isActive()).count(),
-                "migrated_policy_count", migratedPolicies.size(),
-                "missing_policies", missingPolicies,
-                "differences", differences
-        );
-    }
-
-    @PreAuthorize("hasRole('ADMIN')")
-    @GetMapping("/client/pdp/policies")
-    public List<PolicySummary> policies() {
-        List<PolicySummary> policies = policyRepository.policies();
-        addDescription(policies);
-        return policies;
-    }
-
-    @PreAuthorize("hasRole('ADMIN')")
-    @GetMapping("/client/pdp/migrated_policies")
-    public List<PolicySummary> migratedPolicies() {
-        List<PolicySummary> policies = policyRepository.migratedPolicies();
-        addDescription(policies);
-        return policies;
-    }
-
-    @PreAuthorize("hasRole('ADMIN')")
-    @PostMapping(value =  "/client/pdp/decide")
+    @PostMapping(value = "/client/pdp/decide")
     public String decideManage(@RequestBody String payload) {
         HttpEntity<?> requestEntity = new HttpEntity<>(payload, headers);
         return pdpRestTemplate.exchange(this.decideUrl, HttpMethod.POST, requestEntity, String.class).getBody();
     }
 
     @PreAuthorize("hasRole('ADMIN')")
-    @GetMapping(value =  "/client/pdp/missing-enforcements")
+    @GetMapping(value = "/client/pdp/missing-enforcements")
     public List<MetaData> policiesWithMissingPolicyEnforcementDecisionRequired() {
         return metaDataRepository.policiesWithMissingPolicyEnforcementDecisionRequired();
     }
 
-    @PreAuthorize("hasRole('ADMIN')")
-    @PutMapping("/client/pdp/import_policies")
-    public Map<String, List<Object>> importPolicies() throws JsonProcessingException {
-        HttpEntity<?> requestEntity = new HttpEntity<>(headers);
-        List<PdpPolicyDefinition> policyDefinitions = pdpRestTemplate.exchange(this.policyUrl, HttpMethod.GET, requestEntity, List.class).getBody();
-        String json = objectMapper.writeValueAsString(policyDefinitions);
-        List<Map<String, Object>> dataList = objectMapper.readValue(json, new TypeReference<>() {
-        });
-        this.metaDataService.deleteCollection(EntityType.PDP);
-        Map<String, List<Object>> results = Map.of("imported", new ArrayList<>(), "errors", new ArrayList<>());
-        dataList.forEach(data -> {
-            PdpPolicyDefinition.updateProviderStructure(data);
-            MetaData metaData = new MetaData(EntityType.PDP.getType(), data);
-            Map<String, List<String>> missingReferences = this.missingReferences(metaData);
-            if (missingReferences.values().stream().anyMatch(references -> !references.isEmpty())) {
-                results.get("errors").add(Map.of(
-                        "name",
-                        metaData.getData().get("name"),
-                        "error",
-                        "Unknown providers: " +
-                                missingReferences.entrySet().stream().filter(e -> !e.getValue().isEmpty()).collect(toList())));
-            } else {
-                try {
-                    MetaData savedMetaData = this.metaDataService.doPost(metaData, new APIUser("PDP import", List.of(Scope.SYSTEM)), false);
-                    results.get("imported").add(savedMetaData);
-                } catch (RuntimeException e) {
-                    results.get("errors").add(Map.of(
-                            "name",
-                            metaData.getData().get("name"),
-                            "error",
-                            e.getMessage()));
-                }
-            }
-        });
-        return results;
-    }
-
-    private void addDescription(List<PolicySummary> policies) {
-        policies.forEach(policy -> {
-            Matcher matcher = descriptionPattern.matcher((String) policy.getXml());
-            matcher.find();
-            policy.setDescription(matcher.group(1));
-        });
-    }
-
-    private Map<String, List<String>> missingReferences(MetaData newMetaData) {
-        String serviceProviderIds = "serviceProviderIds";
-        String identityProviderIds = "identityProviderIds";
-        Map<String, List<EntityType>> relationsToCheck = Map.of(
-                serviceProviderIds, Arrays.asList(SP, RP),
-                identityProviderIds, singletonList(IDP)
-        );
-        Map<String, List<String>> missingReferences = Map.of(
-                serviceProviderIds, new ArrayList<>(),
-                identityProviderIds, new ArrayList<>()
-        );
-        relationsToCheck.forEach((key, value) -> {
-            if (newMetaData.getData().containsKey(key)) {
-                List<Map<String, String>> references = (List<Map<String, String>>) newMetaData.getData().get(key);
-                if (!CollectionUtils.isEmpty(references)) {
-                    List<String> missingProviders = references.stream()
-                            .filter(map -> value.stream()
-                                    .allMatch(entityType ->
-                                            CollectionUtils.isEmpty(
-                                                    metaDataRepository.findRaw(entityType.getType(),
-                                                            String.format("{\"data.entityid\" : \"%s\"}", map.get("name")))))
-                            ).map(m -> m.get("name"))
-                            .collect(toList());
-                    missingReferences.get(key).addAll(missingProviders);
-                }
-            }
-        });
-        return missingReferences;
-    }
-
-    private boolean nameEquality(PolicySummary policy, PolicySummary otherPolicy) {
-        return policy.getName().trim().equals(otherPolicy.getName().trim());
-    }
-
 }
diff --git a/manage-server/src/main/java/manage/policies/PolicyRepository.java b/manage-server/src/main/java/manage/policies/PolicyRepository.java
deleted file mode 100644
index 724d8f16..00000000
--- a/manage-server/src/main/java/manage/policies/PolicyRepository.java
+++ /dev/null
@@ -1,33 +0,0 @@
-package manage.policies;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.jdbc.core.JdbcTemplate;
-import org.springframework.jdbc.core.RowMapper;
-import org.springframework.stereotype.Repository;
-
-import java.util.List;
-import java.util.Map;
-
-@Repository
-public class PolicyRepository {
-
-    private final JdbcTemplate jdbcTemplate;
-    private final RowMapper<PolicySummary> rowMapper = (rs, rowNum) -> new PolicySummary(
-            rs.getString(1),
-            rs.getString(2),
-            rs.getBoolean(3)
-    );
-
-    @Autowired
-    public PolicyRepository(JdbcTemplate jdbcTemplate) {
-        this.jdbcTemplate = jdbcTemplate;
-    }
-
-    public List<PolicySummary> policies() {
-        return jdbcTemplate.query("SELECT name, policy_xml, is_active FROM pdp_policies WHERE latest_revision = 1", this.rowMapper);
-    }
-
-    public List<PolicySummary> migratedPolicies() {
-        return jdbcTemplate.query("SELECT name, policy_xml, is_active FROM pdp_migrated_policies", this.rowMapper);
-    }
-}
diff --git a/manage-server/src/main/resources/application.yml b/manage-server/src/main/resources/application.yml
index ddb7f3f7..e66a62bd 100644
--- a/manage-server/src/main/resources/application.yml
+++ b/manage-server/src/main/resources/application.yml
@@ -84,16 +84,6 @@ spring:
   data:
     mongodb:
       uri: mongodb://localhost:27017/metadata_test
-  datasource:
-    url: jdbc:mysql://localhost:3306/pdpserver?permitMysqlScheme
-    username: root
-    password:
-    driverClassName: org.mariadb.jdbc.Driver
-  jpa:
-    database-platform: org.hibernate.dialect.MariaDB53Dialect
-  sql:
-    init:
-      continue-on-error: true
   main:
     banner-mode: "off"
 
diff --git a/manage-server/src/test/java/manage/control/MetaDataControllerTest.java b/manage-server/src/test/java/manage/control/MetaDataControllerTest.java
index d5691a7d..d5302a9e 100644
--- a/manage-server/src/test/java/manage/control/MetaDataControllerTest.java
+++ b/manage-server/src/test/java/manage/control/MetaDataControllerTest.java
@@ -1966,7 +1966,7 @@ public void saveWithNewValidationNameIDFormat() {
         Map<String, Object> data = readValueFromFile("/metadata_templates/saml20_idp.template.json");
 
         data.put("entityid", "https://unique_entity_id");
-        Map.class.cast(data.get("metaDataFields")).put("NameIDFormat", "urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress");
+        Map.class.cast(data.get("metaDataFields")).put("NameIDFormat", "urn:oasis:names:tc:SAML:2.0:nameid-format:emailAddress");
 
         MetaData metaData = new MetaData(EntityType.IDP.getType(), data);
         Map<String, Object> results = given()
@@ -2006,4 +2006,25 @@ public void saveWithInvalidationNameIDFormat() {
                 results.get("validations"));
     }
 
+    @Test
+    public void removeExtraneousKeys() {
+        MetaData metaData = metaDataRepository.findById("1", "saml20_sp");
+        metaData.metaDataFields().put("extraneous_key", "bogus");
+        metaData.metaDataFields().put("1_is_not_2", "bogus");
+        metaDataRepository.update(metaData);
+
+        List<String> entityIdentifiers = given()
+                .auth()
+                .preemptive()
+                .basic("sysadmin", "secret")
+                .when()
+                .body(List.of("extraneous_key", "1_is_not_2"))
+                .header("Content-type", "application/json")
+                .pathParam("type", "saml20_sp")
+                .put("manage/api/internal/removeExtraneousKeys/{type}")
+                .as(new TypeRef<>() {
+                });
+        assertEquals(1, entityIdentifiers.size());
+        assertEquals(metaData.getData().get("entityid"), entityIdentifiers.getFirst());
+    }
 }
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index a45d7f81..0ac51e6b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -5,7 +5,7 @@
 
     <groupId>org.openconext</groupId>
     <artifactId>manage</artifactId>
-    <version>8.0.4-SNAPSHOT</version>
+    <version>9.0.0-SNAPSHOT</version>
     <packaging>pom</packaging>
 
     <name>manage</name>