CPR-593 Add population endpoint for person match (#819)

* CPR-593 Add initial population endpoint for person match

* CPR-593 Add feign client for person match

* CPR-593 Format

* CPR-593 Get Tests working

* CPR-593 Test request is in correct format

* CPR-593 Add runbook

* CPR-593 Run format

* CPR-593 Not required for message consumption to be paused

* CPR-593 Add log statement

* CPR-593 Add elapsed time to log

* CPR-593 Run format

* CPR-593 Add extra test + review comments

* CPR-593 Remove wildcard import

* CPR-593 Fix tests

* CPR-593 Use helper methods

* CPR-593 Update match id

* CPR-593 Fix tests

* CPR-593 Fix lint

helm_deploy/hmpps-person-record/values.yaml

@@ -38,6 +38,10 @@ generic-service:
           deny all;
           return 401;
         }
+        location /populatepersonmatch {
+          deny all;
+          return 401;
+        }
 
   # Environment variables to load into the deployment
   env:

helm_deploy/values-dev.yaml

@@ -15,6 +15,7 @@ generic-service:
 
   env:
     SPRING_PROFILES_ACTIVE: "dev"
+    PERSON_MATCH_BASE_URL: https://hmpps-person-match-dev.hmpps.service.justice.gov.uk
     MATCH_SCORE_BASE_URL: https://hmpps-person-match-score-dev.hmpps.service.justice.gov.uk
     NOMIS_OAUTH_BASE_URL: https://sign-in-dev.hmpps.service.justice.gov.uk
     PRISONER_SEARCH_BASE_URL: https://prisoner-search-dev.prison.service.justice.gov.uk

helm_deploy/values-preprod.yaml

@@ -8,6 +8,7 @@ generic-service:
 
   env:
     SPRING_PROFILES_ACTIVE: "preprod"
+    PERSON_MATCH_BASE_URL: https://hmpps-person-match-preprod.hmpps.service.justice.gov.uk
     MATCH_SCORE_BASE_URL: https://hmpps-person-match-score-preprod.hmpps.service.justice.gov.uk
     NOMIS_OAUTH_BASE_URL: https://sign-in-preprod.hmpps.service.justice.gov.uk
     PRISONER_SEARCH_BASE_URL: https://prisoner-search-preprod.prison.service.justice.gov.uk

helm_deploy/values-prod.yaml

@@ -8,6 +8,7 @@ generic-service:
 
   env:
     SPRING_PROFILES_ACTIVE: "prod"
+    PERSON_MATCH_BASE_URL: https://hmpps-person-match.hmpps.service.justice.gov.uk
     MATCH_SCORE_BASE_URL: https://hmpps-person-match-score.hmpps.service.justice.gov.uk
     NOMIS_OAUTH_BASE_URL: https://sign-in.hmpps.service.justice.gov.uk
     PRISONER_SEARCH_BASE_URL: https://prisoner-search.prison.service.justice.gov.uk

runbooks/004-Seeding-Person-Match.md

@@ -0,0 +1,41 @@
+# 001 - Seeding Person Match
+
+This is the runbook to send person match all the person data that it needs to use to match with.
+
+## Prerequisites
+
+* Must have kubernetes access to the desired namespace
+* Must have `kubectl`, which can be installed [here](https://kubernetes.io/docs/tasks/tools/#kubectl)
+
+## Namespace
+
+For hmpps-person-record the namespaces are listed below:
+* `hmpps-person-record-dev`
+* `hmpps-person-record-preprod`
+* `hmpps-person-record-prod`
+
+## 1. Start Seeding Person Match
+
+To kick off the process you must connect to the hmpps-person-record pod first, by:
+
+```shell
+kubectl exec -it deployment/hmpps-person-record -n <namespace> -- bash
+```
+
+Then within the pod run, to kick off the desired process:
+
+    > WARNING:
+    > You must not deploy to the environment that scheduled for seeding once the job has started. Otherwise, it will be cancelled.
+
+To trigger process:
+```shell
+curl -i -X POST http://localhost:8080/populatepersonmatch
+```
+Once the process has completed it will output the number of pages and records to processed.
+It will notify once finished with: `Finished populating person-match, total pages: <totalPages>, total elements: <totalElements>, time elapsed: <time_elapsed>"`
+
+## Troubleshooting
+
+### Seeding Processing Fails
+
+If the processing of messages fails to create the new records. Either from a mapping issue or api issue. Prepare a fix then follow from step 2 to proceed with seeding the data.

src/main/kotlin/uk/gov/justice/digital/hmpps/personrecord/client/PersonMatchClient.kt

@@ -0,0 +1,16 @@
+package uk.gov.justice.digital.hmpps.personrecord.client
+
+import org.springframework.cloud.openfeign.FeignClient
+import org.springframework.web.bind.annotation.PostMapping
+import org.springframework.web.bind.annotation.RequestBody
+import uk.gov.justice.digital.hmpps.personrecord.client.model.match.PersonMatchRequest
+
+@FeignClient(
+  name = "person-match",
+  url = "\${person-match.base-url}",
+)
+interface PersonMatchClient {
+
+  @PostMapping("/person/migrate")
+  fun postPersonMigrate(@RequestBody personMatchRequest: PersonMatchRequest)
+}

src/main/kotlin/uk/gov/justice/digital/hmpps/personrecord/client/model/match/PersonMatchRecord.kt

@@ -0,0 +1,39 @@
+package uk.gov.justice.digital.hmpps.personrecord.client.model.match
+
+import uk.gov.justice.digital.hmpps.personrecord.jpa.entity.PersonEntity
+import uk.gov.justice.digital.hmpps.personrecord.jpa.entity.PersonEntity.Companion.getType
+import uk.gov.justice.digital.hmpps.personrecord.model.types.IdentifierType
+
+data class PersonMatchRecord(
+  val matchId: String,
+  val sourceSystem: String? = "",
+  val firstName: String? = "",
+  val middleNames: String? = "",
+  val lastName: String? = "",
+  val dateOfBirth: String? = "",
+  val firstNameAliases: List<String> = listOf(),
+  val lastNameAliases: List<String> = listOf(),
+  val dateOfBirthAliases: List<String> = listOf(),
+  val postcodes: List<String> = listOf(),
+  val cros: List<String> = listOf(),
+  val pncs: List<String> = listOf(),
+  val sentenceDates: List<String> = listOf(),
+) {
+  companion object {
+    fun from(personEntity: PersonEntity): PersonMatchRecord = PersonMatchRecord(
+      matchId = personEntity.matchId.toString(),
+      sourceSystem = personEntity.sourceSystem.name,
+      firstName = personEntity.firstName ?: "",
+      middleNames = personEntity.middleNames ?: "",
+      lastName = personEntity.lastName ?: "",
+      dateOfBirth = personEntity.dateOfBirth?.toString() ?: "",
+      firstNameAliases = personEntity.pseudonyms.mapNotNull { it.firstName },
+      lastNameAliases = personEntity.pseudonyms.mapNotNull { it.lastName },
+      dateOfBirthAliases = personEntity.pseudonyms.mapNotNull { it.dateOfBirth }.map { it.toString() },
+      postcodes = personEntity.addresses.mapNotNull { it.postcode },
+      cros = personEntity.references.getType(IdentifierType.CRO).mapNotNull { it.identifierValue },
+      pncs = personEntity.references.getType(IdentifierType.PNC).mapNotNull { it.identifierValue },
+      sentenceDates = personEntity.sentenceInfo.mapNotNull { it.sentenceDate }.map { it.toString() },
+    )
+  }
+}

src/main/kotlin/uk/gov/justice/digital/hmpps/personrecord/client/model/match/PersonMatchRequest.kt

@@ -0,0 +1,5 @@
+package uk.gov.justice.digital.hmpps.personrecord.client.model.match
+
+data class PersonMatchRequest(
+  val records: List<PersonMatchRecord> = listOf(),
+)

src/main/kotlin/uk/gov/justice/digital/hmpps/personrecord/config/SecurityConfiguration.kt

@@ -39,6 +39,7 @@ class SecurityConfiguration {
         "/populatefromprison",
         "/populatefromprobation",
         "/updatefromprobation",
+        "/populatepersonmatch",
       )
     }
   }

src/main/kotlin/uk/gov/justice/digital/hmpps/personrecord/seeding/PopulatePersonMatch.kt

@@ -0,0 +1,85 @@
+package uk.gov.justice.digital.hmpps.personrecord.seeding
+
+import jakarta.transaction.Transactional
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.launch
+import org.slf4j.LoggerFactory
+import org.springframework.data.domain.Page
+import org.springframework.data.domain.PageRequest
+import org.springframework.web.bind.annotation.RequestMapping
+import org.springframework.web.bind.annotation.RequestMethod
+import org.springframework.web.bind.annotation.RestController
+import uk.gov.justice.digital.hmpps.personrecord.client.PersonMatchClient
+import uk.gov.justice.digital.hmpps.personrecord.client.model.match.PersonMatchRecord
+import uk.gov.justice.digital.hmpps.personrecord.client.model.match.PersonMatchRequest
+import uk.gov.justice.digital.hmpps.personrecord.jpa.entity.PersonEntity
+import uk.gov.justice.digital.hmpps.personrecord.jpa.repository.PersonRepository
+import uk.gov.justice.digital.hmpps.personrecord.service.RetryExecutor
+import kotlin.time.Duration
+import kotlin.time.measureTime
+
+private const val OK = "OK"
+
+@RestController
+class PopulatePersonMatch(
+  private val personRepository: PersonRepository,
+  private val personMatchClient: PersonMatchClient,
+  private val retryExecutor: RetryExecutor,
+) {
+
+  @RequestMapping(method = [RequestMethod.POST], value = ["/populatepersonmatch"])
+  suspend fun populate(): String {
+    runPopulation()
+    return OK
+  }
+
+  @Transactional
+  suspend fun runPopulation() {
+    CoroutineScope(Dispatchers.Default).launch {
+      log.info("Starting population of person-match")
+      val executionResults = forPage { page ->
+        log.info("Populating person match, page: ${page.pageable.pageNumber + 1}")
+        val personMatchRecords = page.content.map { PersonMatchRecord.from(it) }
+        val personMatchRequest = PersonMatchRequest(records = personMatchRecords)
+        retryExecutor.runWithRetryHTTP { personMatchClient.postPersonMigrate(personMatchRequest) }
+      }
+      log.info(
+        "Finished populating person-match, total pages: ${executionResults.totalPages}, " +
+          "total elements: ${executionResults.totalElements}, " +
+          "elapsed time: ${executionResults.elapsedTime}",
+      )
+    }
+  }
+
+  private inline fun forPage(page: (Page<PersonEntity>) -> Unit): ExecutionResult {
+    var pageNumber = 0
+    var personRecords: Page<PersonEntity>
+    val elapsedTime: Duration = measureTime {
+      do {
+        val pageable = PageRequest.of(pageNumber, BATCH_SIZE)
+
+        personRecords = personRepository.findAll(pageable)
+        page(personRecords)
+
+        pageNumber++
+      } while (personRecords.hasNext())
+    }
+    return ExecutionResult(
+      totalPages = personRecords.totalPages,
+      totalElements = personRecords.totalElements,
+      elapsedTime = elapsedTime,
+    )
+  }
+
+  private data class ExecutionResult(
+    val totalPages: Int,
+    val totalElements: Long,
+    val elapsedTime: Duration,
+  )
+
+  companion object {
+    private const val BATCH_SIZE = 1000
+    private val log = LoggerFactory.getLogger(this::class.java)
+  }
+}

src/main/resources/application-local.yml

@@ -1,3 +1,6 @@
+person-match:
+  base-url: https://hmpps-person-match-dev.hmpps.service.justice.gov.uk
+
 match-score:
   base-url: https://hmpps-person-match-score-dev.hmpps.service.justice.gov.uk
 

src/test/kotlin/uk/gov/justice/digital/hmpps/personrecord/client/model/match/PersonMatchRecordTest.kt

@@ -0,0 +1,49 @@
+package uk.gov.justice.digital.hmpps.personrecord.client.model.match
+
+import org.assertj.core.api.Assertions.assertThat
+import org.junit.jupiter.api.Test
+import uk.gov.justice.digital.hmpps.personrecord.jpa.entity.PersonEntity
+import uk.gov.justice.digital.hmpps.personrecord.jpa.entity.PseudonymEntity
+import uk.gov.justice.digital.hmpps.personrecord.jpa.entity.SentenceInfoEntity
+import uk.gov.justice.digital.hmpps.personrecord.model.types.NameType
+import uk.gov.justice.digital.hmpps.personrecord.model.types.SourceSystemType.DELIUS
+import java.time.LocalDate
+import java.util.UUID
+
+class PersonMatchRecordTest {
+
+  @Test
+  fun `should return empty strings when building request if is null`() {
+    val personEntity = PersonEntity(
+      matchId = UUID.randomUUID(),
+      sourceSystem = DELIUS,
+    )
+    val personMatchRecord = PersonMatchRecord.from(personEntity)
+    assertThat(personMatchRecord.firstName).isEmpty()
+    assertThat(personMatchRecord.middleNames).isEmpty()
+    assertThat(personMatchRecord.lastName).isEmpty()
+    assertThat(personMatchRecord.dateOfBirth).isEmpty()
+    assertThat(personMatchRecord.firstNameAliases).isEmpty()
+    assertThat(personMatchRecord.lastNameAliases).isEmpty()
+    assertThat(personMatchRecord.dateOfBirthAliases).isEmpty()
+    assertThat(personMatchRecord.postcodes).isEmpty()
+    assertThat(personMatchRecord.cros).isEmpty()
+    assertThat(personMatchRecord.pncs).isEmpty()
+    assertThat(personMatchRecord.sentenceDates).isEmpty()
+  }
+
+  @Test
+  fun `should build dates in correct YYYY-MM-dd format`() {
+    val date = LocalDate.of(1970, 1, 1)
+    val personEntity = PersonEntity(
+      dateOfBirth = date,
+      pseudonyms = mutableListOf(PseudonymEntity(type = NameType.ALIAS, dateOfBirth = date)),
+      sentenceInfo = mutableListOf(SentenceInfoEntity(sentenceDate = date)),
+      sourceSystem = DELIUS,
+    )
+    val personMatchRecord = PersonMatchRecord.from(personEntity)
+    assertThat(personMatchRecord.dateOfBirth).isEqualTo("1970-01-01")
+    assertThat(personMatchRecord.dateOfBirthAliases).isEqualTo(listOf("1970-01-01"))
+    assertThat(personMatchRecord.sentenceDates).isEqualTo(listOf("1970-01-01"))
+  }
+}

src/test/kotlin/uk/gov/justice/digital/hmpps/personrecord/config/IntegrationTestBase.kt

@@ -10,6 +10,7 @@ import org.awaitility.kotlin.atMost
 import org.awaitility.kotlin.await
 import org.awaitility.kotlin.untilAsserted
 import org.awaitility.kotlin.untilNotNull
+import org.jmock.lib.concurrent.Blitzer
 import org.json.JSONObject
 import org.junit.jupiter.api.extension.RegisterExtension
 import org.springframework.beans.factory.annotation.Autowired
@@ -73,7 +74,7 @@ class IntegrationTestBase {
     }
   }
 
-  internal fun awaitAssert(function: () -> Unit) = await atMost (Duration.ofSeconds(3)) untilAsserted function
+  internal fun awaitAssert(timeout: Long = 3, function: () -> Unit) = await atMost (Duration.ofSeconds(timeout)) untilAsserted function
 
   internal fun awaitNotNullPerson(function: () -> PersonEntity?): PersonEntity = await atMost (Duration.ofSeconds(3)) untilNotNull function
 
@@ -157,6 +158,17 @@ class IntegrationTestBase {
     )
   }
 
+  fun blitz(actionCount: Int, threadCount: Int, action: () -> Unit) {
+    val blitzer = Blitzer(actionCount, threadCount)
+    try {
+      blitzer.blitz {
+        action()
+      }
+    } finally {
+      blitzer.shutdown()
+    }
+  }
+
   companion object {
 
     internal const val BASE_SCENARIO = "baseScenario"

src/test/kotlin/uk/gov/justice/digital/hmpps/personrecord/config/MessagingMultiNodeTestBase.kt

@@ -5,7 +5,6 @@ import com.github.tomakehurst.wiremock.stubbing.Scenario.STARTED
 import org.awaitility.kotlin.await
 import org.awaitility.kotlin.matches
 import org.awaitility.kotlin.untilCallTo
-import org.jmock.lib.concurrent.Blitzer
 import org.junit.jupiter.api.BeforeEach
 import org.junit.jupiter.api.extension.ExtendWith
 import org.springframework.beans.factory.annotation.Autowired
@@ -285,17 +284,6 @@ abstract class MessagingMultiNodeTestBase : IntegrationTestBase() {
     )
   }
 
-  fun blitz(actionCount: Int, threadCount: Int, action: () -> Unit) {
-    val blitzer = Blitzer(actionCount, threadCount)
-    try {
-      blitzer.blitz {
-        action()
-      }
-    } finally {
-      blitzer.shutdown()
-    }
-  }
-
   @BeforeEach
   fun beforeEachMessagingTest() {
     purgeQueueAndDlq(courtEventsQueue)