diff --git a/.github/workflows/api-tests.yml b/.github/workflows/api-tests.yml index 13e581d87..3746eed93 100644 --- a/.github/workflows/api-tests.yml +++ b/.github/workflows/api-tests.yml @@ -15,10 +15,10 @@ jobs: steps: - name: Checkout out branch uses: actions/checkout@v2 - - name: Set up JDK 11 + - name: Set up JDK 17 uses: actions/setup-java@v2 with: - java-version: '11' + java-version: '17' distribution: 'adopt' - name: Maven version run: mvn -version diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index b596fd51c..45605844f 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -15,10 +15,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up JDK 11 + - name: Set up JDK 17 uses: actions/setup-java@v2 with: - java-version: '11' + java-version: '17' distribution: 'adopt' - name: Maven version run: mvn -version diff --git a/api/pom.xml b/api/pom.xml index ec22da6a0..839b27d1c 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -4,7 +4,7 @@ edu.wgu.osmt osmt-parent - 2.6.0-SNAPSHOT + 2.6.1-SNAPSHOT @@ -27,11 +27,11 @@ edu.wgu.osmt osmt-api - 2.6.0-SNAPSHOT + 2.6.1-SNAPSHOT WGU Open Skills Management Toolset - 11 + 2.6.1-SNAPSHOT 1.7.21 official 1.5.1 @@ -42,8 +42,8 @@ 4.1.3 edu.wgu.osmt.ApplicationKt - 1.17.6 - 4.4.1 + 1.18.3 + 5.1.3 3.9.2 2.17.1 4.10.0 @@ -54,7 +54,7 @@ edu.wgu.osmt osmt-ui - 2.6.0-SNAPSHOT + ${app.version} org.springframework.boot @@ -77,7 +77,7 @@ com.github.sonus21 rqueue-spring-boot-starter - 2.13.0-RELEASE + 3.1.0-RELEASE com.fasterxml.jackson.core @@ -121,6 +121,15 @@ + + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + 7.17.8 + + + org.apache.logging.log4j log4j-api diff --git a/api/src/main/kotlin/edu/wgu/osmt/PropertyLogger.kt b/api/src/main/kotlin/edu/wgu/osmt/PropertyLogger.kt index 5f6590229..5b0299c72 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/PropertyLogger.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/PropertyLogger.kt @@ -15,7 +15,7 @@ import java.util.stream.StreamSupport @Component -@Profile("dev") +@Profile("debug") class PropertyLogger { @EventListener fun handleContextRefresh(event: ContextRefreshedEvent) { @@ -43,4 +43,4 @@ class PropertyLogger { companion object { private val LOGGER = LoggerFactory.getLogger(PropertyLogger::class.java) } -} \ No newline at end of file +} diff --git a/api/src/main/kotlin/edu/wgu/osmt/RoutePaths.kt b/api/src/main/kotlin/edu/wgu/osmt/RoutePaths.kt index 4bc7c9f75..f3c72a6f2 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/RoutePaths.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/RoutePaths.kt @@ -57,6 +57,7 @@ object RoutePaths { const val COLLECTION_REMOVE = "$COLLECTION_DETAIL/remove" const val WORKSPACE_PATH = "/workspace" + const val WORKSPACE_LIST = WORKSPACE_PATH private const val TASKS_PATH = "/results" const val TASK_DETAIL_TEXT = "$TASKS_PATH/text/{uuid}" diff --git a/api/src/main/kotlin/edu/wgu/osmt/api/ApiErrorHandler.kt b/api/src/main/kotlin/edu/wgu/osmt/api/ApiErrorHandler.kt index 39d04bc2b..43a809a89 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/api/ApiErrorHandler.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/api/ApiErrorHandler.kt @@ -33,7 +33,9 @@ class GeneralApiExceptionHandler : ResponseEntityExceptionHandler() { @ControllerAdvice class ApiErrorHandler : ResponseEntityExceptionHandler() { - override fun handleHttpMessageNotReadable( +// No longer abstract method in spring-webmvc:6.0.11 +// override fun handleHttpMessageNotReadable( + fun handleHttpMessageNotReadable( ex: HttpMessageNotReadableException, headers: HttpHeaders, status: HttpStatus, @@ -57,8 +59,8 @@ class ApiErrorHandler : ResponseEntityExceptionHandler() { @ExceptionHandler(ResponseStatusException::class) fun handleResponseStatus(ex: ResponseStatusException): ResponseEntity { - val apiError = ApiError(ex.status.toString()) - return ResponseEntity(apiError, ex.status) + val apiError = ApiError(ex.message) + return ResponseEntity(apiError, ex.statusCode) } } diff --git a/api/src/main/kotlin/edu/wgu/osmt/auditlog/AuditLogDao.kt b/api/src/main/kotlin/edu/wgu/osmt/auditlog/AuditLogDao.kt index 56d323b49..de4d87432 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/auditlog/AuditLogDao.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/auditlog/AuditLogDao.kt @@ -1,7 +1,7 @@ package edu.wgu.osmt.auditlog -import com.google.common.reflect.TypeToken import com.google.gson.Gson +import com.google.gson.reflect.TypeToken import edu.wgu.osmt.db.OutputsModel import org.jetbrains.exposed.dao.LongEntity import org.jetbrains.exposed.dao.LongEntityClass diff --git a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionController.kt b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionController.kt index 42d24f414..02e5d28d2 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionController.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionController.kt @@ -3,14 +3,7 @@ package edu.wgu.osmt.collection import edu.wgu.osmt.HasAllPaginated import edu.wgu.osmt.RoutePaths import edu.wgu.osmt.api.GeneralApiException -import edu.wgu.osmt.api.model.ApiCollection -import edu.wgu.osmt.api.model.ApiCollectionUpdate -import edu.wgu.osmt.api.model.ApiCollectionV2 -import edu.wgu.osmt.api.model.ApiSearch -import edu.wgu.osmt.api.model.ApiSearchV2 -import edu.wgu.osmt.api.model.ApiSkillListUpdate -import edu.wgu.osmt.api.model.ApiStringListUpdate -import edu.wgu.osmt.api.model.CollectionSortEnum +import edu.wgu.osmt.api.model.* import edu.wgu.osmt.auditlog.AuditLog import edu.wgu.osmt.auditlog.AuditLogRepository import edu.wgu.osmt.auditlog.AuditLogSortEnum @@ -20,34 +13,18 @@ import edu.wgu.osmt.db.PublishStatus import edu.wgu.osmt.elasticsearch.OffsetPageable import edu.wgu.osmt.richskill.RichSkillRepository import edu.wgu.osmt.security.OAuthHelper -import edu.wgu.osmt.task.AppliesToType -import edu.wgu.osmt.task.CsvTask -import edu.wgu.osmt.task.CsvTaskV2 -import edu.wgu.osmt.task.PublishTask -import edu.wgu.osmt.task.PublishTaskV2 -import edu.wgu.osmt.task.RemoveCollectionSkillsTask -import edu.wgu.osmt.task.Task -import edu.wgu.osmt.task.TaskMessageService -import edu.wgu.osmt.task.TaskResult -import edu.wgu.osmt.task.UpdateCollectionSkillsTask -import edu.wgu.osmt.task.XlsxTask +import edu.wgu.osmt.task.* import org.springframework.beans.factory.annotation.Autowired import org.springframework.http.HttpEntity import org.springframework.http.HttpStatus +import org.springframework.http.HttpStatus.NOT_FOUND import org.springframework.http.MediaType import org.springframework.http.ResponseEntity import org.springframework.security.core.annotation.AuthenticationPrincipal import org.springframework.security.oauth2.jwt.Jwt import org.springframework.stereotype.Controller import org.springframework.transaction.annotation.Transactional -import org.springframework.web.bind.annotation.DeleteMapping -import org.springframework.web.bind.annotation.GetMapping -import org.springframework.web.bind.annotation.PathVariable -import org.springframework.web.bind.annotation.PostMapping -import org.springframework.web.bind.annotation.RequestBody -import org.springframework.web.bind.annotation.RequestMapping -import org.springframework.web.bind.annotation.RequestParam -import org.springframework.web.bind.annotation.ResponseBody +import org.springframework.web.bind.annotation.* import org.springframework.web.server.ResponseStatusException import org.springframework.web.util.UriComponentsBuilder @@ -346,7 +323,7 @@ class CollectionController @Autowired constructor( @PathVariable uuid: String ): HttpEntity> { val pageable = OffsetPageable(0, Int.MAX_VALUE, AuditLogSortEnum.forValueOrDefault(AuditLogSortEnum.DateDesc.apiValue).sort) - val collection = collectionRepository.findByUUID(uuid) + val collection = collectionRepository.findByUUID(uuid) ?: throw ResponseStatusException(NOT_FOUND, "Collection with id $uuid not ready or not found") val sizedIterable = auditLogRepository.findByTableAndId(CollectionTable.tableName, entityId = collection!!.id.value, offsetPageable = pageable) return ResponseEntity.status(200).body(sizedIterable.toList().map { it.toModel() }) diff --git a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionDoc.kt b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionDoc.kt index 8ea53781a..79ce013c6 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionDoc.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionDoc.kt @@ -5,7 +5,7 @@ import com.fasterxml.jackson.annotation.JsonInclude import com.fasterxml.jackson.annotation.JsonProperty import edu.wgu.osmt.config.INDEX_COLLECTION_DOC import edu.wgu.osmt.db.PublishStatus -import org.elasticsearch.core.Nullable +import javax.annotation.Nullable import org.springframework.data.annotation.Id import org.springframework.data.elasticsearch.annotations.* import org.springframework.data.elasticsearch.annotations.FieldType.* diff --git a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt index 355e8f8bd..7b95ec0cb 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt @@ -5,24 +5,32 @@ import edu.wgu.osmt.api.model.ApiSearch import edu.wgu.osmt.config.INDEX_COLLECTION_DOC import edu.wgu.osmt.db.PublishStatus import edu.wgu.osmt.elasticsearch.FindsAllByPublishStatus +import edu.wgu.osmt.elasticsearch.WguQueryHelper.convertToNativeQuery +import edu.wgu.osmt.elasticsearch.WguQueryHelper.createTermsDslQuery import edu.wgu.osmt.richskill.RichSkillDoc import edu.wgu.osmt.richskill.RichSkillEsRepo import org.apache.lucene.search.join.ScoreMode import org.elasticsearch.index.query.* +import org.slf4j.Logger +import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.annotation.Configuration import org.springframework.data.domain.Page import org.springframework.data.domain.PageRequest import org.springframework.data.domain.Pageable import org.springframework.data.domain.Sort -import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate +import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate +import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates -import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.repository.ElasticsearchRepository import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories +/** + * This have been partially converted to use the ElasticSearch 8.7.X apis. Need to do full conversion to use + * the v8.7.x ES Java API client, https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/8.10/searching.html + */ interface CustomCollectionQueries : FindsAllByPublishStatus { val richSkillEsRepo: RichSkillEsRepo @@ -43,11 +51,11 @@ interface CustomCollectionQueries : FindsAllByPublishStatus { } class CustomCollectionQueriesImpl @Autowired constructor( - override val elasticSearchTemplate: ElasticsearchRestTemplate, + override val elasticSearchTemplate: ElasticsearchTemplate, override val richSkillEsRepo: RichSkillEsRepo ) : CustomCollectionQueries { - + val log: Logger = LoggerFactory.getLogger(CustomCollectionQueriesImpl::class.java) override val javaClass = CollectionDoc::class.java override fun collectionPropertiesMultiMatch(query: String): AbstractQueryBuilder<*> { @@ -77,21 +85,25 @@ class CustomCollectionQueriesImpl @Autowired constructor( } } + /** + * TODO upgrade to ElasticSearch v8.7.x api style; see KeywordEsRepo.kt & FindsAllByPublishStatus.kt + */ override fun byApiSearch( apiSearch: ApiSearch, publishStatus: Set, pageable: Pageable ): SearchHits { - val nsq: NativeSearchQueryBuilder = NativeSearchQueryBuilder().withPageable(Pageable.unpaged()) + val nsqb1 = NativeSearchQueryBuilder().withPageable(Pageable.unpaged()) val bq = QueryBuilders.boolQuery() + val filterDslQuery = createTermsDslQuery(RichSkillDoc::publishStatus.name, publishStatus.map { ps -> ps.toString() }) val filter = BoolQueryBuilder().must( - QueryBuilders.termsQuery( - RichSkillDoc::publishStatus.name, - publishStatus.map { ps -> ps.toString() } - ) + QueryBuilders.termsQuery( + RichSkillDoc::publishStatus.name, + publishStatus.map { ps -> ps.toString() } + ) ) - nsq.withFilter(filter) - nsq.withQuery(bq) + nsqb1.withFilter(filter) + nsqb1.withQuery(bq) var collectionMultiPropertyResults: List = listOf() @@ -101,76 +113,57 @@ class CustomCollectionQueriesImpl @Autowired constructor( bq.should( BoolQueryBuilder() .must(richSkillEsRepo.richSkillPropertiesMultiMatch(apiSearch.query)) - .must( - QueryBuilders.nestedQuery( - RichSkillDoc::collections.name, - QueryBuilders.matchAllQuery(), - ScoreMode.Avg - ).innerHit(InnerHitBuilder()) - ) + .must(createNestedQueryBuilder()) ) bq.should(richSkillEsRepo.occupationQueries(apiSearch.query)) + val nsqb = NativeSearchQueryBuilder() + .withQuery( collectionPropertiesMultiMatch(apiSearch.query) ) + .withPageable(Pageable.unpaged()) + .withFilter(filter) // search on collection specific properties - collectionMultiPropertyResults = elasticSearchTemplate.search( - NativeSearchQueryBuilder().withQuery( - collectionPropertiesMultiMatch(apiSearch.query) - ).withPageable(Pageable.unpaged()).withFilter(filter).build(), CollectionDoc::class.java - ).searchHits.map { it.content.uuid } + val query = convertToNativeQuery(Pageable.unpaged(), filterDslQuery, nsqb, "CustomCollectionQueriesImpl.byApiSearch()1", log) + collectionMultiPropertyResults = elasticSearchTemplate + .search(query, CollectionDoc::class.java) + .searchHits + .map { it.content.uuid } } else if (apiSearch.advanced != null) { richSkillEsRepo.generateBoolQueriesFromApiSearch(bq, apiSearch.advanced) if (!apiSearch.advanced.collectionName.isNullOrBlank()) { - if (apiSearch.advanced.collectionName.contains("\"")) { - collectionMultiPropertyResults = elasticSearchTemplate.search( - NativeSearchQueryBuilder().withQuery( - QueryBuilders.simpleQueryStringQuery(apiSearch.advanced.collectionName).field("${CollectionDoc::name.name}.raw").defaultOperator(Operator.AND) - ).withPageable(Pageable.unpaged()).withFilter(filter).build(), CollectionDoc::class.java - ).searchHits.map { it.content.uuid } - } else { - collectionMultiPropertyResults = elasticSearchTemplate.search( - NativeSearchQueryBuilder().withQuery( - QueryBuilders.matchPhrasePrefixQuery( - CollectionDoc::name.name, - apiSearch.advanced.collectionName - ) - ).withPageable(Pageable.unpaged()).withFilter(filter).build(), CollectionDoc::class.java - ).searchHits.map { it.content.uuid } - } + collectionMultiPropertyResults = getCollectionUuids(pageable, filterDslQuery, apiSearch.advanced.collectionName ) } else { - bq.must( - QueryBuilders.nestedQuery( - RichSkillDoc::collections.name, - QueryBuilders.matchAllQuery(), - ScoreMode.Avg - ).innerHit(InnerHitBuilder()) - ) + bq.must(createNestedQueryBuilder()) } } else { // query nor advanced search was provided, return all collections - bq.must( - QueryBuilders.nestedQuery( - RichSkillDoc::collections.name, - QueryBuilders.matchAllQuery(), - ScoreMode.Avg - ).innerHit(InnerHitBuilder()) - ) + bq.must(createNestedQueryBuilder()) } - val results = elasticSearchTemplate.search(nsq.build(), RichSkillDoc::class.java) + var query = convertToNativeQuery(Pageable.unpaged(), filterDslQuery, nsqb1, "CustomCollectionQueriesImpl.byApiSearch().innerHitCollectionUuids", log) + val innerHitCollectionUuids = elasticSearchTemplate + .search(query, RichSkillDoc::class.java) + .searchHits.mapNotNull { it.getInnerHits("collections")?.searchHits?.mapNotNull { it.content as CollectionDoc } } + .flatten() + .map { it.uuid } + .distinct() + return getCollectionFromUuids(pageable, filterDslQuery, (innerHitCollectionUuids + collectionMultiPropertyResults).distinct(), "CustomCollectionQueriesImpl.byApiSearch()2", log) + } - val innerHitCollectionUuids = - results.searchHits.mapNotNull { it.getInnerHits("collections")?.searchHits?.mapNotNull { it.content as CollectionDoc } } - .flatten().map { it.uuid }.distinct() + @Deprecated("Upgrade to ES v8.x queries", ReplaceWith("createNestQueryDslQuery"), DeprecationLevel.WARNING ) + private fun createNestedQueryBuilder(): NestedQueryBuilder { + return QueryBuilders.nestedQuery( + RichSkillDoc::collections.name, + QueryBuilders.matchAllQuery(), + ScoreMode.Avg + ).innerHit(InnerHitBuilder()) + } - return elasticSearchTemplate.search( - NativeSearchQueryBuilder().withQuery( - QueryBuilders.termsQuery( - "_id", - (innerHitCollectionUuids + collectionMultiPropertyResults).distinct() - ) - ).withFilter(filter).withPageable(pageable).build(), CollectionDoc::class.java - ) + private fun getCollectionUuids(pageable: Pageable, filter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, collectionName: String) : List { + return if (collectionName.contains("\"")) + getCollectionUuidsFromComplexName(pageable, filter, collectionName, "getCollectionUuids", log) + else + getCollectionUuidsFromName(pageable, filter, collectionName, "getCollectionUuids", log) } } diff --git a/api/src/main/kotlin/edu/wgu/osmt/collection/UpdateCollectionSkillsTaskProcessor.kt b/api/src/main/kotlin/edu/wgu/osmt/collection/UpdateCollectionSkillsTaskProcessor.kt index 6a0a84d43..9292eb026 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/collection/UpdateCollectionSkillsTaskProcessor.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/collection/UpdateCollectionSkillsTaskProcessor.kt @@ -11,7 +11,7 @@ import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.annotation.Profile import org.springframework.stereotype.Component -import javax.transaction.Transactional +import org.springframework.transaction.annotation.Transactional @Component @Profile("apiserver") @@ -63,4 +63,4 @@ class UpdateCollectionSkillsTaskProcessor { logger.info("Task ${task.uuid} completed") } -} \ No newline at end of file +} diff --git a/api/src/main/kotlin/edu/wgu/osmt/db/DbConfig.kt b/api/src/main/kotlin/edu/wgu/osmt/db/DbConfig.kt index 83a5fe250..d58ded2a0 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/db/DbConfig.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/db/DbConfig.kt @@ -1,9 +1,7 @@ package edu.wgu.osmt.db import org.springframework.boot.context.properties.ConfigurationProperties -import org.springframework.boot.context.properties.ConstructorBinding -@ConstructorBinding @ConfigurationProperties(prefix = "db", ignoreInvalidFields = true) data class DbConfig( val name: String, diff --git a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/ElasticsearchClientManager.kt b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/ElasticsearchClientManager.kt index 1a3ac08b5..b47dba227 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/ElasticsearchClientManager.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/ElasticsearchClientManager.kt @@ -1,17 +1,24 @@ package edu.wgu.osmt.elasticsearch +import co.elastic.clients.elasticsearch.ElasticsearchClient +import co.elastic.clients.json.jackson.JacksonJsonpMapper +import co.elastic.clients.transport.rest_client.RestClientTransport import org.apache.http.HttpHost +import org.apache.http.auth.AuthScope +import org.apache.http.auth.UsernamePasswordCredentials +import org.apache.http.client.CredentialsProvider +import org.apache.http.impl.client.BasicCredentialsProvider import org.elasticsearch.client.RestClient -import org.elasticsearch.client.RestHighLevelClient import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import org.springframework.core.convert.converter.Converter import org.springframework.data.convert.ReadingConverter import org.springframework.data.convert.WritingConverter -import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate +import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate import org.springframework.data.elasticsearch.core.convert.ElasticsearchCustomConversions import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories +import org.springframework.util.StringUtils import java.time.LocalDateTime import java.time.format.DateTimeFormatter import java.util.* @@ -25,13 +32,17 @@ class ElasticsearchClientManager { @Override @Bean - fun elasticSearchClient(): RestHighLevelClient { - return RestHighLevelClient(RestClient.builder(HttpHost.create(esConfig.uri))) + fun elasticSearchClient(): ElasticsearchClient { + val transport = RestClientTransport( + createRestClient(), + JacksonJsonpMapper() + ) + return ElasticsearchClient(transport) } @Bean - fun elasticsearchTemplate(): ElasticsearchRestTemplate { - return ElasticsearchRestTemplate(elasticSearchClient()) + fun elasticsearchTemplate(): ElasticsearchTemplate { + return ElasticsearchTemplate(elasticSearchClient()) } @Bean @@ -71,4 +82,36 @@ class ElasticsearchClientManager { return UUID.fromString(source) } } + + private fun createRestClient(): RestClient { + val restClientBuilder = RestClient.builder(createHttpHost()) + val credentialsProvider = getCredentialsProvider() + + credentialsProvider?.let { + restClientBuilder.setHttpClientConfigCallback { b -> b.setDefaultCredentialsProvider(it) } + } + return restClientBuilder.build() + } + + private fun createHttpHost(): HttpHost { + val scheme = StringUtils.split(esConfig.uri, "://") + if(scheme.isNullOrEmpty()){ + val params = StringUtils.split(esConfig.uri, ":") + return HttpHost(params!![0], params[1].toInt()) + } else { + val params = StringUtils.split(scheme!![1], ":") + return HttpHost(params!![0], params[1].toInt(), scheme!![0]) + } + } + + private fun getCredentialsProvider(): CredentialsProvider? { + if (esConfig.username.isNullOrBlank() || esConfig.password.isNullOrBlank()) { + return null + } + + val credentialsProvider = BasicCredentialsProvider() + val credential = UsernamePasswordCredentials(esConfig.username, esConfig.password) + credentialsProvider.setCredentials(AuthScope.ANY, credential) + return credentialsProvider + } } diff --git a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/EsConfig.kt b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/EsConfig.kt index 877028a22..c019b5e94 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/EsConfig.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/EsConfig.kt @@ -1,8 +1,10 @@ package edu.wgu.osmt.elasticsearch import org.springframework.boot.context.properties.ConfigurationProperties -import org.springframework.boot.context.properties.ConstructorBinding -@ConstructorBinding @ConfigurationProperties(prefix = "es", ignoreInvalidFields = true) -data class EsConfig(val uri: String) +data class EsConfig( + val uri: String, + val username: String?, + val password: String? +) diff --git a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt index 2bd6d2891..71911ad3d 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt @@ -1,43 +1,69 @@ package edu.wgu.osmt.elasticsearch +import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.matchAll +import edu.wgu.osmt.collection.CollectionDoc import edu.wgu.osmt.db.PublishStatus -import org.elasticsearch.index.query.BoolQueryBuilder -import org.elasticsearch.index.query.QueryBuilders +import edu.wgu.osmt.elasticsearch.WguQueryHelper.createMatchPhrasePrefixDslQuery +import edu.wgu.osmt.elasticsearch.WguQueryHelper.createNativeQuery +import edu.wgu.osmt.elasticsearch.WguQueryHelper.createSimpleQueryDslQuery +import edu.wgu.osmt.elasticsearch.WguQueryHelper.createTermsDslQuery +import edu.wgu.osmt.richskill.RichSkillDoc +import org.slf4j.Logger import org.springframework.data.domain.Pageable -import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate +import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate +import org.springframework.data.elasticsearch.client.elc.NativeQuery import org.springframework.data.elasticsearch.core.SearchHits -import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder - +import java.util.stream.Collectors interface FindsAllByPublishStatus { - val elasticSearchTemplate: ElasticsearchRestTemplate + val elasticSearchTemplate: ElasticsearchTemplate val javaClass: Class fun findAllFilteredByPublishStatus(publishStatus: Set, pageable: Pageable): SearchHits { - val nsq: NativeSearchQueryBuilder = buildQuery(pageable, publishStatus) - return elasticSearchTemplate.search(nsq.build(), javaClass) + val nativeQuery = createMatchAllRichSkillNativeQuery(pageable, publishStatus) + return elasticSearchTemplate.search(nativeQuery, javaClass) } fun countAllFilteredByPublishStatus(publishStatus: Set, pageable: Pageable): Long { - val nsq: NativeSearchQueryBuilder = buildQuery(pageable, publishStatus) - return elasticSearchTemplate.count(nsq.build(), javaClass) + val nativeQuery = createMatchAllRichSkillNativeQuery(pageable, publishStatus) + return elasticSearchTemplate.count(nativeQuery, javaClass) } - fun buildQuery( - pageable: Pageable, - publishStatus: Set - ): NativeSearchQueryBuilder { - val nsq: NativeSearchQueryBuilder = NativeSearchQueryBuilder().withPageable(pageable) - nsq.withQuery(QueryBuilders.matchAllQuery()) - nsq.withFilter( - BoolQueryBuilder().should( - QueryBuilders.termsQuery( - "publishStatus", - publishStatus.map { ps -> ps.toString() } - ) - ) - ) - return nsq + fun createMatchAllRichSkillNativeQuery(pageable: Pageable, publishStatus: Set): NativeQuery { + val MATCH_ALL = matchAll().build()._toQuery() + var filterValues = publishStatus + .stream() + .map { ps -> ps.name} + .collect(Collectors.toList()) + var filter = createTermsDslQuery( RichSkillDoc::publishStatus.name, filterValues, false) + + return createNativeQuery(pageable, filter, MATCH_ALL) + } + + fun getCollectionUuidsFromComplexName(pageable: Pageable, dslFilter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, collectionName: String, msgPrefix: String, log: Logger) : List { + var dslQuery = createSimpleQueryDslQuery("${CollectionDoc::name.name}.raw", collectionName) + var nativeQuery = createNativeQuery(pageable, dslFilter, dslQuery, msgPrefix, log) + + return elasticSearchTemplate + .search( nativeQuery, CollectionDoc::class.java ) + .searchHits + .map { it.content.uuid } } -} + fun getCollectionUuidsFromName(pageable: Pageable, dslFilter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, collectionName: String, msgPrefix: String, log: Logger) : List { + var dslQuery = createMatchPhrasePrefixDslQuery(CollectionDoc::name.name, collectionName) + var nativeQuery = createNativeQuery(pageable, dslFilter, dslQuery, msgPrefix, log) + + return elasticSearchTemplate + .search( nativeQuery, CollectionDoc::class.java ) + .searchHits + .map { it.content.uuid } + } + + fun getCollectionFromUuids(pageable: Pageable, dslFilter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, uuids: List, msgPrefix: String, log: Logger ): SearchHits { + var dslQuery = createTermsDslQuery("_id", uuids) + var nativeQuery = createNativeQuery(pageable, dslFilter, dslQuery, msgPrefix, log) + + return elasticSearchTemplate.search(nativeQuery, CollectionDoc::class.java) + } +} diff --git a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/WguQueryHelper.kt b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/WguQueryHelper.kt new file mode 100644 index 000000000..3f2c16c0d --- /dev/null +++ b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/WguQueryHelper.kt @@ -0,0 +1,118 @@ +package edu.wgu.osmt.elasticsearch + +import co.elastic.clients.elasticsearch._types.FieldValue +import co.elastic.clients.elasticsearch._types.SortOptions +import co.elastic.clients.elasticsearch._types.SortOrder +import co.elastic.clients.elasticsearch._types.query_dsl.ChildScoreMode +import co.elastic.clients.elasticsearch._types.query_dsl.Operator +import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders +import co.elastic.clients.elasticsearch._types.query_dsl.TermsQueryField +import co.elastic.clients.elasticsearch.core.search.InnerHits +import org.slf4j.Logger +import org.springframework.data.domain.Pageable +import org.springframework.data.elasticsearch.client.elc.NativeQuery +import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder +import org.springframework.data.elasticsearch.core.query.Query +import org.springframework.data.elasticsearch.core.query.StringQuery +import java.util.stream.Collectors + + +/** + * Utility class for leveraging latest ElasticSearch v8.7.X Java API + */ +object WguQueryHelper { + @Deprecated("Upgrade to ES v8.x queries", ReplaceWith("createNativeQuery"), DeprecationLevel.WARNING ) + fun convertToNativeQuery(pageable: Pageable, dslFilter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, nsqb: NativeSearchQueryBuilder, msgPrefix: String, log: Logger): Query { + val springDataQuery = StringQuery(nsqb.build().query.toString()) + return createNativeQuery(pageable, dslFilter, springDataQuery, msgPrefix, log) + } + + @Deprecated("Upgrade to ES v8.x queries", ReplaceWith("createNativeQuery"), DeprecationLevel.WARNING ) + fun createNativeQuery(pageable: Pageable, dslFilter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, springDataQuery: Query, msgPrefix: String? = null, log: Logger? = null): NativeQuery { + val query = NativeQuery + .builder() + .withFilter(dslFilter) + .withQuery(springDataQuery) + .withPageable(pageable) + .build() + log(query, msgPrefix, log) + return query; + } + + fun createNativeQuery(pageable: Pageable, dslFilter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, dslQuery: co.elastic.clients.elasticsearch._types.query_dsl.Query, msgPrefix: String? = null, log: Logger? = null): NativeQuery { + val query = NativeQuery + .builder() + .withFilter(dslFilter) + .withQuery(dslQuery) + .withPageable(pageable) +// .withSort(createSort("blah")) + .build() + log(query, msgPrefix, log) + return query; + } + + private fun log(nativeQuery: NativeQuery, msgPrefix: String?, log: Logger?) { + if (nativeQuery.springDataQuery == null || msgPrefix == null || log == null) return + log.debug(String.Companion.format("\n%s springDataQuery:\n\t\t%s", msgPrefix, (nativeQuery.springDataQuery as StringQuery).source)) + log.debug(String.Companion.format("\n%s dslFilter:\n\t\t%s", msgPrefix, nativeQuery.filter.toString())) + } + + fun createMatchPhrasePrefixDslQuery(fieldName: String, searchStr: String, boostVal : Float? = null): co.elastic.clients.elasticsearch._types.query_dsl.Query { + return QueryBuilders.matchPhrasePrefix { qb -> qb.field(fieldName).query(searchStr).boost(boostVal) } + } + + fun createMatchBoolPrefixDslQuery(fieldName: String, searchStr: String, boostVal : Float? = null): co.elastic.clients.elasticsearch._types.query_dsl.Query { + return QueryBuilders.matchBoolPrefix { qb -> qb.field(fieldName).query(searchStr).boost(boostVal) } + } + + fun createSimpleQueryDslQuery(fieldName: String, searchStr: String, boostVal : Float? = null): co.elastic.clients.elasticsearch._types.query_dsl.Query { + return QueryBuilders.simpleQueryString { qb -> + qb.fields(fieldName).query(searchStr).boost(boostVal).defaultOperator(Operator.And) + } + } + + fun createNestedQueryDslQuery(path: String, scoreMode: ChildScoreMode, query: co.elastic.clients.elasticsearch._types.query_dsl.Query? = null, innerHits: InnerHits? = null): co.elastic.clients.elasticsearch._types.query_dsl.Query { + query ?: QueryBuilders.matchAll { b -> b } + innerHits ?: InnerHits.Builder().build() + return QueryBuilders.nested { qb -> + qb.path(path) + .scoreMode(ChildScoreMode.Avg) + .innerHits(innerHits) + .query(QueryBuilders.matchAll { b -> b }) + } + } + + fun createTermsDslQuery(fieldName: String, filterValues: List, andFlag: Boolean = true): co.elastic.clients.elasticsearch._types.query_dsl.Query { + val values = filterValues + .stream() + .map { FieldValue.of(it) } + .collect(Collectors.toList()) + val tqf = TermsQueryField.Builder() + .value(values) + .build() + val terms = QueryBuilders.terms() + .field(fieldName) + .terms(tqf) + .build() + ._toQuery() + /* Short hand version https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/8.10/searching.html + val terms2 = terms { qb -> qb.field(fieldName).terms(tqf) } + return bool { qb -> if (andFlag) + qb.must(terms2) + else + qb.should(terms2) } + */ + + return QueryBuilders.bool() + .let { + if (andFlag) it.must(terms) + else it.should(terms) } + .build() + ._toQuery() + } + + // TODO handle case sensitivity + fun createSort(fieldName: String, sortOrder: SortOrder = SortOrder.Asc) : SortOptions { + return SortOptions.Builder().field{f -> f.field(fieldName).order(sortOrder)}.build() + } +} diff --git a/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCode.kt b/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCode.kt index f00bcb7ca..f6a73bd1c 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCode.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCode.kt @@ -3,7 +3,7 @@ package edu.wgu.osmt.jobcode import com.fasterxml.jackson.annotation.JsonIgnore import edu.wgu.osmt.config.INDEX_JOBCODE_DOC import edu.wgu.osmt.db.DatabaseData -import org.elasticsearch.core.Nullable +import javax.annotation.Nullable import org.springframework.data.elasticsearch.annotations.* import java.time.LocalDateTime import java.time.ZoneOffset diff --git a/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt index 6bc1208bc..2c21ed012 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt @@ -2,22 +2,30 @@ package edu.wgu.osmt.jobcode import edu.wgu.osmt.config.INDEX_JOBCODE_DOC import edu.wgu.osmt.elasticsearch.OffsetPageable +import edu.wgu.osmt.elasticsearch.WguQueryHelper.convertToNativeQuery import org.elasticsearch.index.query.BoolQueryBuilder import org.elasticsearch.index.query.Operator import org.elasticsearch.index.query.QueryBuilders.* import org.elasticsearch.search.sort.SortBuilders import org.elasticsearch.search.sort.SortOrder +import org.slf4j.Logger +import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.annotation.Configuration -import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate +import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate +import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates -import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.repository.ElasticsearchRepository import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories + +/** + * This have been partially converted to use the ElasticSearch 8.7.X apis. Need to do full conversion to use + * the v8.x ES Java API client, https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/8.10/searching.html + */ interface CustomJobCodeRepository { - val elasticSearchTemplate: ElasticsearchRestTemplate + val elasticSearchTemplate: ElasticsearchTemplate fun typeAheadSearch(query: String): SearchHits fun deleteIndex() { @@ -25,27 +33,37 @@ interface CustomJobCodeRepository { } } -class CustomJobCodeRepositoryImpl @Autowired constructor(override val elasticSearchTemplate: ElasticsearchRestTemplate) : +class CustomJobCodeRepositoryImpl @Autowired constructor(override val elasticSearchTemplate: ElasticsearchTemplate) : CustomJobCodeRepository { + val log: Logger = LoggerFactory.getLogger(CustomJobCodeRepositoryImpl::class.java) + /** + * TODO upgrade to ElasticSearch v8.7.x api style; see KeywordEsRepo.kt & FindsAllByPublishStatus.kt + */ + @Deprecated("Upgrade to ES v8.x queries", ReplaceWith(""), DeprecationLevel.WARNING ) override fun typeAheadSearch(query: String): SearchHits { - val nsq: NativeSearchQueryBuilder + val disjunctionQuery = JobCodeQueries.multiPropertySearch(query) + val nqb = NativeSearchQueryBuilder() + .withPageable(createOffsetPageable(query)) + .withQuery(disjunctionQuery) + .withSort(SortBuilders.fieldSort("${JobCode::code.name}.keyword").order(SortOrder.ASC)) + val query = convertToNativeQuery(createOffsetPageable(query), null, nqb, "CustomJobCodeRepositoryImpl.typeAheadSearch()", log) + return elasticSearchTemplate.search(query, JobCode::class.java) + } - val limitedPageable: OffsetPageable = if (query.isEmpty()) { + private fun createOffsetPageable(query: String): OffsetPageable { + val limitedPageable = if (query.isEmpty()) { OffsetPageable(0, 10000, null) } else { OffsetPageable(0, 20, null) - } - val disjunctionQuery = JobCodeQueries.multiPropertySearch(query) - nsq = - NativeSearchQueryBuilder().withPageable(limitedPageable).withQuery(disjunctionQuery) - .withSort(SortBuilders.fieldSort("${JobCode::code.name}.keyword").order(SortOrder.ASC)) - return elasticSearchTemplate.search(nsq.build(), JobCode::class.java) + return limitedPageable } } +@Deprecated("Upgrade to ES v8.x queries", ReplaceWith("JobCodeQueriesEx"), DeprecationLevel.WARNING ) object JobCodeQueries { + //TODO Convert to ES v8.7.x apis and return the newer BoolQuery.Builder instance; see KeywordEsRep.kt fun multiPropertySearch(query: String, parentDocPath: String? = null): BoolQueryBuilder { val disjunctionQuery = disMaxQuery() val path = parentDocPath?.let { "${it}." } ?: "" diff --git a/api/src/main/kotlin/edu/wgu/osmt/keyword/Keyword.kt b/api/src/main/kotlin/edu/wgu/osmt/keyword/Keyword.kt index 7878080c4..52646054d 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/keyword/Keyword.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/keyword/Keyword.kt @@ -6,7 +6,7 @@ import edu.wgu.osmt.db.HasUpdateDate import edu.wgu.osmt.db.NullableFieldUpdate import edu.wgu.osmt.db.TableWithUpdate import edu.wgu.osmt.db.UpdateObject -import org.elasticsearch.core.Nullable +import javax.annotation.Nullable import org.jetbrains.exposed.dao.id.LongIdTable import org.jetbrains.exposed.sql.Column import org.jetbrains.exposed.sql.`java-time`.datetime diff --git a/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt index 984864a3a..ca9670dae 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt @@ -1,22 +1,28 @@ package edu.wgu.osmt.keyword +import co.elastic.clients.elasticsearch._types.query_dsl.* import edu.wgu.osmt.config.INDEX_KEYWORD_DOC import edu.wgu.osmt.config.SORT_INSENSITIVE import edu.wgu.osmt.elasticsearch.OffsetPageable +import edu.wgu.osmt.elasticsearch.WguQueryHelper +import edu.wgu.osmt.elasticsearch.WguQueryHelper.convertToNativeQuery +import edu.wgu.osmt.jobcode.CustomJobCodeRepositoryImpl import org.elasticsearch.index.query.QueryBuilders import org.elasticsearch.search.sort.SortBuilders import org.elasticsearch.search.sort.SortOrder +import org.slf4j.Logger +import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.annotation.Configuration -import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate +import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate +import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates -import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.repository.ElasticsearchRepository import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories interface CustomKeywordRepository { - val elasticSearchTemplate: ElasticsearchRestTemplate + val elasticSearchTemplate: ElasticsearchTemplate fun typeAheadSearch(query: String, type: KeywordTypeEnum): SearchHits fun deleteIndex() { @@ -24,44 +30,88 @@ interface CustomKeywordRepository { } } -class CustomKeywordRepositoryImpl @Autowired constructor(override val elasticSearchTemplate: ElasticsearchRestTemplate) : +/** + * This have been partially converted to use the ElasticSearch 8.7.X apis. For full conversion + * replace typeAheadSearch() with TypeAheadSearchNu() + */ +class CustomKeywordRepositoryImpl @Autowired constructor(override val elasticSearchTemplate: ElasticsearchTemplate) : CustomKeywordRepository { - override fun typeAheadSearch(query: String, type: KeywordTypeEnum): SearchHits { + val log: Logger = LoggerFactory.getLogger(CustomJobCodeRepositoryImpl::class.java) + + @Deprecated("Upgrade to ES v8.x queries", ReplaceWith("typeAheadSearchNu"), DeprecationLevel.WARNING ) + override fun typeAheadSearch(searchStr: String, type: KeywordTypeEnum): SearchHits { val limitedPageable: OffsetPageable val bq = QueryBuilders.boolQuery() - val nsq: NativeSearchQueryBuilder + val nqb: NativeSearchQueryBuilder - if(query.isEmpty()){ //retrieve all + if(searchStr.isEmpty()){ //retrieve all limitedPageable = OffsetPageable(0, 10000, null) - nsq = NativeSearchQueryBuilder().withPageable(limitedPageable).withQuery(bq) + nqb = NativeSearchQueryBuilder() + .withPageable(limitedPageable) + .withQuery(bq) .withSort(SortBuilders.fieldSort("${Keyword::value.name}$SORT_INSENSITIVE").order(SortOrder.ASC)) bq .must(QueryBuilders.termQuery(Keyword::type.name, type.name)) - .should( - QueryBuilders.matchAllQuery() - ) + .should( QueryBuilders.matchAllQuery() ) } else { limitedPageable = OffsetPageable(0, 20, null) - nsq = NativeSearchQueryBuilder().withPageable(limitedPageable).withQuery(bq) + nqb = NativeSearchQueryBuilder() + .withPageable(limitedPageable) + .withQuery(bq) .withSort(SortBuilders.fieldSort("${Keyword::value.name}$SORT_INSENSITIVE").order(SortOrder.ASC)) bq .must(QueryBuilders.termQuery(Keyword::type.name, type.name)) .should( - QueryBuilders.matchBoolPrefixQuery( - Keyword::value.name, - query - ) + QueryBuilders.matchBoolPrefixQuery( Keyword::value.name, searchStr ) ) .should( - QueryBuilders.matchPhraseQuery( - Keyword::value.name, - query - ).boost(5f) + QueryBuilders.matchPhraseQuery( Keyword::value.name, searchStr ).boost(5f) ).minimumShouldMatch(1) } - return elasticSearchTemplate.search(nsq.build(), Keyword::class.java) + val query = convertToNativeQuery( limitedPageable, null, nqb, "CustomKeywordRepositoryImpl.typeAheadSearch()", log ) + return elasticSearchTemplate.search(query, Keyword::class.java) + } + + /** + * Uses the latest ES 8.7.x Java Client API + */ + fun typeAheadSearchNu(searchStr: String, type: KeywordTypeEnum): SearchHits { + val pageable: OffsetPageable + val criteria: Query + + if (searchStr.isEmpty()) { + pageable = OffsetPageable(0, 10000, null) + criteria = searchAll(type) + } else { + pageable = OffsetPageable(0, 20, null) + criteria = searchSpecific(searchStr, type) + } +// log.debug(String.Companion.format("\ntypeAheadSearchNu query:\n\t\t%s", criteria.bool().toString())) +// return elasticSearchTemplate.search( NativeQuery.builder() +// .withPageable(pageable) +// .withQuery(criteria) +// .build(), Keyword::class.java ) + + var nativeQuery = WguQueryHelper.createNativeQuery(pageable, null, criteria) + return elasticSearchTemplate.search(nativeQuery, Keyword::class.java) + } + + fun searchAll(type: KeywordTypeEnum): Query { + return co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.bool { builder: BoolQuery.Builder -> + builder + .must(co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.term { qt: TermQuery.Builder -> qt.field(Keyword::type.name).value(type.name) } ) + .should(co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.matchAll { q : MatchAllQuery.Builder -> q } ) } + } + + fun searchSpecific(searchStr: String, type: KeywordTypeEnum): Query { + return co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.bool { builder: BoolQuery.Builder -> + builder + .must(co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.term { qt: TermQuery.Builder -> qt.field(Keyword::type.name).value(type.name) } ) + .should(co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.matchBoolPrefix { q : MatchBoolPrefixQuery.Builder -> q.field(Keyword::value.name).query(searchStr)} ) + .should(co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.matchPhrase { q : MatchPhraseQuery.Builder -> q.field(Keyword::value.name).query(searchStr)} ) + .minimumShouldMatch("1") } } } diff --git a/api/src/main/kotlin/edu/wgu/osmt/richskill/CreateSkillsTaskProcessor.kt b/api/src/main/kotlin/edu/wgu/osmt/richskill/CreateSkillsTaskProcessor.kt index d232bf286..7f1d852b0 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/richskill/CreateSkillsTaskProcessor.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/richskill/CreateSkillsTaskProcessor.kt @@ -10,7 +10,7 @@ import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.annotation.Profile import org.springframework.stereotype.Component -import javax.transaction.Transactional +import org.springframework.transaction.annotation.Transactional @Component @@ -47,4 +47,4 @@ class CreateSkillsTaskProcessor { logger.info("Task ${task.uuid} completed") } -} \ No newline at end of file +} diff --git a/api/src/main/kotlin/edu/wgu/osmt/richskill/PublishTaskProcessor.kt b/api/src/main/kotlin/edu/wgu/osmt/richskill/PublishTaskProcessor.kt index 1a1292f36..5d08f50d1 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/richskill/PublishTaskProcessor.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/richskill/PublishTaskProcessor.kt @@ -11,7 +11,7 @@ import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.annotation.Profile import org.springframework.stereotype.Component -import javax.transaction.Transactional +import org.springframework.transaction.annotation.Transactional @Component @@ -49,4 +49,4 @@ class PublishTaskProcessor { logger.info("Task ${publishTask.uuid} completed") } -} \ No newline at end of file +} diff --git a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillController.kt b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillController.kt index 30e16ebcd..468ab5ad3 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillController.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillController.kt @@ -3,15 +3,7 @@ package edu.wgu.osmt.richskill import edu.wgu.osmt.HasAllPaginated import edu.wgu.osmt.RoutePaths import edu.wgu.osmt.api.GeneralApiException -import edu.wgu.osmt.api.model.ApiSearch -import edu.wgu.osmt.api.model.ApiSearchV2 -import edu.wgu.osmt.api.model.ApiSkill -import edu.wgu.osmt.api.model.ApiSkillUpdate -import edu.wgu.osmt.api.model.ApiSkillUpdateMapper -import edu.wgu.osmt.api.model.ApiSkillUpdateV2 -import edu.wgu.osmt.api.model.ApiSkillV2 -import edu.wgu.osmt.api.model.SkillSortEnum -import edu.wgu.osmt.api.model.SortOrder +import edu.wgu.osmt.api.model.* import edu.wgu.osmt.auditlog.AuditLog import edu.wgu.osmt.auditlog.AuditLogRepository import edu.wgu.osmt.auditlog.AuditLogSortEnum @@ -23,39 +15,17 @@ import edu.wgu.osmt.io.csv.RichSkillCsvExport import edu.wgu.osmt.io.csv.RichSkillCsvExportV2 import edu.wgu.osmt.keyword.KeywordDao import edu.wgu.osmt.security.OAuthHelper -import edu.wgu.osmt.task.AppliesToType -import edu.wgu.osmt.task.CreateSkillsTask -import edu.wgu.osmt.task.CreateSkillsTaskV2 -import edu.wgu.osmt.task.CsvTask -import edu.wgu.osmt.task.CsvTaskV2 -import edu.wgu.osmt.task.ExportSkillsToCsvTask -import edu.wgu.osmt.task.ExportSkillsToCsvTaskV2 -import edu.wgu.osmt.task.ExportSkillsToXlsxTask -import edu.wgu.osmt.task.PublishTask -import edu.wgu.osmt.task.PublishTaskV2 -import edu.wgu.osmt.task.Task -import edu.wgu.osmt.task.TaskMessageService -import edu.wgu.osmt.task.TaskResult -import edu.wgu.osmt.task.XlsxTask +import edu.wgu.osmt.task.* import org.apache.commons.lang3.StringUtils import org.springframework.beans.factory.annotation.Autowired import org.springframework.data.domain.Pageable -import org.springframework.http.HttpEntity -import org.springframework.http.HttpHeaders -import org.springframework.http.HttpStatus -import org.springframework.http.MediaType -import org.springframework.http.ResponseEntity +import org.springframework.http.* +import org.springframework.http.HttpStatus.NOT_FOUND import org.springframework.security.core.annotation.AuthenticationPrincipal import org.springframework.security.oauth2.jwt.Jwt import org.springframework.stereotype.Controller import org.springframework.transaction.annotation.Transactional -import org.springframework.web.bind.annotation.GetMapping -import org.springframework.web.bind.annotation.PathVariable -import org.springframework.web.bind.annotation.PostMapping -import org.springframework.web.bind.annotation.RequestBody -import org.springframework.web.bind.annotation.RequestMapping -import org.springframework.web.bind.annotation.RequestParam -import org.springframework.web.bind.annotation.ResponseBody +import org.springframework.web.bind.annotation.* import org.springframework.web.server.ResponseStatusException import org.springframework.web.util.UriComponentsBuilder @@ -418,7 +388,7 @@ class RichSkillController @Autowired constructor( @PathVariable uuid: String ): HttpEntity> { val pageable = OffsetPageable(0, Int.MAX_VALUE, AuditLogSortEnum.forValueOrDefault(AuditLogSortEnum.DateDesc.apiValue).sort) - val skill = richSkillRepository.findByUUID(uuid) + val skill = richSkillRepository.findByUUID(uuid) ?: throw ResponseStatusException(NOT_FOUND, "Skill with id $uuid not ready or not found") val sizedIterable = auditLogRepository.findByTableAndId(RichSkillDescriptorTable.tableName, entityId = skill!!.id.value, offsetPageable = pageable) return ResponseEntity.status(200).body(sizedIterable.toList().map { it.toModel() }) diff --git a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillDoc.kt b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillDoc.kt index 11ef82f4b..0010ed8ed 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillDoc.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillDoc.kt @@ -9,7 +9,7 @@ import edu.wgu.osmt.config.INDEX_RICHSKILL_DOC import edu.wgu.osmt.db.PublishStatus import edu.wgu.osmt.jobcode.JobCode import edu.wgu.osmt.keyword.KeywordTypeEnum -import org.elasticsearch.core.Nullable +import javax.annotation.Nullable import org.springframework.data.annotation.Id import org.springframework.data.elasticsearch.annotations.* import org.springframework.data.elasticsearch.annotations.FieldType.* diff --git a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillDocV2.kt b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillDocV2.kt index 880240702..42d2bc4f8 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillDocV2.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillDocV2.kt @@ -10,7 +10,7 @@ import edu.wgu.osmt.config.SEMICOLON import edu.wgu.osmt.db.PublishStatus import edu.wgu.osmt.jobcode.JobCode import edu.wgu.osmt.keyword.KeywordTypeEnum -import org.elasticsearch.core.Nullable +import javax.annotation.Nullable import org.springframework.data.annotation.Id import org.springframework.data.elasticsearch.annotations.* import org.springframework.data.elasticsearch.annotations.FieldType.* diff --git a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt index 198acf07b..7bb657020 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt @@ -1,15 +1,19 @@ package edu.wgu.osmt.richskill +import co.elastic.clients.elasticsearch._types.query_dsl.Query +import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.* import edu.wgu.osmt.PaginationDefaults -import edu.wgu.osmt.api.model.ApiAdvancedSearch -import edu.wgu.osmt.api.model.ApiFilteredSearch -import edu.wgu.osmt.api.model.ApiSearch -import edu.wgu.osmt.api.model.ApiSimilaritySearch +import edu.wgu.osmt.api.model.* import edu.wgu.osmt.config.INDEX_RICHSKILL_DOC import edu.wgu.osmt.config.QUOTED_SEARCH_REGEX_PATTERN import edu.wgu.osmt.db.PublishStatus import edu.wgu.osmt.elasticsearch.FindsAllByPublishStatus import edu.wgu.osmt.elasticsearch.OffsetPageable +import edu.wgu.osmt.elasticsearch.WguQueryHelper.convertToNativeQuery +import edu.wgu.osmt.elasticsearch.WguQueryHelper.createMatchBoolPrefixDslQuery +import edu.wgu.osmt.elasticsearch.WguQueryHelper.createMatchPhrasePrefixDslQuery +import edu.wgu.osmt.elasticsearch.WguQueryHelper.createSimpleQueryDslQuery +import edu.wgu.osmt.elasticsearch.WguQueryHelper.createTermsDslQuery import edu.wgu.osmt.jobcode.JobCodeQueries import edu.wgu.osmt.nullIfEmpty import org.apache.commons.lang3.StringUtils @@ -17,22 +21,30 @@ import org.apache.lucene.search.join.ScoreMode import org.elasticsearch.index.query.* import org.elasticsearch.index.query.QueryBuilders.* import org.elasticsearch.script.Script +import org.slf4j.Logger +import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.annotation.Configuration import org.springframework.data.domain.Page import org.springframework.data.domain.PageRequest import org.springframework.data.domain.Pageable import org.springframework.data.domain.Sort -import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate +import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate +import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates -import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.repository.ElasticsearchRepository import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories import org.springframework.security.oauth2.jwt.Jwt +import java.util.function.Consumer +import java.util.stream.Collectors const val collectionsUuid = "collections.uuid" +/** + * This have been partially converted to use the ElasticSearch 8.7.X apis. Need to do full conversion to use + * the v8.7.x ES Java API client, https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/8.10/searching.html + */ interface CustomRichSkillQueries : FindsAllByPublishStatus { fun getUuidsFromApiSearch( apiSearch: ApiSearch, @@ -67,8 +79,9 @@ interface CustomRichSkillQueries : FindsAllByPublishStatus { } } -class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSearchTemplate: ElasticsearchRestTemplate) : +class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSearchTemplate: ElasticsearchTemplate) : CustomRichSkillQueries { + val log: Logger = LoggerFactory.getLogger(CustomRichSkillQueriesImpl::class.java) override val javaClass = RichSkillDoc::class.java override fun getUuidsFromApiSearch( @@ -95,6 +108,7 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear return uuids } + @Deprecated("ElasticSearch 7.X has been deprecated", ReplaceWith("buildNestedQueriesNu"), DeprecationLevel.WARNING) override fun occupationQueries(query: String): NestedQueryBuilder { val jobCodePath = RichSkillDoc::jobCodes.name return QueryBuilders.nestedQuery( @@ -104,6 +118,21 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear ) } + /** + * ElasticSearch v8.7.X version + */ + fun occupationQueriesNu(query: String): Query? { + /* + val multiPropQuery = null + return co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.nested { + qb -> qb.path(RichSkillDoc::jobCodes.name) + .query(JobCodeQueries.multiPropertySearch(query, jobCodePath),) + .scoreMode( ChildScoreMode.Max)} + */ + return null; + } + + @Deprecated("ElasticSearch 7.X has been deprecated", ReplaceWith("buildNestedQueriesNu"), DeprecationLevel.WARNING) private fun buildNestedQueries(path: String?=null, queryParams: List) : BoolQueryBuilder { val disjunctionQuery = disMaxQuery() val queries = ArrayList() @@ -121,8 +150,23 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear return boolQuery().must(existsQuery("$path.keyword")).must(disjunctionQuery) } + /** + * ElasticSearch v8.7.X version + */ + private fun buildNestedQueriesNu(path: String?=null, queryParams: List) : Query { + val prefixQueries = ArrayList() + queryParams.forEach(Consumer { s: String? -> + val q = prefix { qb -> qb.field( "$path.keyword").value(s) } + prefixQueries.add(q) + }) + + val disMaxQuery = disMax {qb -> qb.queries(prefixQueries)} + val existQuery = exists { qb -> qb.field("$path.keyword")} + return bool { qb -> qb.must(disMaxQuery).must(existQuery)} + } // Query clauses for Rich Skill properties + @Deprecated("ElasticSearch 7.X has been deprecated", ReplaceWith("generateBoolQueriesFromApiSearchNu"), DeprecationLevel.WARNING) override fun generateBoolQueriesFromApiSearch(bq: BoolQueryBuilder, advancedQuery: ApiAdvancedSearch) { with(advancedQuery) { // boolQuery.must for logical AND @@ -192,8 +236,7 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear it.mapNotNull { it.name }.map { s -> if (s.contains("\"")) { bq.must( - simpleQueryStringQuery(s).field("${RichSkillDoc::certifications.name}.raw") - .defaultOperator(Operator.AND) + simpleQueryStringQuery(s).field("${RichSkillDoc::certifications.name}.raw").defaultOperator(Operator.AND) ) } else { bq.must(matchBoolPrefixQuery(RichSkillDoc::certifications.name, s)) @@ -227,7 +270,50 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear } } + /** + * ElasticSearch v8.7.X version + */ + fun generateBoolQueriesFromApiSearchNu(advancedQuery: ApiAdvancedSearch): Query { + with(advancedQuery) { + return bool { bq -> + skillName.nullIfEmpty()?.let { bq.must(createQueryFromString(RichSkillDoc::name.name, it)) } + category.nullIfEmpty()?.let { bq.must(createQueryFromString(RichSkillDoc::categories.name, it, QUOTED_SEARCH_REGEX_PATTERN)) } + author.nullIfEmpty()?.let { bq.must(createQueryFromString(RichSkillDoc::authors.name, it)) } + skillStatement.nullIfEmpty()?.let { bq.must(createQueryFromString(RichSkillDoc::statement.name, it)) } + keywords?.let { bq.must(createQueryFromStringList(RichSkillDoc::searchingKeywords.name, it)) } +//TODO implement this +// occupations.nullIfEmpty()?.let { bq.must(createQueryFromString(RichSkillDoc::name.name, it)) } + + standards?.let { bq.must(createQueryFromApiNameList(RichSkillDoc::standards.name, it)) } + certifications?.let { bq.must(createQueryFromApiNameList(RichSkillDoc::certifications.name, it)) } + employers?.let { bq.must(createQueryFromApiNameList(RichSkillDoc::employers.name, it)) } + alignments?.let { bq.must(createQueryFromApiNameList(RichSkillDoc::alignments.name, it)) } + } + } + } + + private fun createQueryFromString(fieldName: String, searchStr: String, regEx: String? = null): Query { + val isComplex = searchStr.contains("\"") || (regEx != null && searchStr.matches(Regex(regEx))) + return if (isComplex) + createSimpleQueryDslQuery(String.format("%s.raw", fieldName), searchStr) + else + createMatchBoolPrefixDslQuery(fieldName, searchStr) + } + + private fun createQueryFromStringList(fieldName: String, searchStrList: List): List { + return searchStrList + .stream() + .map { createQueryFromString(fieldName, it ?: "") } + .collect(Collectors.toList()) + } + + private fun createQueryFromApiNameList(fieldName: String, searchStrList: List): List { + return createQueryFromStringList(fieldName, searchStrList.map { it.name?: "" }) + } + + @Deprecated("ElasticSearch 7.X has been deprecated", ReplaceWith("generateBoolQueriesFromApiSearchWithFiltersNu"), DeprecationLevel.WARNING) override fun generateBoolQueriesFromApiSearchWithFilters(bq: BoolQueryBuilder, filteredQuery: ApiFilteredSearch, publishStatus: Set) { + bq.must( termsQuery( RichSkillDoc::publishStatus.name, @@ -276,25 +362,76 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear } } + /** + * TODO Fix the NPE at the return. + fun generateBoolQueriesFromApiSearchWithFiltersNu(filteredQuery: ApiFilteredSearch, publishStatus: Set) : Query { + val values = publishStatus + .stream() + .map { FieldValue.of(it.toString()) } + .collect(Collectors.toList()) + val tqf = TermsQueryField.Builder() + .value(values) + .build() + val terms2 = terms { qb -> qb.field(RichSkillDoc::publishStatus.name).terms(tqf) } + val qb = bool().must(terms2) + + with(filteredQuery) { + categories?. let { qb.must(buildNestedQueriesNu(RichSkillDoc::categories.name, it)) } + keywords?. let { + it.mapNotNull { qb.must(generateTermsSetQueryBuilderNu(RichSkillDoc::searchingKeywords.name, keywords)) } + } + standards?. let { + it.mapNotNull { qb.must(generateTermsSetQueryBuilderNu(RichSkillDoc::standards.name, standards)) } + } + certifications?. let { + it.mapNotNull { qb.must(generateTermsSetQueryBuilderNu(RichSkillDoc::certifications.name, certifications)) } + } + alignments?. let { + it.mapNotNull { qb.must(generateTermsSetQueryBuilderNu(RichSkillDoc::alignments.name, alignments)) } + } + employers?. let { + it.mapNotNull { qb.must(generateTermsSetQueryBuilderNu(RichSkillDoc::employers.name, employers)) } + } + authors?. let { + qb.must(buildNestedQueriesNu(RichSkillDoc::authors.name, it)) + } + occupations?.let { + it.mapNotNull { value -> qb.must( occupationQueriesNu(value) ) } + } + } + val s = qb.build()._toQuery() + return s + } + */ + + @Deprecated("ElasticSearch 7.X has been deprecated", ReplaceWith("generateTermsSetQueryBuilderNu"), DeprecationLevel.WARNING) private fun generateTermsSetQueryBuilder(fieldName: String, list: List): TermsSetQueryBuilder { return TermsSetQueryBuilder("$fieldName.keyword", list).setMinimumShouldMatchScript(Script(list.size.toString())) } + /** + * ElasticSearch v8.7.X version + */ + private fun generateTermsSetQueryBuilderNu(fieldName: String, list: List): Query { + val sb = co.elastic.clients.elasticsearch._types.Script.Builder().inline { il -> il.source(list.size.toString())} + return termsSet { + qb -> qb.field("$fieldName.keyword") + .terms(list) + .minimumShouldMatchScript(sb.build()) + } + } + + @Deprecated("ElasticSearch 7.X has been deprecated", ReplaceWith("richSkillPropertiesMultiMatchNu"), DeprecationLevel.WARNING) override fun richSkillPropertiesMultiMatch(query: String): BoolQueryBuilder { val isComplex = query.contains("\"") - val boolQuery = boolQuery() - val complexQueries = listOf( - simpleQueryStringQuery(query).field("${RichSkillDoc::name.name}.raw").boost(2.0f) - .defaultOperator(Operator.AND), + simpleQueryStringQuery(query).field("${RichSkillDoc::name.name}.raw").boost(2.0f) .defaultOperator(Operator.AND), simpleQueryStringQuery(query).field("${RichSkillDoc::statement.name}.raw").defaultOperator(Operator.AND), simpleQueryStringQuery(query).field("${RichSkillDoc::categories.name}.raw").defaultOperator(Operator.AND), - simpleQueryStringQuery(query).field("${RichSkillDoc::searchingKeywords.name}.raw") - .defaultOperator(Operator.AND), + simpleQueryStringQuery(query).field("${RichSkillDoc::searchingKeywords.name}.raw") .defaultOperator(Operator.AND), simpleQueryStringQuery(query).field("${RichSkillDoc::standards.name}.raw").defaultOperator(Operator.AND), - simpleQueryStringQuery(query).field("${RichSkillDoc::certifications.name}.raw") - .defaultOperator(Operator.AND), + simpleQueryStringQuery(query).field("${RichSkillDoc::certifications.name}.raw") .defaultOperator(Operator.AND), simpleQueryStringQuery(query).field("${RichSkillDoc::employers.name}.raw").defaultOperator(Operator.AND), simpleQueryStringQuery(query).field("${RichSkillDoc::alignments.name}.raw").defaultOperator(Operator.AND), simpleQueryStringQuery(query).field("${RichSkillDoc::authors.name}.raw").defaultOperator(Operator.AND) @@ -317,19 +454,62 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear } else { queries.map { boolQuery.should(it) } } - return boolQuery } + /** + * ElasticSearch v8.7.X version + */ + fun richSkillPropertiesMultiMatchNu(searchStr: String): Query { + var queries = if (searchStr.contains("\"")) + createComplexMultiMatchQueries(searchStr) + else + createMultiMatchQueries(searchStr) + return bool {qb -> qb.should(queries) } + } + + private fun createComplexMultiMatchQueries(searchStr: String) : List{ + return listOf( + createSimpleQueryDslQuery("${RichSkillDoc::name.name}.raw", searchStr, 2.0f), + createSimpleQueryDslQuery("${RichSkillDoc::statement.name}.raw", searchStr), + createSimpleQueryDslQuery("${RichSkillDoc::categories.name}.raw", searchStr), + createSimpleQueryDslQuery("${RichSkillDoc::searchingKeywords.name}.raw", searchStr), + createSimpleQueryDslQuery("${RichSkillDoc::standards.name}.raw", searchStr), + createSimpleQueryDslQuery("${RichSkillDoc::certifications.name}.raw", searchStr), + createSimpleQueryDslQuery("${RichSkillDoc::employers.name}.raw", searchStr), + createSimpleQueryDslQuery("${RichSkillDoc::alignments.name}.raw", searchStr), + createSimpleQueryDslQuery("${RichSkillDoc::authors.name}.raw", searchStr) + ) + } + + private fun createMultiMatchQueries(searchStr: String) : List{ + return listOf( + createMatchPhrasePrefixDslQuery(RichSkillDoc::name.name, searchStr, 2.0f), + createMatchPhrasePrefixDslQuery(RichSkillDoc::statement.name, searchStr), + createMatchPhrasePrefixDslQuery(RichSkillDoc::categories.name, searchStr), + createMatchPhrasePrefixDslQuery(RichSkillDoc::searchingKeywords.name, searchStr), + createMatchPhrasePrefixDslQuery(RichSkillDoc::standards.name, searchStr), + createMatchPhrasePrefixDslQuery(RichSkillDoc::certifications.name, searchStr), + createMatchPhrasePrefixDslQuery(RichSkillDoc::employers.name, searchStr), + createMatchPhrasePrefixDslQuery(RichSkillDoc::alignments.name, searchStr), + createMatchPhrasePrefixDslQuery(RichSkillDoc::authors.name, searchStr) + ) + } + override fun byApiSearch( apiSearch: ApiSearch, publishStatus: Set, pageable: Pageable, collectionId: String? ): SearchHits { - val nsq: NativeSearchQueryBuilder = buildQuery(pageable, publishStatus, apiSearch, collectionId) - - return elasticSearchTemplate.search(nsq.build(), RichSkillDoc::class.java) + val query = convertToNativeQuery( + pageable, + createFilter(apiSearch, publishStatus), + buildQuery(publishStatus, apiSearch, collectionId), + "CustomRichSkillQueriesImpl.byApiSearch()", + log + ) + return elasticSearchTemplate.search(query, RichSkillDoc::class.java) } override fun countByApiSearch( @@ -338,30 +518,39 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear pageable: Pageable, collectionId: String? ): Long { - val nsq: NativeSearchQueryBuilder = buildQuery(pageable, publishStatus, apiSearch, collectionId) + val query = convertToNativeQuery( + pageable, + createFilter(apiSearch, publishStatus), + buildQuery(publishStatus, apiSearch, collectionId), + "CustomRichSkillQueriesImpl.countByApiSearch()", + log + ) + return elasticSearchTemplate.count(query, RichSkillDoc::class.java) + } + + private fun createFilter( apiSearch: ApiSearch, publishStatus: Set) : Query { + var fieldName = RichSkillDoc::publishStatus.name + var filterValues = publishStatus.map { ps -> ps.toString() } - return elasticSearchTemplate.count(nsq.build(), RichSkillDoc::class.java) + if ( !apiSearch.uuids.isNullOrEmpty() ) { + fieldName = RichSkillDoc::uuid.name + filterValues = apiSearch.uuids.filter { x: String? -> x != "" } + } + return createTermsDslQuery(fieldName, filterValues) } - fun buildQuery( - pageable: Pageable, + /** + * TODO upgrade to ElasticSearch v8.7.x api style; see KeywordEsRepo.kt & FindsAllByPublishStatus.kt + */ + private fun buildQuery( publishStatus: Set, apiSearch: ApiSearch, collectionId: String? ): NativeSearchQueryBuilder { - val nsq: NativeSearchQueryBuilder = NativeSearchQueryBuilder().withPageable(pageable) + val nsqb = NativeSearchQueryBuilder() val bq = boolQuery() - nsq.withQuery(bq) - nsq.withFilter( - BoolQueryBuilder().must( - termsQuery( - RichSkillDoc::publishStatus.name, - publishStatus.map { ps -> ps.toString() } - ) - ) - ) - + nsqb.withQuery(bq) apiSearch.filtered?.let { generateBoolQueriesFromApiSearchWithFilters(bq, it, publishStatus) } // treat the presence of query property to mean multi field search with that term @@ -436,7 +625,8 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear var apiSearchUuids = apiSearch.uuids?.filterNotNull()?.filter { x: String? -> x != "" } if (!apiSearchUuids.isNullOrEmpty()) { - nsq.withFilter( + // This is not needed & ignored by WguQueryHelper.convertToNativeQuery() + nsqb.withFilter( BoolQueryBuilder().must( termsQuery( RichSkillDoc::uuid.name, @@ -449,8 +639,7 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear bq.must( nestedQuery( RichSkillDoc::collections.name, - boolQuery() - .must(matchQuery(collectionsUuid, collectionId)), + boolQuery().must(matchQuery(collectionsUuid, collectionId)), ScoreMode.Avg ) ) @@ -458,16 +647,20 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear } } - return nsq + return nsqb } override fun findSimilar(apiSimilaritySearch: ApiSimilaritySearch): SearchHits { - val limitedPageable = OffsetPageable(0, 10, null) - val nsq: NativeSearchQueryBuilder = NativeSearchQueryBuilder().withPageable(limitedPageable).withQuery( - MatchPhraseQueryBuilder(RichSkillDoc::statement.name, apiSimilaritySearch.statement).slop(4) + val query = convertToNativeQuery( + OffsetPageable(0, 10, null), + null, + NativeSearchQueryBuilder().withQuery( MatchPhraseQueryBuilder(RichSkillDoc::statement.name, apiSimilaritySearch.statement).slop(4)), + "CustomRichSkillQueriesImpl.findSimilar()", + log ) - return elasticSearchTemplate.search(nsq.build(), RichSkillDoc::class.java) + return elasticSearchTemplate.search(query, RichSkillDoc::class.java) } + } @@ -482,4 +675,3 @@ interface RichSkillEsRepo : ElasticsearchRepository, CustomRi ): Page } - diff --git a/api/src/main/kotlin/edu/wgu/osmt/security/SecurityConfig.kt b/api/src/main/kotlin/edu/wgu/osmt/security/SecurityConfig.kt index bcb7124f1..f0e990d68 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/security/SecurityConfig.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/security/SecurityConfig.kt @@ -4,6 +4,8 @@ import com.fasterxml.jackson.databind.ObjectMapper import edu.wgu.osmt.RoutePaths import edu.wgu.osmt.api.model.ApiError import edu.wgu.osmt.config.AppConfig +import jakarta.servlet.http.HttpServletRequest +import jakarta.servlet.http.HttpServletResponse import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration @@ -11,19 +13,18 @@ import org.springframework.context.annotation.Profile import org.springframework.http.HttpMethod.* import org.springframework.security.config.annotation.web.builders.HttpSecurity import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity -import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter import org.springframework.security.core.Authentication import org.springframework.security.core.AuthenticationException import org.springframework.security.oauth2.core.oidc.user.OidcUser import org.springframework.security.web.AuthenticationEntryPoint import org.springframework.security.web.DefaultRedirectStrategy +import org.springframework.security.web.SecurityFilterChain import org.springframework.security.web.authentication.AuthenticationSuccessHandler import org.springframework.stereotype.Component import org.springframework.web.cors.CorsConfiguration import org.springframework.web.cors.CorsConfigurationSource import org.springframework.web.cors.UrlBasedCorsConfigurationSource -import javax.servlet.http.HttpServletRequest -import javax.servlet.http.HttpServletResponse + /** * Security configurations @@ -33,7 +34,7 @@ import javax.servlet.http.HttpServletResponse @Configuration @EnableWebSecurity @Profile("oauth2-okta | OTHER-OAUTH-PROFILE") -class SecurityConfig : WebSecurityConfigurerAdapter() { +class SecurityConfig { @Autowired lateinit var appConfig: AppConfig @@ -44,66 +45,67 @@ class SecurityConfig : WebSecurityConfigurerAdapter() { @Autowired lateinit var returnUnauthorized: ReturnUnauthorized - @Override - override fun configure(http: HttpSecurity) { + @Bean + fun securityFilterChain(http: HttpSecurity) : SecurityFilterChain { http .cors().and() .csrf().disable() .httpBasic().disable() - .authorizeRequests() - - .mvcMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_AUDIT_LOG}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILL_AUDIT_LOG}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILL_AUDIT_LOG}").authenticated() - .mvcMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_AUDIT_LOG}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_AUDIT_LOG}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_AUDIT_LOG}").authenticated() - .mvcMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.TASK_DETAIL_SKILLS}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.TASK_DETAIL_SKILLS}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.TASK_DETAIL_SKILLS}").authenticated() - .mvcMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.TASK_DETAIL_BATCH}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.TASK_DETAIL_BATCH}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.TASK_DETAIL_BATCH}").authenticated() - .mvcMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SEARCH_JOBCODES_PATH}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SEARCH_JOBCODES_PATH}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SEARCH_JOBCODES_PATH}").authenticated() - .mvcMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SEARCH_KEYWORDS_PATH}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SEARCH_KEYWORDS_PATH}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SEARCH_KEYWORDS_PATH}").authenticated() - - // public search endpoints - .mvcMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SEARCH_SKILLS}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SEARCH_SKILLS}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SEARCH_SKILLS}").permitAll() - .mvcMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SEARCH_COLLECTIONS}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SEARCH_COLLECTIONS}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SEARCH_COLLECTIONS}").permitAll() - - // public canonical URL endpoints - .mvcMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_DETAIL}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILL_DETAIL}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILL_DETAIL}").permitAll() - .mvcMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_DETAIL}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_DETAIL}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_DETAIL}").permitAll() - - .mvcMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_SKILLS}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_SKILLS}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_SKILLS}").permitAll() - .mvcMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_CSV}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_CSV}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_CSV}").permitAll() - .mvcMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.TASK_DETAIL_TEXT}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.TASK_DETAIL_TEXT}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.TASK_DETAIL_TEXT}").permitAll() - .mvcMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_XLSX}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_XLSX}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_XLSX}").permitAll() - .mvcMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.TASK_DETAIL_MEDIA}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.TASK_DETAIL_MEDIA}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.TASK_DETAIL_MEDIA}").permitAll() - - .and().exceptionHandling().authenticationEntryPoint(returnUnauthorized) + .authorizeHttpRequests { auth -> + auth + .requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_AUDIT_LOG}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILL_AUDIT_LOG}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILL_AUDIT_LOG}").authenticated() + .requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_AUDIT_LOG}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_AUDIT_LOG}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_AUDIT_LOG}").authenticated() + .requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.TASK_DETAIL_SKILLS}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.TASK_DETAIL_SKILLS}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.TASK_DETAIL_SKILLS}").authenticated() + .requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.TASK_DETAIL_BATCH}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.TASK_DETAIL_BATCH}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.TASK_DETAIL_BATCH}").authenticated() + .requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SEARCH_JOBCODES_PATH}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SEARCH_JOBCODES_PATH}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SEARCH_JOBCODES_PATH}").authenticated() + .requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SEARCH_KEYWORDS_PATH}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SEARCH_KEYWORDS_PATH}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SEARCH_KEYWORDS_PATH}").authenticated() + + // public search endpoints + .requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SEARCH_SKILLS}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SEARCH_SKILLS}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SEARCH_SKILLS}").permitAll() + .requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SEARCH_COLLECTIONS}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SEARCH_COLLECTIONS}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SEARCH_COLLECTIONS}").permitAll() + + // public canonical URL endpoints + .requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_DETAIL}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILL_DETAIL}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILL_DETAIL}").permitAll() + .requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_DETAIL}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_DETAIL}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_DETAIL}").permitAll() + + .requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_SKILLS}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_SKILLS}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_SKILLS}").permitAll() + .requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_CSV}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_CSV}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_CSV}").permitAll() + .requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.TASK_DETAIL_TEXT}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.TASK_DETAIL_TEXT}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.TASK_DETAIL_TEXT}").permitAll() + .requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_XLSX}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_XLSX}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_XLSX}").permitAll() + .requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.TASK_DETAIL_MEDIA}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.TASK_DETAIL_MEDIA}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.TASK_DETAIL_MEDIA}").permitAll() + } + + .exceptionHandling().authenticationEntryPoint(returnUnauthorized) .and().oauth2Login().successHandler(redirectToFrontend) .and().oauth2ResourceServer().jwt() @@ -112,6 +114,8 @@ class SecurityConfig : WebSecurityConfigurerAdapter() { } else { configureForNoRoles(http) } + + return http.build(); } fun configureForRoles(http: HttpSecurity) { @@ -121,93 +125,120 @@ class SecurityConfig : WebSecurityConfigurerAdapter() { val READ = appConfig.scopeRead if (appConfig.allowPublicLists) { - http.authorizeRequests() - .mvcMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_LIST}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILLS_LIST}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILLS_LIST}").permitAll() - .mvcMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTIONS_LIST}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTIONS_LIST}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTIONS_LIST}").permitAll() + http.authorizeHttpRequests { auth -> + auth + .requestMatchers( POST, + "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_FILTER}" + ).permitAll() + .requestMatchers( GET, + "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.CATEGORY_LIST}", + "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.WORKSPACE_LIST}", + + "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_LIST}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILLS_LIST}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILLS_LIST}", + + "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTIONS_LIST}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTIONS_LIST}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTIONS_LIST}") + .permitAll() + } } else { - http.authorizeRequests() - .mvcMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_LIST}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILLS_LIST}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILLS_LIST}").hasAnyAuthority(ADMIN, CURATOR, VIEW, READ) - .mvcMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTIONS_LIST}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTIONS_LIST}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTIONS_LIST}").hasAnyAuthority(ADMIN, CURATOR, VIEW, READ) + http.authorizeHttpRequests { auth -> + auth + .requestMatchers( POST, + "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_FILTER}") + .hasAnyAuthority(ADMIN, CURATOR, VIEW, READ) + .requestMatchers( GET, + "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.CATEGORY_LIST}", + "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.WORKSPACE_LIST}", + + "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_LIST}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILLS_LIST}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILLS_LIST}", + + "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTIONS_LIST}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTIONS_LIST}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTIONS_LIST}") + .hasAnyAuthority(ADMIN, CURATOR, VIEW, READ) + } } - http.authorizeRequests() - .mvcMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_UPDATE}", + http.authorizeHttpRequests { auth -> + auth + .requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_UPDATE}", "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILL_UPDATE}", "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILL_UPDATE}").hasAnyAuthority(ADMIN, CURATOR) - .mvcMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_CREATE}", + .requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_CREATE}", "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILLS_CREATE}", "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILLS_CREATE}").hasAnyAuthority(ADMIN, CURATOR) - .mvcMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_PUBLISH}", + .requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_PUBLISH}", "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILL_PUBLISH}", "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILL_PUBLISH}").hasAnyAuthority(ADMIN) - .mvcMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_CREATE}", + .requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_CREATE}", "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_CREATE}", "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_CREATE}").hasAnyAuthority(ADMIN, CURATOR) - .mvcMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_PUBLISH}", + .requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_PUBLISH}", "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_PUBLISH}", "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_PUBLISH}").hasAnyAuthority(ADMIN) - .mvcMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_UPDATE}", + .requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_UPDATE}", "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_UPDATE}", "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_UPDATE}").hasAnyAuthority(ADMIN, CURATOR) - .mvcMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_SKILLS_UPDATE}", + .requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_SKILLS_UPDATE}", "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_SKILLS_UPDATE}", "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_SKILLS_UPDATE}").hasAnyAuthority(ADMIN, CURATOR) - .mvcMatchers(DELETE, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_REMOVE}", + .requestMatchers(DELETE, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_REMOVE}", "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_REMOVE}", "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_REMOVE}").hasAnyAuthority(ADMIN) - .mvcMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.WORKSPACE_PATH}", + .requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.WORKSPACE_PATH}", "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.WORKSPACE_PATH}", "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.WORKSPACE_PATH}").hasAnyAuthority(ADMIN, CURATOR) - .mvcMatchers("/api/**").hasAnyAuthority(ADMIN, CURATOR, VIEW, READ) + .requestMatchers("/api/**").hasAnyAuthority(ADMIN, CURATOR, VIEW, READ) + .requestMatchers("/**").permitAll() + } } fun configureForNoRoles(http: HttpSecurity) { - http.authorizeRequests() - .mvcMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_LIST}", + http.authorizeHttpRequests { auth -> + auth + .requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_LIST}", "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILLS_LIST}", "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILLS_LIST}").permitAll() - .mvcMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTIONS_LIST}", + .requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTIONS_LIST}", "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTIONS_LIST}", "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTIONS_LIST}").permitAll() - .mvcMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_UPDATE}", + .requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_UPDATE}", "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILL_UPDATE}", "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILL_UPDATE}").authenticated() - .mvcMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_CREATE}", + .requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_CREATE}", "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILLS_CREATE}", "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILLS_CREATE}").authenticated() - .mvcMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_PUBLISH}", + .requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_PUBLISH}", "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILL_PUBLISH}", "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILL_PUBLISH}").authenticated() - .mvcMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_CREATE}", + .requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_CREATE}", "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_CREATE}", "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_CREATE}").authenticated() - .mvcMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_PUBLISH}", + .requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_PUBLISH}", "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_PUBLISH}", "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_PUBLISH}").authenticated() - .mvcMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_UPDATE}", + .requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_UPDATE}", "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_UPDATE}", "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_UPDATE}").authenticated() - .mvcMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_SKILLS_UPDATE}", + .requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_SKILLS_UPDATE}", "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_SKILLS_UPDATE}", "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_SKILLS_UPDATE}").authenticated() - .mvcMatchers(DELETE, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_REMOVE}", + .requestMatchers(DELETE, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_REMOVE}", "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_REMOVE}", "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_REMOVE}").denyAll() - // fall-through - .mvcMatchers("/api/**").permitAll() + // fall-through + .requestMatchers("/**").permitAll() + } } @Bean diff --git a/api/src/main/kotlin/edu/wgu/osmt/task/TaskMessageService.kt b/api/src/main/kotlin/edu/wgu/osmt/task/TaskMessageService.kt index 4f41a10df..40ecfa7ee 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/task/TaskMessageService.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/task/TaskMessageService.kt @@ -1,6 +1,6 @@ package edu.wgu.osmt.task -import com.github.sonus21.rqueue.core.RqueueMessageSender +import com.github.sonus21.rqueue.core.RqueueMessageEnqueuer import org.springframework.beans.factory.annotation.Autowired import org.springframework.data.redis.core.RedisTemplate import org.springframework.stereotype.Service @@ -9,7 +9,7 @@ import org.springframework.stereotype.Service class TaskMessageService { @Autowired - lateinit var rqueueMessageSender: RqueueMessageSender + lateinit var rqueueMessageSender: RqueueMessageEnqueuer @Autowired lateinit var redisTaskTemplate: RedisTemplate diff --git a/api/src/main/resources/config/application-dev.properties b/api/src/main/resources/config/application-dev.properties index 95d30312c..5de384416 100644 --- a/api/src/main/resources/config/application-dev.properties +++ b/api/src/main/resources/config/application-dev.properties @@ -14,6 +14,10 @@ management.endpoint.health.show-details=always spring.flyway.enabled=true # Common debuging log levels for OSMT development -#logging.level.org.springframework.data.elasticsearch.client.WIRE=trace -#logging.level.org.elasticsearch.client.RestClient=DEBUG -#logging.level.org.springframework=DEBUG +logging.level.org.springframework.data.elasticsearch.client.WIRE=trace +logging.level.org.elasticsearch.client.RestClient=DEBUG +logging.level.org.springframework=DEBUG +logging.level.edu.wgu.osmt=DEBUG + +app.enableRoles=false +app.allowPublicLists=false diff --git a/api/src/main/resources/config/application.properties b/api/src/main/resources/config/application.properties index f9bfaffe0..b1233e9fa 100644 --- a/api/src/main/resources/config/application.properties +++ b/api/src/main/resources/config/application.properties @@ -14,6 +14,8 @@ spring.datasource.url=${db.composedUrl} # Elasticsearch es.uri=${ELASTICSEARCH_URI:localhost:9200} +#es.username= +#es.password= # Redis redis.uri=${REDIS_URI:localhost:6379} diff --git a/api/src/test/kotlin/edu/wgu/osmt/keyword/KeywordEsRepoTest.kt b/api/src/test/kotlin/edu/wgu/osmt/keyword/KeywordEsRepoTest.kt index 83b67c6e5..3ec936838 100644 --- a/api/src/test/kotlin/edu/wgu/osmt/keyword/KeywordEsRepoTest.kt +++ b/api/src/test/kotlin/edu/wgu/osmt/keyword/KeywordEsRepoTest.kt @@ -54,6 +54,8 @@ class KeywordEsRepoTest @Autowired constructor( assertThat(result3.searchHits.count()).isEqualTo(2) assertThat(result4.searchHits.count()).isEqualTo(1) assertThat(result4.searchHits.first().content.value).isEqualTo("Yellow") - assertThat(result5.searchHits).hasSize(56) + + // Pagination causes the searchHits.count to be only 10 + assertThat(result5.totalHits).isEqualTo(56) } } diff --git a/api/src/test/kotlin/edu/wgu/osmt/richskill/RichSkillControllerTest.kt b/api/src/test/kotlin/edu/wgu/osmt/richskill/RichSkillControllerTest.kt index 9c1f72f22..aa22ad259 100644 --- a/api/src/test/kotlin/edu/wgu/osmt/richskill/RichSkillControllerTest.kt +++ b/api/src/test/kotlin/edu/wgu/osmt/richskill/RichSkillControllerTest.kt @@ -83,7 +83,7 @@ internal class RichSkillControllerTest @Autowired constructor( richSkillEsRepo.saveAll(listOfSkills) // Act - val result = richSkillController.allPaginatedV2( + val result = richSkillController.allPaginated( UriComponentsBuilder.newInstance(), size, 0, diff --git a/bin/lib/common.sh b/bin/lib/common.sh index f88e929a3..b50fecede 100755 --- a/bin/lib/common.sh +++ b/bin/lib/common.sh @@ -52,7 +52,8 @@ source_env_file() { source_env_file_unless_provided_oauth() { local env_file="${1}" - # gracefully bypass sourcing env file if these 4 OAUTH values are provided + # gracefully bypass sourcing env file if these 4 OAUTH_ values are provided, i.e. as secrets + # via build automation if [[ \ -n "${OAUTH_ISSUER}" && \ -n "${OAUTH_CLIENTID}" && \ @@ -63,22 +64,25 @@ source_env_file_unless_provided_oauth() { return 0 fi + echo_info "OAUTH_ values are not provided by environment variables. Sourcing ${env_file} env file." source_env_file "${env_file}" } source_env_file_unless_provided_okta() { local env_file="${1}" - # gracefully bypass sourcing env file if these 4 OAUTH values are provided + # gracefully bypass sourcing env file if these 3 OKTA_ values are provided, i.e. as secrets + # via build automation if [[ \ -n "${OKTA_URL}" && \ -n "${OKTA_USERNAME}" && \ -n "${OKTA_PASSWORD}" \ ]]; then - echo_info "Okta values are provided by environment variables. Not sourcing ${env_file} env file." + echo_info "OKTA_ values are provided by environment variables. Not sourcing ${env_file} env file." return 0 fi + echo_info "Okta values are not provided by environment variables. Sourcing ${env_file} env file." source_env_file "${env_file}" } @@ -265,7 +269,7 @@ _validate_java_version() { echo echo_info "Checking Java..." # OSMT requires at least Java 11 - local -i req_java_major=11 + local -i req_java_major=17 local det_java_version local -i det_java_major @@ -317,10 +321,10 @@ _validate_osmt_dev_dependencies() { echo_info "Maven version: $(mvn --version)" echo - echo_info "OSMT development recommends NodeJS version v16.13.0 or greater. Maven uses an embedded copy of NodeJS v16.13.0 via frontend-maven-plugin." + echo_info "OSMT development recommends NodeJS version v18.18.2 or greater. Maven uses an embedded copy of NodeJS v16.13.0 via frontend-maven-plugin." echo_info "NodeJS version: $(node --version)" echo - echo_info "OSMT development recommends npm version 8.1.0 or greater. Maven uses an embedded copy of npm 8.1.0 via frontend-maven-plugin." + echo_info "OSMT development recommends npm version 9.8.1 or greater. Maven uses an embedded copy of npm 8.1.0 via frontend-maven-plugin." echo_info "npm version: $(npm --version)" if [[ "${is_dependency_valid}" -ne 0 ]]; then echo diff --git a/pom.xml b/pom.xml index c44351960..780b4973c 100644 --- a/pom.xml +++ b/pom.xml @@ -5,13 +5,13 @@ org.springframework.boot spring-boot-starter-parent - 2.7.7 + 3.1.2 edu.wgu.osmt osmt-parent - 2.6.0-SNAPSHOT + 2.6.1-SNAPSHOT pom OSMT @@ -25,6 +25,32 @@ + + UTF-8 + UTF-8 + 17 + 8.0.31 + + + + + mysql + mysql-connector-java + ${mysql.version} + + + + + + + mysql + mysql-connector-java + + + com.mysql + mysql-connector-j + + Open Source at WGU @@ -34,12 +60,6 @@ - - UTF-8 - UTF-8 - 11 - - scm:git:ssh://git@github.com/wgu-opensource/osmt.git scm:git:ssh://git@github.com/wgu-opensource/osmt.git diff --git a/test/pom.xml b/test/pom.xml index d62816fc8..3fdaa0534 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -6,12 +6,12 @@ edu.wgu.osmt osmt-parent - 2.6.0-SNAPSHOT + 2.6.1-SNAPSHOT edu.wgu.osmt osmt-api-test - 2.6.0-SNAPSHOT + 2.6.1-SNAPSHOT OSMT API Tests diff --git a/ui/pom.xml b/ui/pom.xml index 428d32285..e4cc8c079 100644 --- a/ui/pom.xml +++ b/ui/pom.xml @@ -4,12 +4,12 @@ edu.wgu.osmt osmt-parent - 2.6.0-SNAPSHOT + 2.6.1-SNAPSHOT edu.wgu.osmt osmt-ui - 2.6.0-SNAPSHOT + 2.6.1-SNAPSHOT jar