diff --git a/.env.template b/.env.template index 407b99d50..abe859391 100644 --- a/.env.template +++ b/.env.template @@ -3,6 +3,8 @@ USER_ID= GROUP_ID= +ORCID_SANDBOX_AUTHENTICATION=use the Sandbox Orcid, false=>use the Production Orcid. Defaults to false.> + # Authentication variables OAUTH_CLIENT_ID= OAUTH_CLIENT_SECRET= @@ -59,4 +61,12 @@ AWS_GENO_BUCKET= AWS_S3_ENDPOINT= BRAPI_VENDOR_SUBMISSION_ENABLED=false #can a submission be sent to a vendor via BrAPI -BRAPI_VENDOR_CHECK_FREQUENCY=1d #how often to check for vendor updates for sample submissions \ No newline at end of file +BRAPI_VENDOR_CHECK_FREQUENCY=1d #how often to check for vendor updates for sample submissions + +#The initial caching of each type of object needs to be staggered by the prescribed number of seconds +GERMPLASM_START_DELAY=5s +STUDY_START_DELAY=10s +TRIAL_START_DELAY=15s +TRAIT_START_DELAY=20s +OBSERVATION_START_DELAY=25s +OBSERVATION_UNIT_START_DELAY=30s \ No newline at end of file diff --git a/pom.xml b/pom.xml index 1bfde3251..fe89f2ca6 100644 --- a/pom.xml +++ b/pom.xml @@ -490,7 +490,7 @@ shade - + *:* @@ -499,9 +499,9 @@ META-INF/*.RSA - - - + + + ${exec.mainClass} @@ -536,98 +536,98 @@ ${maven.compiler.target} - - org.jooq - jooq-codegen-maven - ${jooq.version} - - - - generate - - - + + org.jooq + jooq-codegen-maven + ${jooq.version} + + + + generate + + + + + + + org.postgresql.Driver + jdbc:postgresql://${DB_SERVER}/${DB_NAME} + ${DB_USER} + ${DB_PASSWORD} + + + org.breedinginsight.generation.JooqDaoGenerator + + org.jooq.meta.postgres.PostgresDatabase + .* + public + true + true + false + true + true + true + true + false + + flyway_schema_history|base_entity|base_track_edit_entity|spatial_ref_sys + + - - - org.postgresql.Driver + + + + + + PASCAL + $0_ENTITY + + + PASCAL + $0_TABLE + +
+ + + + + + org.breedinginsight.dao.db + target/generated-sources/jooq + + + true + true + false + true + + + + + + org.breedinginsight + bi-jooq-codegen + ${jooq.version} + + + + + org.flywaydb + flyway-maven-plugin + ${flyway.version} + jdbc:postgresql://${DB_SERVER}/${DB_NAME} ${DB_USER} ${DB_PASSWORD} - - - org.breedinginsight.generation.JooqDaoGenerator - - org.jooq.meta.postgres.PostgresDatabase - .* - public - true - true - false - true - true - true - true - false - - flyway_schema_history|base_entity|base_track_edit_entity|spatial_ref_sys - - - - - - - - - PASCAL - $0_ENTITY - - - PASCAL - $0_TABLE - -
- - - - - - org.breedinginsight.dao.db - target/generated-sources/jooq - - - true - true - false - true - - - - - - org.breedinginsight - bi-jooq-codegen - ${jooq.version} - - - - - org.flywaydb - flyway-maven-plugin - ${flyway.version} - - jdbc:postgresql://${DB_SERVER}/${DB_NAME} - ${DB_USER} - ${DB_PASSWORD} - - - - org.postgresql - postgresql - ${postgres.version} - - - + + + + org.postgresql + postgresql + ${postgres.version} + + + diff --git a/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java b/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java index a6e172c94..8619a140a 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java @@ -298,4 +298,43 @@ public HttpResponse deleteExperimentalCollaborator( } } + + /** + * Deletes an experiment. + * @param programId The UUID of the program + * @param experimentId The UUID of the experiment + * @param hard Specifies hard or soft delete + * @return A Http Response + */ + @Delete("/${micronaut.bi.api.version}/programs/{programId}/experiments/{experimentId}{?hard}") + @ProgramSecured(roles = {ProgramSecuredRole.PROGRAM_ADMIN, ProgramSecuredRole.SYSTEM_ADMIN}) + @Produces(MediaType.APPLICATION_JSON) + public HttpResponse deleteExperiment( + @PathVariable("programId") UUID programId, + @PathVariable("experimentId") UUID experimentId, + @QueryValue(defaultValue = "false") Boolean hard + ) throws ApiException { + try { + Optional program = programService.getById(programId); + if(program.isEmpty()) { + return HttpResponse.notFound(); + } + int observationCount = experimentService.deleteExperiment(program.get(), experimentId, hard); + if (observationCount > 0 && hard) { + // 409 Conflict. https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/409 + return HttpResponse.status(HttpStatus.CONFLICT); + } + // 204 No Content indicates successful delete. https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/204 + return HttpResponse.noContent(); + } catch (ApiException e) { + log.error("Error deleting experiment.\n\tprogramId: " + programId + "\n\texperimentId: " + experimentId + "\n\thard: " + hard); + if (e.getCode() == 404) { + return HttpResponse.notFound(); + } else { + return HttpResponse.serverError(); + } + } + } + + } diff --git a/src/main/java/org/breedinginsight/api/v1/controller/geno/SampleSubmissionController.java b/src/main/java/org/breedinginsight/api/v1/controller/geno/SampleSubmissionController.java index e99bb5e70..e01ab2c08 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/geno/SampleSubmissionController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/geno/SampleSubmissionController.java @@ -281,4 +281,42 @@ public HttpResponse> checkVendorStatus(@PathVariable return HttpResponse.serverError(); } } + + /** + * Delete sample submission. + * Deletes the bidb submission record and BrAPI samples & plates + * @param programId bi-api id of program + * @param submissionId bi-api id of submission + * @return HttpResponse + * @throws ApiException + */ + @Delete("programs/{programId}/submissions/{submissionId}") + @Produces(MediaType.APPLICATION_JSON) + // sys admin and program admin roles to match file import permissions + @ProgramSecured(roles = {ProgramSecuredRole.SYSTEM_ADMIN, ProgramSecuredRole.PROGRAM_ADMIN}) + public HttpResponse deleteSubmissionById(@PathVariable UUID programId, @PathVariable UUID submissionId) throws ApiException { + + // program validation + Optional program = programService.getById(programId); + if(program.isEmpty()) { + log.info(String.format("programId not found: %s", programId.toString())); + return HttpResponse.notFound(); + } + + // sample status validation + Optional submissionOpt = sampleSubmissionService.getSampleSubmission(program.get(), submissionId, false); + + if(submissionOpt.isEmpty()) { + return HttpResponse.notFound(); + } + SampleSubmission submission = submissionOpt.get(); + if (!submission.isDeletable()) { + return HttpResponse.notAllowed(); + } + + sampleSubmissionService.deleteSampleSubmission(program.get(), submissionId); + + return HttpResponse.ok(); + } + } diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIGermplasmController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIGermplasmController.java index 948e68c20..84a6a17f0 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIGermplasmController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIGermplasmController.java @@ -1,5 +1,6 @@ package org.breedinginsight.brapi.v2; +import com.drew.lang.annotations.Nullable; import io.micronaut.context.annotation.Property; import io.micronaut.http.HttpHeaders; import io.micronaut.http.HttpResponse; @@ -19,10 +20,11 @@ import org.brapi.v2.model.BrAPIIndexPagination; import org.brapi.v2.model.BrAPIMetadata; import org.brapi.v2.model.BrAPIStatus; -import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.germ.*; import org.brapi.v2.model.germ.request.BrAPIGermplasmSearchRequest; -import org.brapi.v2.model.germ.response.*; +import org.brapi.v2.model.germ.response.BrAPIGermplasmListResponse; +import org.brapi.v2.model.germ.response.BrAPIGermplasmPedigreeResponse; +import org.brapi.v2.model.germ.response.BrAPIGermplasmProgenyResponse; import org.breedinginsight.api.auth.ProgramSecured; import org.breedinginsight.api.auth.ProgramSecuredRoleGroup; import org.breedinginsight.api.model.v1.request.query.SearchRequest; @@ -33,27 +35,25 @@ import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; import org.breedinginsight.brapi.v2.model.request.query.GermplasmQuery; -import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; -import org.breedinginsight.model.Program; -import org.breedinginsight.services.ProgramService; -import org.breedinginsight.utilities.Utilities; -import org.breedinginsight.utilities.response.mappers.GermplasmQueryMapper; import org.breedinginsight.brapi.v2.services.BrAPIGermplasmService; import org.breedinginsight.brapps.importer.model.exports.FileType; import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.model.DownloadFile; import org.breedinginsight.model.GermplasmGenotype; +import org.breedinginsight.model.Program; +import org.breedinginsight.services.ProgramService; import org.breedinginsight.services.brapi.BrAPIEndpointProvider; import org.breedinginsight.services.exceptions.AuthorizationException; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.geno.GenotypeService; +import org.breedinginsight.utilities.Utilities; import org.breedinginsight.utilities.response.ResponseUtils; +import org.breedinginsight.utilities.response.mappers.GermplasmQueryMapper; import javax.inject.Inject; import javax.validation.Valid; import java.util.*; import java.util.regex.Pattern; -import java.util.stream.Collectors; @Slf4j @Controller("/${micronaut.bi.api.version}") @@ -209,7 +209,8 @@ public HttpResponse>>> getGermplasm( germplasmQueryMapper.setDateDisplayFormat(dateFormatParam); } - List germplasm = germplasmService.getGermplasm(programId); + // Fetch all germplasm in the program unless a list id is supplied to return only germplasm in that collection + List germplasm = queryParams.getListDbId() == null ? germplasmService.getGermplasm(programId) : germplasmService.getGermplasmByList(programId, queryParams.getListDbId()); SearchRequest searchRequest = queryParams.constructSearchRequest(); return ResponseUtils.getBrapiQueryResponse(germplasm, germplasmQueryMapper, queryParams, searchRequest); } catch (ApiException e) { @@ -221,52 +222,17 @@ public HttpResponse>>> getGermplasm( } } - @Get("/programs/{programId}/germplasm/lists/{listDbId}/records{?queryParams*}") - @Produces(MediaType.APPLICATION_JSON) - @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.PROGRAM_SCOPED_ROLES}) - public HttpResponse>>> getGermplasmListRecords( - @PathVariable("programId") UUID programId, - @PathVariable("listDbId") String listDbId, - @QueryValue @QueryValid(using = GermplasmQueryMapper.class) @Valid GermplasmQuery queryParams) { - try { - List germplasm = germplasmService.getGermplasmByList(programId, listDbId); - SearchRequest searchRequest = queryParams.constructSearchRequest(); - return ResponseUtils.getBrapiQueryResponse(germplasm, germplasmQueryMapper, queryParams, searchRequest); - } catch (Exception e) { - log.info(e.getMessage(), e); - return HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "Error retrieving germplasm list records"); - } - } - - @Get("/programs/{programId}/germplasm/lists/{listDbId}/export{?fileExtension}") - @Produces(value = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") - @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.PROGRAM_SCOPED_ROLES}) - public HttpResponse germplasmListExport( - @PathVariable("programId") UUID programId, @PathVariable("listDbId") String listDbId, @QueryValue(defaultValue = "XLSX") String fileExtension) { - String downloadErrorMessage = "An error occurred while generating the download file. Contact the development team at bidevteam@cornell.edu."; - try { - FileType extension = Enum.valueOf(FileType.class, fileExtension); - DownloadFile germplasmListFile = germplasmService.exportGermplasmList(programId, listDbId, extension); - HttpResponse germplasmListExport = HttpResponse.ok(germplasmListFile.getStreamedFile()).header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename="+germplasmListFile.getFileName()+extension.getExtension()); - return germplasmListExport; - } - catch (Exception e) { - log.info(e.getMessage(), e); - e.printStackTrace(); - HttpResponse response = HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, downloadErrorMessage).contentType(MediaType.TEXT_PLAIN).body(downloadErrorMessage); - return response; - } - } - - @Get("/programs/{programId}/germplasm/export{?fileExtension}") + @Get("/programs/{programId}/germplasm/export{?fileExtension,list}") @Produces(value = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.PROGRAM_SCOPED_ROLES}) public HttpResponse germplasmExport( - @PathVariable("programId") UUID programId, @QueryValue(defaultValue = "XLSX") String fileExtension) { + @PathVariable("programId") UUID programId, + @QueryValue(defaultValue = "XLSX") String fileExtension, + @QueryValue Optional list) { String downloadErrorMessage = "An error occurred while generating the download file. Contact the development team at bidevteam@cornell.edu."; try { FileType extension = Enum.valueOf(FileType.class, fileExtension); - DownloadFile germplasmListFile = germplasmService.exportGermplasm(programId, extension); + DownloadFile germplasmListFile = list.isEmpty() ? germplasmService.exportGermplasm(programId, extension) : germplasmService.exportGermplasmList(programId, list.get(), extension); HttpResponse germplasmExport = HttpResponse.ok(germplasmListFile.getStreamedFile()).header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename="+germplasmListFile.getFileName()+extension.getExtension()); return germplasmExport; } diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIListController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIListController.java index 3dd6fd1f4..10528e298 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIListController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIListController.java @@ -1,29 +1,15 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package org.breedinginsight.brapi.v2; +import io.micronaut.http.HttpRequest; import io.micronaut.http.HttpResponse; import io.micronaut.http.HttpStatus; import io.micronaut.http.MediaType; import io.micronaut.http.annotation.*; import io.micronaut.security.annotation.Secured; import io.micronaut.security.rules.SecurityRule; +import io.reactivex.rxjava3.core.Single; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Pair; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIListSummary; import org.brapi.v2.model.core.BrAPIListTypes; @@ -41,31 +27,35 @@ import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.utilities.response.ResponseUtils; import org.breedinginsight.utilities.response.mappers.ListQueryMapper; +import reactor.core.publisher.Mono; +import reactor.util.function.Tuples; import javax.inject.Inject; import javax.validation.Valid; +import javax.validation.Validator; import java.util.List; +import java.util.Optional; import java.util.UUID; @Slf4j -@Controller +@Controller("/${micronaut.bi.api.version}/programs/{programId}" + BrapiVersion.BRAPI_V2) @Secured(SecurityRule.IS_AUTHENTICATED) public class BrAPIListController { - private final ProgramService programService; - private final BrAPIListService listService; + private final BrAPIListService brapiListService; private final ListQueryMapper listQueryMapper; + private final Validator validator; @Inject - public BrAPIListController(ProgramService programService, BrAPIListService listService, - ListQueryMapper listQueryMapper) { + public BrAPIListController(ProgramService programService, BrAPIListService brapiListService, + ListQueryMapper listQueryMapper, Validator validator) { this.programService = programService; - this.listService = listService; + this.brapiListService = brapiListService; this.listQueryMapper = listQueryMapper; + this.validator = validator; } - //@Get(BrapiVersion.BRAPI_V2 + "/lists") - @Get("/${micronaut.bi.api.version}/programs/{programId}" + BrapiVersion.BRAPI_V2 + "/lists{?queryParams*}") + @Get("/lists{?queryParams*}") @Produces(MediaType.APPLICATION_JSON) @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.PROGRAM_SCOPED_ROLES}) public HttpResponse>> getLists( @@ -77,7 +67,6 @@ public HttpResponse>> getLists( .getById(programId) .orElseThrow(() -> new DoesNotExistException("Program does not exist")); - // get germplasm lists by default BrAPIListTypes type = BrAPIListTypes.fromValue(queryParams.getListType()); String source = null; String id = null; @@ -93,7 +82,7 @@ public HttpResponse>> getLists( if (dateFormatParam != null) { listQueryMapper.setDateDisplayFormat(dateFormatParam); } - List brapiLists = listService.getListSummariesByTypeAndXref(type, source, id, program); + List brapiLists = brapiListService.getListSummariesByTypeAndXref(type, source, id, program); SearchRequest searchRequest = queryParams.constructSearchRequest(); return ResponseUtils.getBrapiQueryResponse(brapiLists, listQueryMapper, queryParams, searchRequest); @@ -105,4 +94,25 @@ public HttpResponse>> getLists( throw new RuntimeException(e); } } + + @Delete("/lists/{listDbId}") + @Produces(MediaType.APPLICATION_JSON) + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.PROGRAM_SCOPED_ROLES}) + public HttpResponse deleteListById( + @PathVariable("programId") UUID programId, + @PathVariable("listDbId") String listDbId, + HttpRequest request + ) { + boolean hardDelete = false; + if (request.getParameters().contains("hardDelete")) { + String paramValue = request.getParameters().get("hardDelete"); + hardDelete = "true".equals(paramValue); + } + try { + return brapiListService.deleteBrAPIList(listDbId, programId, hardDelete); + } catch (Exception e) { + log.info(e.getMessage(), e); + return HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "Error deleting germplasm list"); + } + } } diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationsController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationsController.java index cbb476910..27ac4e34c 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationsController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationsController.java @@ -28,13 +28,17 @@ import org.brapi.client.v2.ApiResponse; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.modules.phenotype.ObservationsApi; +import org.brapi.v2.model.BrAPIIndexPagination; +import org.brapi.v2.model.BrAPIMetadata; +import org.brapi.v2.model.BrAPIStatus; import org.brapi.v2.model.BrAPIWSMIMEDataTypes; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.pheno.BrAPIObservation; -import org.brapi.v2.model.pheno.response.BrAPIObservationTableResponse; +import org.brapi.v2.model.pheno.response.*; import org.breedinginsight.api.auth.ProgramSecured; import org.breedinginsight.api.auth.ProgramSecuredRoleGroup; import org.breedinginsight.brapi.v1.controller.BrapiVersion; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; import org.breedinginsight.brapi.v2.model.request.query.ObservationQuery; import org.breedinginsight.daos.ProgramDAO; @@ -47,6 +51,8 @@ import javax.annotation.Nullable; import javax.inject.Inject; import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; @Slf4j @Controller("/${micronaut.bi.api.version}/programs/{programId}" + BrapiVersion.BRAPI_V2) @@ -57,13 +63,20 @@ public class BrAPIObservationsController { private final ProgramDAO programDAO; private final BrAPIStudyDAO brAPIStudyDAO; private final BrAPIEndpointProvider brAPIEndpointProvider; + private final BrAPIObservationDAO observationDAO; @Inject - public BrAPIObservationsController(ProgramService programService, ProgramDAO programDAO, ProgramDAO programDAO1, BrAPIStudyDAO brAPIStudyDAO, BrAPIEndpointProvider brAPIEndpointProvider) { + public BrAPIObservationsController(ProgramService programService, + ProgramDAO programDAO, + ProgramDAO programDAO1, + BrAPIStudyDAO brAPIStudyDAO, + BrAPIEndpointProvider brAPIEndpointProvider, + BrAPIObservationDAO brAPIObservationDAO) { this.programService = programService; this.programDAO = programDAO1; this.brAPIStudyDAO = brAPIStudyDAO; this.brAPIEndpointProvider = brAPIEndpointProvider; + this.observationDAO = brAPIObservationDAO; } @Get("/observations") @@ -93,8 +106,99 @@ public HttpResponse observationsGet(@PathVariable("programId") UUID programId, @Nullable @QueryValue("externalReferenceSource") String externalReferenceSource, @Nullable @QueryValue("page") Integer page, @Nullable @QueryValue("pageSize") Integer pageSize) { - //TODO - return HttpResponse.notFound(); + log.debug("observationsGet: fetching observations by filters"); + Optional program = programService.getById(programId); + if(program.isEmpty()) { + log.warn("Program id: " + programId + " not found"); + return HttpResponse.notFound(); + } + + try { + + // TODO: BI-2506 - implement support for all query parameters. + List unsupportedParams = Stream.of( + observationDbId, + observationUnitDbId, + observationVariableDbId, + locationDbId, + seasonDbId, + observationTimeStampRangeStart, + observationTimeStampRangeEnd, + observationUnitLevelName, + observationUnitLevelOrder, + observationUnitLevelCode, + observationUnitLevelRelationshipName, + observationUnitLevelRelationshipOrder, + observationUnitLevelRelationshipCode, + observationUnitLevelRelationshipDbId, + commonCropName, + programDbId, + trialDbId, + germplasmDbId, + externalReferenceID, + externalReferenceId, + externalReferenceSource + ).filter(Objects::nonNull).collect(Collectors.toList()); + + if (!unsupportedParams.isEmpty()) { + return HttpResponse.status(HttpStatus.NOT_IMPLEMENTED).body( + new BrAPIObservationListResponse().metadata(new BrAPIMetadata().status(List.of(new BrAPIStatus().messageType(BrAPIStatus.MessageTypeEnum.ERROR) + .message("Unsupported query parameter. Only studyDbId, page, and pageSize are supported.")))) + ); + } + + // Get a filtered list of observations. + List observations = observationDAO.getObservationsByFilters(program.get(), studyDbId); + + // If page is not provided, set it to the default value 0. + if (page == null) page = 0; + // If pageSize is not provided, set it to the default value 1000. + if (pageSize == null) pageSize = 1000; + + // Total number of records in the unpaged super set. + int totalCount = observations.size(); + // The least of pageSize and totalCount, unless pageSize is zero, in which case use totalCount. + int requestedPageSize = pageSize > 0 ? Math.min(pageSize, totalCount) : totalCount; + // Integer division and round up. + int totalPages = totalCount / requestedPageSize + ((totalCount % requestedPageSize == 0) ? 0 : 1); + log.info("(Pagination) totalCount: " + totalCount + " page (0-indexed): " + page + " requestedPageSize: " + requestedPageSize + " totalPages: " + totalPages); + + // Determine validity of pagination query parameters. + boolean pageSizeValid = pageSize > 0; + boolean pageValid = page >= 0 && page < totalPages; + + // Only paginate if valid pagination values were sent. + if (pageSizeValid && pageValid) { + int start = page * requestedPageSize; + // Account for last page, which may have fewer than requestedPageSize items, or exactly requestedPageSize items. + int end = (page == (totalPages - 1) && totalCount % requestedPageSize != 0) ? (start + (totalCount % requestedPageSize)) : Math.min(((page + 1) * requestedPageSize), totalCount); + log.info("(Pagination) start " + start + " end " + end); + // Sort observations so that paging is consistent and coherent. + observations.sort(Comparator.comparing(BrAPIObservation::getObservationDbId)); + // Paginate response. + observations = observations.subList(start, end); + } else { + String errorMessage = "Invalid query parameters: page, pageSize"; + return HttpResponse.badRequest(new BrAPIObservationListResponse().metadata(new BrAPIMetadata().status(List.of(new BrAPIStatus().messageType(BrAPIStatus.MessageTypeEnum.ERROR) + .message(errorMessage))))); + } + + return HttpResponse.ok(new BrAPIObservationListResponse().metadata(new BrAPIMetadata().pagination(new BrAPIIndexPagination() + .currentPage(page) + .totalPages(totalPages) + .pageSize(observations.size()) + .totalCount(totalCount))) + .result(new BrAPIObservationListResponseResult().data(observations))); + + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + return HttpResponse.serverError(new BrAPIObservationListResponse().metadata(new BrAPIMetadata().status(List.of(new BrAPIStatus().messageType(BrAPIStatus.MessageTypeEnum.ERROR) + .message("Error fetching observations"))))); + } catch (Exception e) { + log.error("Error fetching Observations", e); + return HttpResponse.serverError(new BrAPIObservationListResponse().metadata(new BrAPIMetadata().status(List.of(new BrAPIStatus().messageType(BrAPIStatus.MessageTypeEnum.ERROR) + .message("Error fetching observations"))))); + } } @Get("/observations/{observationDbId}") diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPICachedDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPICachedDAO.java new file mode 100644 index 000000000..2d623b02b --- /dev/null +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPICachedDAO.java @@ -0,0 +1,15 @@ +package org.breedinginsight.brapi.v2.dao; + +import org.breedinginsight.daos.cache.ProgramCache; + +import java.util.UUID; + +public abstract class BrAPICachedDAO { + + protected ProgramCache programCache; + + public void repopulateCache(UUID programId) { + this.programCache.invalidate(programId); + this.programCache.populate(programId); + } +} diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java index 2e9371be4..75226babf 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java @@ -78,7 +78,7 @@ public BrAPIGermplasmDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUti this.brAPIEndpointProvider = brAPIEndpointProvider; } - @Scheduled(initialDelay = "2s") + @Scheduled(initialDelay = "${startup.delay.germplasm}") public void setup() { if(!runScheduledTasks) { return; diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIListDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIListDAO.java index 801b9775c..5ed73df60 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIListDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIListDAO.java @@ -17,10 +17,12 @@ package org.breedinginsight.brapi.v2.dao; +import io.micronaut.http.HttpResponse; import lombok.extern.slf4j.Slf4j; +import okhttp3.HttpUrl; +import okhttp3.Request; import org.brapi.client.v2.ApiResponse; import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.client.v2.model.queryParams.core.ListQueryParams; import org.brapi.client.v2.modules.core.ListsApi; import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.BrAPIResponse; @@ -29,8 +31,10 @@ import org.brapi.v2.model.core.BrAPIListTypes; import org.brapi.v2.model.core.request.BrAPIListNewRequest; import org.brapi.v2.model.core.request.BrAPIListSearchRequest; -import org.brapi.v2.model.core.response.*; -import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.core.response.BrAPIListsListResponse; +import org.brapi.v2.model.core.response.BrAPIListsListResponseResult; +import org.brapi.v2.model.core.response.BrAPIListsSingleResponse; import org.breedinginsight.brapps.importer.daos.ImportDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.daos.ProgramDAO; @@ -54,14 +58,17 @@ public class BrAPIListDAO { private final BrAPIEndpointProvider brAPIEndpointProvider; @Inject - public BrAPIListDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil brAPIDAOUtil, BrAPIEndpointProvider brAPIEndpointProvider) { + public BrAPIListDAO(ProgramDAO programDAO, + ImportDAO importDAO, + BrAPIDAOUtil brAPIDAOUtil, + BrAPIEndpointProvider brAPIEndpointProvider) { this.programDAO = programDAO; this.importDAO = importDAO; this.brAPIDAOUtil = brAPIDAOUtil; this.brAPIEndpointProvider = brAPIEndpointProvider; } - public List getListByName(List listNames, UUID programId) throws ApiException { + public List getListsByName(List listNames, UUID programId) throws ApiException { if(listNames.isEmpty()) { return Collections.emptyList(); } @@ -82,7 +89,7 @@ public BrAPIListsSingleResponse getListById(String listId, UUID programId) throw return response.getBody(); } - public List getListBySearch(@NotNull BrAPIListSearchRequest searchRequest, UUID programId) throws ApiException { + public List getListsBySearch(@NotNull BrAPIListSearchRequest searchRequest, UUID programId) throws ApiException { ListsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ListsApi.class); List programLists = brAPIDAOUtil.search(api::searchListsPost, api::searchListsSearchResultsDbIdGet, searchRequest); if (searchRequest.getExternalReferenceSources() != null && searchRequest.getExternalReferenceIDs() != null) { @@ -94,7 +101,7 @@ public List getListBySearch(@NotNull BrAPIListSearchRequest se } - public List getListByTypeAndExternalRef(@NotNull BrAPIListTypes listType, UUID programId, String externalReferenceSource, UUID externalReferenceId) throws ApiException { + public List getListsByTypeAndExternalRef(@NotNull BrAPIListTypes listType, UUID programId, String externalReferenceSource, UUID externalReferenceId) throws ApiException { BrAPIListSearchRequest searchRequest = new BrAPIListSearchRequest() .externalReferenceIDs(List.of(externalReferenceId.toString())) .externalReferenceSources(List.of(externalReferenceSource)) @@ -196,4 +203,19 @@ public List createBrAPILists(List brapiLi throw new ApiException("No response after creating list"); } + + public HttpResponse deleteBrAPIList(String listDbId, UUID programId, boolean hardDelete) throws ApiException { + // TODO: Switch to using the ListsApi from the BrAPI client library once the delete endpoints from BI-2397 are merged. + var programBrAPIBaseUrl = brAPIDAOUtil.getProgramBrAPIBaseUrl(programId); + var requestUrl = HttpUrl.parse(programBrAPIBaseUrl + "/lists/" + listDbId).newBuilder(); + requestUrl.addQueryParameter("hardDelete", Boolean.toString(hardDelete)); + HttpUrl url = requestUrl.build(); + var brapiRequest = new Request.Builder().url(url) + .method("DELETE", null) + .addHeader("Content-Type", "application/json") + .build(); + + return brAPIDAOUtil.makeCall(brapiRequest); + } + } diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java index e32dac6d9..10048564c 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java @@ -36,10 +36,12 @@ import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.daos.ProgramDAO; -import org.breedinginsight.daos.cache.ProgramCache; import org.breedinginsight.daos.cache.ProgramCacheProvider; import org.breedinginsight.model.Program; +import org.breedinginsight.model.Trait; +import org.breedinginsight.services.TraitService; import org.breedinginsight.services.brapi.BrAPIEndpointProvider; +import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.utilities.BrAPIDAOUtil; import org.breedinginsight.utilities.Utilities; import org.jetbrains.annotations.NotNull; @@ -54,7 +56,7 @@ @Singleton @Slf4j -public class BrAPIObservationDAO { +public class BrAPIObservationDAO extends BrAPICachedDAO { private ProgramDAO programDAO; private ImportDAO importDAO; @@ -63,7 +65,7 @@ public class BrAPIObservationDAO { private final BrAPIEndpointProvider brAPIEndpointProvider; private final String referenceSource; private boolean runScheduledTasks; - private final ProgramCache programObservationCache; + private final TraitService traitService; @Inject public BrAPIObservationDAO(ProgramDAO programDAO, @@ -73,7 +75,7 @@ public BrAPIObservationDAO(ProgramDAO programDAO, BrAPIEndpointProvider brAPIEndpointProvider, @Property(name = "brapi.server.reference-source") String referenceSource, @Property(name = "micronaut.bi.api.run-scheduled-tasks") boolean runScheduledTasks, - ProgramCacheProvider programCacheProvider) { + ProgramCacheProvider programCacheProvider, TraitService traitService) { this.programDAO = programDAO; this.importDAO = importDAO; this.observationUnitDAO = observationUnitDAO; @@ -81,10 +83,11 @@ public BrAPIObservationDAO(ProgramDAO programDAO, this.brAPIEndpointProvider = brAPIEndpointProvider; this.referenceSource = referenceSource; this.runScheduledTasks = runScheduledTasks; - this.programObservationCache = programCacheProvider.getProgramCache(this::fetchProgramObservations, BrAPIObservation.class); + this.traitService = traitService; + this.programCache = programCacheProvider.getProgramCache(this::fetchProgramObservations, BrAPIObservation.class); } - @Scheduled(initialDelay = "3s") + @Scheduled(initialDelay = "${startup.delay.observation}") public void setup() { if(!runScheduledTasks) { return; @@ -93,7 +96,7 @@ public void setup() { log.debug("populating observation cache"); List programs = programDAO.getActive(); if (programs != null) { - programObservationCache.populate(programs.stream().map(Program::getId).collect(Collectors.toList())); + programCache.populate(programs.stream().map(Program::getId).collect(Collectors.toList())); } } @@ -163,7 +166,7 @@ private void processObservations(String programKey, List obser * Get all observations for a program from the cache. */ private Map getProgramObservations(UUID programId) throws ApiException { - return programObservationCache.get(programId); + return programCache.get(programId); } // Note: not using cache, because unique studyName (with "[ProgramKey-ExtraInfo]") is not stored directly on Observation. @@ -244,6 +247,42 @@ public List getObservationsByObservationUnitsAndStudies(Collec .collect(Collectors.toList()); } + // TODO: implement other filters in BI-2506. + public List getObservationsByFilters(Program program, String studyDbId) throws ApiException, DoesNotExistException { + + String studySource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.STUDIES); + String observationUnitSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.OBSERVATION_UNITS); + String observationSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.OBSERVATIONS); + + // Get all observations for the program. + Collection observations = getProgramObservations(program.getId()).values(); + // Build a hashmap of traits for fast lookup. The key is ObservationVariableDbId, the value is the Trait Id. + HashMap traitIdsByObservationVariableDbId = traitService.getIdsByObservationVariableDbIds(program.getId(), observations.stream().map(BrAPIObservation::getObservationVariableDbId).collect(Collectors.toList())); + + // Lookup studyDbId. + return observations.stream() + .filter(o -> { + // Short circuit if filter is null. + if (studyDbId == null) return true; + Optional xref = Utilities.getExternalReference(o.getExternalReferences(), studySource); + return xref.filter(brAPIExternalReference -> studyDbId.equals(brAPIExternalReference.getReferenceId())).isPresent(); + }) + .peek(o -> { + // Translate ObservationVariableDbId. + o.setObservationVariableDbId(traitIdsByObservationVariableDbId.get(o.getObservationVariableDbId())); + // Translate ObservationUnitDbId. + o.setObservationUnitDbId(Utilities.getExternalReference(o.getExternalReferences(), observationUnitSource) + .orElseThrow(() -> new RuntimeException("observationUnit xref not found on observation")).getReferenceId()); + // Translate ObservationId. + o.setObservationDbId(Utilities.getExternalReference(o.getExternalReferences(), observationSource) + .orElseThrow(() -> new RuntimeException("observation xref not found on observation")).getReferenceId()); + // Translate StudyDbId. + o.setStudyDbId(Utilities.getExternalReference(o.getExternalReferences(), studySource) + .orElseThrow(() -> new RuntimeException("study xref not found on observation")).getReferenceId()); + // TODO: consider translating germplasmDbId in BI-2506. + }).collect(Collectors.toList()); + } + @NotNull private ApiResponse, Optional>> searchObservationsSearchResultsDbIdGet(UUID programId, String searchResultsDbId, Integer page, Integer pageSize) throws ApiException { ObservationsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationsApi.class); @@ -259,7 +298,7 @@ public List createBrAPIObservations(List brA List postResponse = brAPIDAOUtil.post(brAPIObservationList, upload, api::observationsPost, importDAO::update); return processObservationsForCache(postResponse, program.getKey()); }; - return programObservationCache.post(programId, postFunction); + return programCache.post(programId, postFunction); } return new ArrayList<>(); } catch (Exception e) { @@ -287,7 +326,7 @@ public BrAPIObservation updateBrAPIObservation(String dbId, BrAPIObservation obs } return processObservationsForCache(List.of(updatedObservation), program.getKey()); }; - return programObservationCache.post(programId, postFunction).get(0); + return programCache.post(programId, postFunction).get(0); } catch (ApiException e) { log.error(Utilities.generateApiExceptionLogMessage(e)); throw new InternalServerException("Unknown error has occurred: " + e.getMessage(), e); @@ -343,7 +382,7 @@ public List updateBrAPIObservation(Map processedObservations = processObservationsForCache(updatedObservations, program.getKey()); - return programObservationCache.postThese(programId,processedObservations); + return programCache.postThese(programId,processedObservations); } catch (ApiException e) { log.error("Error updating observation: " + Utilities.generateApiExceptionLogMessage(e), e); throw e; diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java index aeb919a2f..9749bf093 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java @@ -41,7 +41,6 @@ import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.daos.ProgramDAO; -import org.breedinginsight.daos.cache.ProgramCache; import org.breedinginsight.daos.cache.ProgramCacheProvider; import org.breedinginsight.model.Program; import org.breedinginsight.services.ProgramService; @@ -61,7 +60,7 @@ @Slf4j @Singleton -public class BrAPIObservationUnitDAO { +public class BrAPIObservationUnitDAO extends BrAPICachedDAO { private final ProgramDAO programDAO; private final ImportDAO importDAO; private final BrAPIDAOUtil brAPIDAOUtil; @@ -75,8 +74,6 @@ public class BrAPIObservationUnitDAO { private final Gson gson = new JSON().getGson(); private final Type treatmentlistType = new TypeToken>(){}.getType(); - private final ProgramCache programObservationUnitCache; - @Inject public BrAPIObservationUnitDAO(ProgramDAO programDAO, ImportDAO importDAO, @@ -95,10 +92,10 @@ public BrAPIObservationUnitDAO(ProgramDAO programDAO, this.runScheduledTasks = runScheduledTasks; this.programService = programService; this.germplasmService = germplasmService; - this.programObservationUnitCache = programCacheProvider.getProgramCache(this::fetchProgramObservationUnits, BrAPIObservationUnit.class); + this.programCache = programCacheProvider.getProgramCache(this::fetchProgramObservationUnits, BrAPIObservationUnit.class); } - @Scheduled(initialDelay = "3s") + @Scheduled(initialDelay = "${startup.delay.observation_unit}") public void setup() { if(!runScheduledTasks) { return; @@ -107,7 +104,7 @@ public void setup() { log.debug("populating observation unit cache"); List programs = programDAO.getActive(); if(programs != null) { - programObservationUnitCache.populate(programs.stream().map(Program::getId).collect(Collectors.toList())); + programCache.populate(programs.stream().map(Program::getId).collect(Collectors.toList())); } } @@ -157,7 +154,7 @@ private Map processObservationUnitsForCache(List getProgramObservationUnits(UUID programId) throws ApiException { - return programObservationUnitCache.get(programId); + return programCache.get(programId); } public List getObservationUnitByName(List observationUnitNames, Program program) throws ApiException { @@ -183,7 +180,7 @@ public List createBrAPIObservationUnits(List ous = brAPIDAOUtil.post(brAPIObservationUnitList, upload, api::observationunitsPost, importDAO::update); return processObservationUnitsForCache(ous, program, false); }; - return programObservationUnitCache.post(programId, postFunction); + return programCache.post(programId, postFunction); } return new ArrayList<>(); } catch (Exception e) { @@ -205,7 +202,7 @@ public List createBrAPIObservationUnits(List ous = brAPIDAOUtil.post(brAPIObservationUnitList, api::observationunitsPost); return processObservationUnitsForCache(ous, program, false); }; - return programObservationUnitCache.post(programId, postFunction); + return programCache.post(programId, postFunction); } return new ArrayList<>(); } catch (Exception e) { diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIStudyDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIStudyDAO.java index fdd404ae3..217d10216 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIStudyDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIStudyDAO.java @@ -30,7 +30,6 @@ import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.daos.ProgramDAO; -import org.breedinginsight.daos.cache.ProgramCache; import org.breedinginsight.daos.cache.ProgramCacheProvider; import org.breedinginsight.model.Program; import org.breedinginsight.services.brapi.BrAPIEndpointProvider; @@ -46,7 +45,7 @@ @Slf4j @Singleton -public class BrAPIStudyDAO { +public class BrAPIStudyDAO extends BrAPICachedDAO { @Property(name = "brapi.server.reference-source") private String referenceSource; @Property(name = "micronaut.bi.api.run-scheduled-tasks") @@ -56,8 +55,6 @@ public class BrAPIStudyDAO { private ImportDAO importDAO; private final BrAPIDAOUtil brAPIDAOUtil; private final BrAPIEndpointProvider brAPIEndpointProvider; - private final ProgramCache programStudyCache; - @Inject public BrAPIStudyDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil brAPIDAOUtil, BrAPIEndpointProvider brAPIEndpointProvider, ProgramCacheProvider programCacheProvider) { @@ -65,10 +62,10 @@ public BrAPIStudyDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil br this.importDAO = importDAO; this.brAPIDAOUtil = brAPIDAOUtil; this.brAPIEndpointProvider = brAPIEndpointProvider; - this.programStudyCache = programCacheProvider.getProgramCache(this::fetchProgramStudy, BrAPIStudy.class); + this.programCache = programCacheProvider.getProgramCache(this::fetchProgramStudy, BrAPIStudy.class); } - @Scheduled(initialDelay = "2s") + @Scheduled(initialDelay = "${startup.delay.study}") public void setup() { if(!runScheduledTasks) { return; @@ -77,7 +74,7 @@ public void setup() { log.debug("populating study cache"); List programs = programDAO.getActive(); if(programs != null) { - programStudyCache.populate(programs.stream().map(Program::getId).collect(Collectors.toList())); + programCache.populate(programs.stream().map(Program::getId).collect(Collectors.toList())); } } @@ -115,7 +112,7 @@ private Map fetchProgramStudy(UUID programId) throws ApiExce * @throws ApiException */ public List getStudies(UUID programId) throws ApiException { - return new ArrayList<>(programStudyCache.get(programId).values()); + return new ArrayList<>(programCache.get(programId).values()); } public Optional getStudyByName(String studyName, Program program) throws ApiException { @@ -153,7 +150,7 @@ public List getStudiesByExperimentID(@NotNull UUID experimentId, Pro } public List getStudiesByEnvironmentIds(@NotNull Collection environmentIds, Program program) throws ApiException { - return programStudyCache.get(program.getId()) + return programCache.get(program.getId()) .entrySet() .stream() .filter(entry -> environmentIds.contains(UUID.fromString(entry.getKey()))) @@ -193,7 +190,7 @@ public List createBrAPIStudies(List brAPIStudyList, UUID .post(brAPIStudyList, upload, api::studiesPost, importDAO::update); return environmentById(postedStudies); }; - createdStudies.addAll(programStudyCache.post(programId, postCallback)); + createdStudies.addAll(programCache.post(programId, postCallback)); } return createdStudies; diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPITrialDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPITrialDAO.java index 83ee3f47b..85298c158 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPITrialDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPITrialDAO.java @@ -45,4 +45,8 @@ List createBrAPITrials(List brAPITrialList, UUID program List getTrialsByDbIds(Collection trialDbIds, Program program) throws ApiException; List getTrialsByExperimentIds(Collection experimentIds, Program program) throws ApiException; + + void deleteBrAPITrial(Program program, BrAPITrial trial, boolean hard) throws ApiException; + + void repopulateCache(UUID programId); } diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/impl/BrAPITrialDAOImpl.java b/src/main/java/org/breedinginsight/brapi/v2/dao/impl/BrAPITrialDAOImpl.java index 066c7017d..b0c771f6d 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/impl/BrAPITrialDAOImpl.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/impl/BrAPITrialDAOImpl.java @@ -21,6 +21,8 @@ import io.micronaut.http.server.exceptions.InternalServerException; import io.micronaut.scheduling.annotation.Scheduled; import lombok.extern.slf4j.Slf4j; +import okhttp3.HttpUrl; +import okhttp3.Request; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.modules.core.TrialsApi; import org.brapi.v2.model.BrAPIExternalReference; @@ -79,7 +81,7 @@ public BrAPITrialDAOImpl(ProgramCacheProvider programCacheProvider, } - @Scheduled(initialDelay = "2s") + @Scheduled(initialDelay = "${startup.delay.trial}") public void setup() { if(!runScheduledTasks) { return; @@ -277,4 +279,25 @@ public List getTrialsByExperimentIds(Collection experimentIds, trialSearch ), program.getKey()); } + + @Override + public void deleteBrAPITrial(Program program, BrAPITrial trial, boolean hard) throws ApiException { + // TODO: Switch to using the TrialsApi from the BrAPI client library once the delete endpoints are merged into it. + var programBrAPIBaseUrl = brAPIDAOUtil.getProgramBrAPIBaseUrl(program.getId()); + var requestUrl = HttpUrl.parse(programBrAPIBaseUrl + "/trials/" + trial.getTrialDbId()).newBuilder(); + requestUrl.addQueryParameter("hardDelete", Boolean.toString(hard)); + HttpUrl url = requestUrl.build(); + var brapiRequest = new Request.Builder().url(url) + .method("DELETE", null) + .addHeader("Content-Type", "application/json") + .build(); + + brAPIDAOUtil.makeCall(brapiRequest); + } + + @Override + public void repopulateCache(UUID programId) { + this.programExperimentCache.invalidate(programId); + this.programExperimentCache.populate(programId); + } } diff --git a/src/main/java/org/breedinginsight/brapi/v2/model/request/query/GermplasmQuery.java b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/GermplasmQuery.java index 88f6d142c..81039b7ef 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/model/request/query/GermplasmQuery.java +++ b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/GermplasmQuery.java @@ -27,6 +27,9 @@ public class GermplasmQuery extends BrapiQuery { // This is a meta-parameter, it describes the display format of any date fields. private String dateDisplayFormat; + // The list id used to get a collection of germplasm + private String listDbId; + public SearchRequest constructSearchRequest() { List filters = new ArrayList<>(); if (!StringUtils.isBlank(getImportEntryNumber())) { diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java index 8023327d7..7e4a01e67 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java @@ -82,7 +82,7 @@ public Optional getGermplasmByDBID(UUID programId, String germpl return germplasmDAO.getGermplasmByDBID(germplasmId, programId); } - public List> processListData(List germplasm, BrAPIListDetails germplasmList, Program program){ + public List> processListData(List germplasm, List listData, Program program){ Map germplasmByName = new HashMap<>(); for (BrAPIGermplasm g: germplasm) { // Use the full, unique germplasmName with programKey and accessionNumber (GID) for 2 reasons: @@ -97,14 +97,14 @@ public List> processListData(List germplasm, // This holds the BrAPI list items or all germplasm in a program if the list is null. List orderedGermplasmNames = new ArrayList<>(); - if (germplasmList == null) { + if (listData == null) { orderedGermplasmNames = germplasm.stream().sorted((left, right) -> { Integer leftAccessionNumber = Integer.parseInt(left.getAccessionNumber()); Integer rightAccessionNumber = Integer.parseInt(right.getAccessionNumber()); return leftAccessionNumber.compareTo(rightAccessionNumber); }).map(BrAPIGermplasm::getGermplasmName).collect(Collectors.toList()); } else { - orderedGermplasmNames = germplasmList.getData(); + orderedGermplasmNames = listData; } // For export, assign entry number sequentially based on BrAPI list order. @@ -124,7 +124,7 @@ public List> processListData(List germplasm, row.put("Source", source); // Use the entry number in the list map if generated - if(germplasmList == null) { + if(listData == null) { // Not downloading a real list, use GID (https://breedinginsight.atlassian.net/browse/BI-2266). row.put("Entry No", Integer.valueOf(germplasmEntry.getAccessionNumber())); } else { @@ -187,26 +187,24 @@ public List getGermplasmByList(UUID programId, String listDbId) // get list BrAPI germplasm variables List germplasmNames = listResponse.getResult().getData(); List germplasm = germplasmDAO.getGermplasmByRawName(germplasmNames, programId); - Map germplasmByName = new HashMap<>(); + Map germplasmByGid = new HashMap<>(); for (BrAPIGermplasm g : germplasm) { // set the list ID in the germplasm additional info - germplasm.forEach(x -> x.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_LIST_ID, listId)); + g.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_LIST_ID, listId); // Add to map. - germplasmByName.put(g.getGermplasmName(), g); + germplasmByGid.put(g.getAccessionNumber(), g); } - // Get the program key. - String programKey = programService.getById(programId) - .orElseThrow(ApiException::new) - .getKey(); + // Extract gids from list names + List gids = germplasmNames.stream().map(Utilities::extractGid).collect(Collectors.toList()); // Build list from BrAPI list that preserves ordering and duplicates and assigns sequential entry numbers. List germplasmList = new ArrayList<>(); int entryNumber = 0; - for (String germplasmName : germplasmNames) { + for (String gid : gids) { ++entryNumber; - BrAPIGermplasm listEntry = cloneBrAPIGermplasm(germplasmByName.get(Utilities.removeProgramKeyAndUnknownAdditionalData(germplasmName, programKey))); + BrAPIGermplasm listEntry = cloneBrAPIGermplasm(germplasmByGid.get(gid)); // Set entry number. listEntry.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_IMPORT_ENTRY_NUMBER, entryNumber); germplasmList.add(listEntry); @@ -254,7 +252,9 @@ public DownloadFile exportGermplasm(UUID programId, FileType fileExtension) thro return new DownloadFile(fileName, downloadFile); } - public DownloadFile exportGermplasmList(UUID programId, String listId, FileType fileExtension) throws IllegalArgumentException, ApiException, IOException, DoesNotExistException { + public DownloadFile exportGermplasmList(UUID programId, + String listId, + FileType fileExtension) throws IllegalArgumentException, ApiException, IOException, DoesNotExistException { List columns = GermplasmFileColumns.getOrderedColumns(); //Retrieve germplasm list data @@ -270,7 +270,7 @@ public DownloadFile exportGermplasmList(UUID programId, String listId, FileType String fileName = createFileName(listData, listName); StreamedFile downloadFile; //Convert list data to List> data to pass into file writer - List> processedData = processListData(germplasm, listData, program); + List> processedData = processListData(germplasm, germplasmNames, program); if (fileExtension == FileType.CSV){ downloadFile = CSVWriter.writeToDownload(columns, processedData, fileExtension); diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIListService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIListService.java index 3353ac17c..2bfeccb22 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIListService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIListService.java @@ -1,6 +1,9 @@ package org.breedinginsight.brapi.v2.services; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; import io.micronaut.context.annotation.Property; +import io.micronaut.http.HttpResponse; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIExternalReference; @@ -8,6 +11,7 @@ import org.brapi.v2.model.core.BrAPIListTypes; import org.brapi.v2.model.core.request.BrAPIListSearchRequest; import org.brapi.v2.model.core.response.BrAPIListsSingleResponse; +import org.brapi.v2.model.germ.BrAPIGermplasm; import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; import org.breedinginsight.brapi.v2.dao.BrAPIListDAO; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; @@ -17,8 +21,7 @@ import javax.inject.Inject; import javax.inject.Singleton; -import java.util.List; -import java.util.Optional; +import java.util.*; import java.util.stream.Collectors; @Slf4j @@ -53,7 +56,7 @@ public List getListSummariesByTypeAndXref( if (xrefId != null && !xrefId.isEmpty()) { searchRequest.externalReferenceIDs(List.of(xrefId)); } - List lists = listDAO.getListBySearch(searchRequest, program.getId()); + List lists = listDAO.getListsBySearch(searchRequest, program.getId()); if (lists == null) { throw new DoesNotExistException("list not returned from BrAPI service"); } @@ -62,30 +65,43 @@ public List getListSummariesByTypeAndXref( Optional programXrefOptional = Utilities.getExternalReference(list.getExternalReferences(),Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS)); return programXrefOptional.isPresent() && programXrefOptional.get().getReferenceID().equals(program.getId().toString()); }).collect(Collectors.toList()); - for (BrAPIListSummary list: programLists) { + // Map of pairs. + HashMap itemsFromEachList = new HashMap<>(); + for (BrAPIListSummary list: programLists) { // remove the program key from the list name list.setListName(Utilities.removeProgramKeyAndUnknownAdditionalData(list.getListName(), program.getKey())); - // set the owner of the list items as the list owner BrAPIListsSingleResponse listDetails = listDAO.getListById(list.getListDbId(), program.getId()); - List listItemNames = listDetails.getResult().getData(); - if (type != null) { - switch (type) { - case GERMPLASM: - String createdBy = germplasmDAO.getGermplasmByRawName(listItemNames, program.getId()).get(0) + // Add first item from list to hashmap. + itemsFromEachList.put(list.getListDbId(), listDetails.getResult().getData().get(0)); + } + if (type == BrAPIListTypes.GERMPLASM) { + // Fetch one germplasm for each list from cache. + List germplasmRepresentatives = germplasmDAO.getGermplasmByRawName((new ArrayList<>(itemsFromEachList.values())), program.getId()); + // Build hashmap of germplasm by name. + HashMap germplasmByName = new HashMap<>(); + for (BrAPIGermplasm germplasm: germplasmRepresentatives) { + germplasmByName.put(germplasm.getGermplasmName(), germplasm); + } + // For each list, set list owner name from createdBy stored in germplasm additional info. + for (BrAPIListSummary list: programLists) { + String strippedName = Utilities.removeProgramKeyAnyAccession(itemsFromEachList.get(list.getListDbId()), program.getKey()); + + list.setListOwnerName( + germplasmByName.get(strippedName) .getAdditionalInfo() .getAsJsonObject("createdBy") .get("userName") - .getAsString(); - list.setListOwnerName(createdBy); - case OBSERVATIONVARIABLES: - default: - break; - } + .getAsString() + ); } } return programLists; } + + public HttpResponse deleteBrAPIList(String listDbId, UUID programId, boolean hardDelete) throws ApiException { + return listDAO.deleteBrAPIList(listDbId, programId, hardDelete); + } } diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 7c37f8f55..6b8a3f233 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -638,7 +638,7 @@ private void addObsVarDataToRow( } public List getDatasetObsVars(String datasetId, Program program) throws ApiException, DoesNotExistException { - List lists = listDAO.getListByTypeAndExternalRef( + List lists = listDAO.getListsByTypeAndExternalRef( BrAPIListTypes.OBSERVATIONVARIABLES, program.getId(), String.format("%s/%s", referenceSource, ExternalReferenceSource.DATASET.getName()), @@ -667,6 +667,41 @@ public BrAPITrial getExperiment(Program program, UUID experimentId) throws ApiEx return experiments.get(0); } + public int deleteExperiment(Program program, UUID experimentId, boolean hard) throws ApiException { + List trials = trialDAO.getTrialsByExperimentIds(List.of(experimentId), program); + if (trials.isEmpty()) { + throw new ApiException(404, "Experiment with UUID " + experimentId + " not found"); + } + BrAPITrial trial = trials.get(0); + + List existingObservations = observationDAO.getObservationsByTrialDbId(List.of(trial.getTrialDbId()), program); + // If there are no observations or a soft delete is requested, proceed. + if (existingObservations.isEmpty() || !hard) { + // Make request to delete experiment. + trialDAO.deleteBrAPITrial(program, trial, hard); + // Get all lists for the trial. + List lists = listDAO + .getListsByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, + program.getId(), + Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.TRIALS), + experimentId); + // TODO: replace with a single call to a batch delete method if that becomes available. + // Iterate over lists, delete each by listDbId. + for (BrAPIListSummary list : lists) { + listDAO.deleteBrAPIList(list.getListDbId(), program.getId(), hard); + } + // TODO: if performance is poor, implement more precise invalidation, possibly using hierarchical cache keys. + // Invalidate and repopulate cache for Trial, Study, Observation, ObservationUnit. + trialDAO.repopulateCache(program.getId()); + studyDAO.repopulateCache(program.getId()); + observationDAO.repopulateCache(program.getId()); + observationUnitDAO.repopulateCache(program.getId()); + } + + // Successful or not, return the number of observations in this experiment. + return existingObservations.size(); + } + private Map createExportRow( BrAPITrial experiment, Program program, diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPISampleDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPISampleDAO.java index 1cb9a531a..372bb63be 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPISampleDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPISampleDAO.java @@ -17,8 +17,15 @@ package org.breedinginsight.brapps.importer.daos; +import com.google.gson.Gson; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import io.micronaut.context.annotation.Property; +import io.micronaut.http.server.exceptions.InternalServerException; import lombok.extern.slf4j.Slf4j; +import okhttp3.*; +import org.brapi.client.v2.JSON; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.modules.genotype.SamplesApi; import org.brapi.v2.model.geno.BrAPISample; @@ -33,9 +40,9 @@ import javax.inject.Inject; import javax.inject.Singleton; +import java.io.IOException; import java.util.Collections; import java.util.List; -import java.util.UUID; @Slf4j @Singleton @@ -47,6 +54,7 @@ public class BrAPISampleDAO { private final ImportDAO importDAO; private final BrAPIDAOUtil brAPIDAOUtil; private final BrAPIEndpointProvider brAPIEndpointProvider; + private final Gson gson = new JSON().getGson(); @Inject public BrAPISampleDAO(ProgramDAO programDAO, @@ -102,4 +110,149 @@ public List readSamplesBySubmissionIds(Program program, List sampleDbIds) throws ApiException { + // create batch of samples, not yet included in brapi client TODO: switch to brapi client when available + String programBrAPIBaseUrl = brAPIDAOUtil.getProgramBrAPIBaseUrl(program.getId()); + String batchDbId = postSamplesBatch(programBrAPIBaseUrl, sampleDbIds); + + // delete samples specified in batch + deleteBatch(programBrAPIBaseUrl, batchDbId); + } + + /** + * Deletes all plates specified in the brapi server + * @param program + * @param plateDbIds + * @throws ApiException + */ + public void deletePlates(Program program, List plateDbIds) throws ApiException { + // create batch of plates, not yet included in brapi client TODO: switch to brapi client when available + String programBrAPIBaseUrl = brAPIDAOUtil.getProgramBrAPIBaseUrl(program.getId()); + String batchDbId = postPlatesBatch(programBrAPIBaseUrl, plateDbIds); + + // delete plates specified in batch + deleteBatch(programBrAPIBaseUrl, batchDbId); + } + + + private String postSamplesBatch(String programBrAPIBaseUrl, List sampleDbIds) throws ApiException { + HttpUrl.Builder requestUrl = HttpUrl.parse(programBrAPIBaseUrl + "/batchDeletes").newBuilder(); + SampleBatchDeleteRequest requestBody = new SampleBatchDeleteRequest(sampleDbIds); + String json = gson.toJson(requestBody); + RequestBody body = RequestBody.create(json, MediaType.get("application/json")); + HttpUrl url = requestUrl.build(); + return postBatch(url, body, programBrAPIBaseUrl); + } + + private String postPlatesBatch(String programBrAPIBaseUrl, List plateDbIds) throws ApiException { + HttpUrl.Builder requestUrl = HttpUrl.parse(programBrAPIBaseUrl + "/batchDeletes").newBuilder(); + PlateBatchDeleteRequest requestBody = new PlateBatchDeleteRequest(plateDbIds); + String json = gson.toJson(requestBody); + RequestBody body = RequestBody.create(json, MediaType.get("application/json")); + HttpUrl url = requestUrl.build(); + return postBatch(url, body, programBrAPIBaseUrl); + } + + private String postBatch(HttpUrl url, RequestBody body, String programBrAPIBaseUrl) throws ApiException { + + Request brapiRequest = new Request.Builder() + .url(url) + .post(body) + .addHeader("Content-Type", "application/json") + .build(); + + String jsonResponse = brAPIDAOUtil.makeCallWithResponse(brapiRequest); + JsonElement rootElement = JsonParser.parseString(jsonResponse); + JsonObject rootObject = rootElement.getAsJsonObject(); + JsonObject resultObject = rootObject.getAsJsonObject("result"); + + // check to see if immediate response or searchResultId + if(resultObject.has("batchDeleteDbId")) { + return resultObject.get("batchDeleteDbId").getAsString(); + } else if (resultObject.has("searchResultsDbId")) { + // TODO: once api stuff is in client use BrAPIDAOUtil::search to handle retries, for now just request once + // brapi server only returns immediate response for batchDeletes so this case won't happen + return getBatchDeleteDbIdFromSearchResult(programBrAPIBaseUrl, resultObject.get("searchResultsDbId").getAsString()); + } else { + throw new InternalServerException("Expected batchDeleteDbId or searchResultsDbId but got " + resultObject); + } + } + + private String getBatchDeleteDbIdFromSearchResult(String programBrAPIBaseUrl, String searchResultDbId) throws ApiException { + HttpUrl.Builder requestUrl = HttpUrl.parse(programBrAPIBaseUrl + "/search/batchDeletes/" + searchResultDbId).newBuilder(); + + HttpUrl url = requestUrl.build(); + Request brapiRequest = new Request.Builder() + .url(url) + .method("GET", null) + .addHeader("Content-Type", "application/json") + .build(); + + String jsonResponse = brAPIDAOUtil.makeCallWithResponse(brapiRequest); + JsonElement rootElement = JsonParser.parseString(jsonResponse); + JsonObject rootObject = rootElement.getAsJsonObject(); + JsonObject resultObject = rootObject.getAsJsonObject("result"); + return resultObject.get("batchDeleteDbId").getAsString(); + } + + private void deleteBatch(String programBrAPIBaseUrl, String batchDbId) throws ApiException { + HttpUrl.Builder requestUrl = HttpUrl.parse(programBrAPIBaseUrl + "/batchDeletes/" + batchDbId).newBuilder(); + requestUrl.addQueryParameter("hardDelete", "true"); + + HttpUrl url = requestUrl.build(); + Request brapiRequest = new Request.Builder() + .url(url) + .method("DELETE", null) + .addHeader("Content-Type", "application/json") + .build(); + + brAPIDAOUtil.makeCall(brapiRequest); + } + + /** + * TODO: temporary minimal model here until brapi client is updated with delete models + */ + public class SampleBatchDeleteRequest { + private String batchDeleteType; + private Search search; + + public SampleBatchDeleteRequest(List sampleDbIds) { + this.batchDeleteType = "samples"; + this.search = new Search(sampleDbIds); + } + + private class Search { + private List sampleDbIds; + + public Search(List sampleDbIds) { + this.sampleDbIds = sampleDbIds; + } + } + } + + public class PlateBatchDeleteRequest { + private String batchDeleteType; + private Search search; + + public PlateBatchDeleteRequest(List plateDbIds) { + this.batchDeleteType = "plates"; + this.search = new Search(plateDbIds); + } + + private class Search { + private List plateDbIds; + + public Search(List plateDbIds) { + this.plateDbIds = plateDbIds; + } + } + } + } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java index 0caebe65e..64d4c3b23 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java @@ -18,18 +18,13 @@ package org.breedinginsight.brapps.importer.model.imports.germplasm; import lombok.extern.slf4j.Slf4j; -import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; -import org.breedinginsight.brapps.importer.services.processors.GermplasmProcessor; +import org.breedinginsight.brapps.importer.services.processors.germplasm.GermplasmProcessor; import org.breedinginsight.brapps.importer.services.processors.Processor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; -import org.breedinginsight.model.Program; -import org.breedinginsight.model.User; -import tech.tablesaw.api.Table; import javax.inject.Inject; import javax.inject.Provider; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 3c3c6a093..039c0b58c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -449,7 +449,7 @@ private void processFile(String workflowId, List finalBrAPIImportLi progress.setMessage(e.getMessage()); progress.setUpdatedBy(actingUser.getId()); importDAO.update(upload); - }catch (ValidatorException e) { + } catch (ValidatorException e) { log.info("Validation errors: \n" + e); ImportProgress progress = upload.getProgress(); progress.setStatuscode((short) HttpStatus.UNPROCESSABLE_ENTITY.getCode()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 34030b795..0666870c6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -2091,7 +2091,7 @@ private Map> initializeObsVarDatas try { List existingDatasets = brAPIListDAO - .getListByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, + .getListsByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, program.getId(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName()), UUID.fromString(datasetId)); @@ -2121,7 +2121,7 @@ private Map> initializeObsVarDatas ).getId().toString(); try { List existingDatasets = brAPIListDAO - .getListByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, + .getListsByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, program.getId(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName()), UUID.fromString(datasetId)); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index ae015aab6..4c79478e8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -44,6 +44,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessedPhenotypeData; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.EntityNotFoundException; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; import org.breedinginsight.model.Program; import org.breedinginsight.model.Scale; @@ -70,8 +71,7 @@ public class ExperimentUtilities { public static final String PREEXISTING_EXPERIMENT_TITLE = "Experiment Title already exists"; public static final String MISSING_OBS_UNIT_ID_ERROR = "Experimental entities are missing ObsUnitIDs"; public static final String UNMATCHED_COLUMN = "Ontology term(s) not found: "; - - + public static final String INVALID_OBS_UNIT_ID_ERROR = "Invalid ObsUnitID"; Gson gson; @@ -294,44 +294,104 @@ public static void addYearToStudyAdditionalInfo(Program program, BrAPIStudy stud } /** - * This method is responsible for collating unique ObsUnit IDs from the provided context data. + * Collates unique Observation Unit IDs from the import context. + * + * This method iterates through all import rows in the given context and + * extracts unique Observation Unit IDs (ObsUnit IDs) that are not null or blank. + * + * @param context The AppendOverwriteMiddlewareContext containing the import data. + * @return A Set of String containing all unique, non-null, non-blank Observation Unit IDs. * - * @param context the AppendOverwriteMiddlewareContext containing the import rows to process - * @return a Set of unique ObsUnit IDs collated from the import rows - * @throws IllegalStateException if any ObsUnit ID is repeated in the import rows - * @throws HttpStatusException if there is a mix of ObsUnit IDs for some but not all rows + * @implNote The method performs the following steps: + * 1. Initializes an empty HashSet to store unique ObsUnit IDs. + * 2. Iterates through each import row in the context. + * 3. For each row, checks if the ObsUnit ID is not null and not blank. + * 4. If valid, adds the ObsUnit ID to the set. + * 5. Returns the set of unique ObsUnit IDs. */ - public static Set collateReferenceOUIds(AppendOverwriteMiddlewareContext context) throws HttpStatusException, IllegalStateException { + public static Set collateUniqueOUIds(AppendOverwriteMiddlewareContext context) { // Initialize variables to track the presence of ObsUnit IDs Set referenceOUIds = new HashSet<>(); - boolean hasNoReferenceUnitIds = true; - boolean hasAllReferenceUnitIds = true; + + // Iterate through the import rows to process ObsUnit IDs + for (int rowNum = 0; rowNum < context.getImportContext().getImportRows().size(); rowNum++) { + ExperimentObservation importRow = (ExperimentObservation) context.getImportContext().getImportRows().get(rowNum); + if (importRow.getObsUnitID() != null && !importRow.getObsUnitID().isBlank()) { + referenceOUIds.add(importRow.getObsUnitID()); + } + } + return referenceOUIds; + } + + /** + * Validates Observation Unit ID values in the import context. + * + * This method checks each import row for the validity of its Observation Unit ID (ObsUnitID). + * It performs the following validations: + * 1. Checks if the ObsUnitID is null or blank. + * 2. Checks if the ObsUnitID is a duplicate within the import data. + * + * @param context The AppendOverwriteMiddlewareContext containing import data and validation error storage. + * @throws HttpStatusException If there's an HTTP-related error during the validation process. + * @throws IllegalStateException If the system encounters an unexpected state during validation. + * + * @implNote The method performs the following steps: + * 1. Retrieves the ValidationErrors object from the context. + * 2. Initializes a HashSet to track unique ObsUnitIDs. + * 3. Iterates through each import row in the context. + * 4. For each row: + * - If ObsUnitID is null or blank, adds a "missing ObsUnitID" error. + * - If ObsUnitID is already in the set (duplicate), adds a "duplicate ObsUnitID" error. + * - Otherwise, adds the ObsUnitID to the set of unique IDs. + * 5. Errors are added using the addRowError method, specifying the OBS_UNIT_ID column and appropriate error messages. + */ + public static void validateReferenceOUIdValues(AppendOverwriteMiddlewareContext context) throws HttpStatusException, IllegalStateException { + ValidationErrors validationErrors = context.getAppendOverwriteWorkflowContext().getValidationErrors(); + Set referenceOUIds = new HashSet<>(); // Iterate through the import rows to process ObsUnit IDs for (int rowNum = 0; rowNum < context.getImportContext().getImportRows().size(); rowNum++) { ExperimentObservation importRow = (ExperimentObservation) context.getImportContext().getImportRows().get(rowNum); - // Check if ObsUnitID is blank if (importRow.getObsUnitID() == null || importRow.getObsUnitID().isBlank()) { - // Set flag to indicate missing ObsUnit ID for current row - hasAllReferenceUnitIds = false; + // Check if ObsUnitID is blank + addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, ExpImportProcessConstants.ErrMessage.MISSING_OBS_UNIT_ID.getValue(), validationErrors, rowNum); } else if (referenceOUIds.contains(importRow.getObsUnitID())) { - // Throw exception if ObsUnitID is repeated - throw new IllegalStateException("ObsUnitId is repeated: " + importRow.getObsUnitID()); + // Check if ObsUnitID is repeated + addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, ExpImportProcessConstants.ErrMessage.DUPLICATE_OBS_UNIT_ID.getValue(), validationErrors, rowNum); } else { // Add ObsUnitID to referenceOUIds referenceOUIds.add(importRow.getObsUnitID()); - // Set flag to indicate presence of ObsUnit ID - hasNoReferenceUnitIds = false; } } + } - if (!hasNoReferenceUnitIds && !hasAllReferenceUnitIds) { - // Throw exception if there is a mix of ObsUnit IDs for some but not all rows - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, ExpImportProcessConstants.ErrMessage.MISSING_OBS_UNIT_ID_ERROR.getValue()); - } + /** + * Adds validation errors for observation units that were not found in the database. + * + * This method processes an EntityNotFoundException and adds corresponding validation errors + * to the context for each import row where the Observation Unit ID was not found. + * + * @param e The EntityNotFoundException containing information about missing Observation Unit IDs. + * @param context The AppendOverwriteMiddlewareContext containing import data and validation error storage. + * + * @implNote The method performs the following steps: + * 1. Retrieves the ValidationErrors object from the context. + * 2. Iterates through each import row in the context. + * 3. For each row, checks if its Observation Unit ID is in the set of missing entity IDs from the exception. + * 4. If a match is found, adds a validation error for that row, indicating an invalid Observation Unit ID. + * 5. The error is added using the addRowError method, specifying the OBS_UNIT_ID column and using a predefined error message. + */ + public static void addValidationErrorsForObsUnitsNotFound(EntityNotFoundException e, AppendOverwriteMiddlewareContext context) { + ValidationErrors validationErrors = context.getAppendOverwriteWorkflowContext().getValidationErrors(); + List errors = new ArrayList<>(); - return referenceOUIds; + for (int rowNum = 0; rowNum < context.getImportContext().getImportRows().size(); rowNum++) { + String rowObsUnitId = ((ExperimentObservation)context.getImportContext().getImportRows().get(rowNum)).getObsUnitID(); + if (e.getMissingEntityIds().contains(rowObsUnitId)) { + addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, ExperimentUtilities.INVALID_OBS_UNIT_ID_ERROR, validationErrors, rowNum); + } + } } /** diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIAction.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIAction.java index 830de5828..2702f9c7a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIAction.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIAction.java @@ -20,9 +20,11 @@ import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPIState; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.EntityNotFoundException; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.exceptions.MissingRequiredInfoException; import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.services.exceptions.ValidatorException; import java.util.Optional; @@ -42,7 +44,7 @@ public interface BrAPIAction { * @return An Optional containing the relevant BrAPI state after executing the action. * @throws ApiException if an error occurs during the execution of the action. */ - Optional> execute() throws ApiException, MissingRequiredInfoException, UnprocessableEntityException, DoesNotExistException; + Optional> execute() throws ApiException, MissingRequiredInfoException, UnprocessableEntityException, DoesNotExistException, EntityNotFoundException; /** * Get the BrAPI entity being acted on based on the provided ExpUnitMiddlewareContext. diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowReadInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowReadInitialization.java index 717a78f97..198c9c1ba 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowReadInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowReadInitialization.java @@ -23,6 +23,8 @@ import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPIState; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.EntityNotFoundException; +import org.breedinginsight.services.exceptions.ValidatorException; import org.breedinginsight.utilities.Utilities; import java.util.List; @@ -44,7 +46,7 @@ protected WorkflowReadInitialization(ExperimentImportEntity entity) { * @return an Optional containing the BrAPIState representing the completed read workflow * @throws ApiException if an error occurs during execution */ - public Optional> execute() throws ApiException { + public Optional> execute() throws ApiException, EntityNotFoundException { try { List fetchedMembers = entity.brapiRead(); entity.initializeWorkflow(fetchedMembers); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/ProcessedDataFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/ProcessedDataFactory.java index 84f8bcf75..6775e6d03 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/ProcessedDataFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/ProcessedDataFactory.java @@ -112,6 +112,10 @@ public static EmptyData emptyData(String brapiReferenceSource, return new EmptyData(brapiReferenceSource, isCommit, germplasmName, study, phenoColumnName, trialId, studyId, unitId, studyYear, observationUnit, user, program, studyService, observationService); } + public static UndefinedDataset undefinedDataset() { + return new UndefinedDataset(); + } + @Bean @Prototype public InitialData initialDataBean(String brapiReferenceSource, @@ -173,5 +177,11 @@ public EmptyData emptyDataBean(String brapiReferenceSource, Program program) { return emptyData(brapiReferenceSource, isCommit, germplasmName, study, phenoColumnName, trialId, studyId, unitId, studyYear, observationUnit, user, program, studyService, observationService); } + + @Bean + @Prototype + public UndefinedDataset undefinedDatasetBean() { + return undefinedDataset(); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/UndefinedDataset.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/UndefinedDataset.java new file mode 100644 index 000000000..d30d917ec --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/UndefinedDataset.java @@ -0,0 +1,43 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data; + +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.AppendStatistic; + +import java.util.List; +import java.util.Optional; + +public class UndefinedDataset extends VisitedObservationData { + @Override + public Optional> getValidationErrors() { + return Optional.empty(); + } + + @Override + public PendingImportObject constructPendingObservation() { + return null; + } + + @Override + public void updateTally(AppendStatistic statistic) { + + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/ExperimentImportEntity.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/ExperimentImportEntity.java index c98d9cf83..e4b2a4aa2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/ExperimentImportEntity.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/ExperimentImportEntity.java @@ -19,9 +19,11 @@ import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.EntityNotFoundException; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.exceptions.MissingRequiredInfoException; import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.services.exceptions.ValidatorException; import java.util.List; @@ -43,7 +45,7 @@ public interface ExperimentImportEntity { * @return List of fetched entities * @throws ApiException if there is an issue with the API call */ - public List brapiRead() throws ApiException; + public List brapiRead() throws ApiException, EntityNotFoundException; /** * Commit objects changed by the workflow to the BrAPI service. diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java index 6bb682ee2..f9ec45a2b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java @@ -26,6 +26,7 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.EntityNotFoundException; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteWorkflowContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; @@ -94,12 +95,12 @@ public List brapiPost(List members) * @throws ApiException if there is an issue with the API call */ @Override - public List brapiRead() throws ApiException { - // Collect deltabreed-generated exp unit ids listed in the import - Set expUnitIds = cache.getReferenceOUIds(); + public List brapiRead() throws ApiException, EntityNotFoundException { + // Collect deltabreed-generated obs unit ids listed in the import + Set obsUnitIds = cache.getReferenceOUIds(); // For each id fetch the observation unit from the brapi data store - return observationUnitService.getObservationUnitsByDbId(new HashSet<>(expUnitIds), importContext.getProgram()); + return observationUnitService.getObservationUnitsById(new HashSet<>(obsUnitIds), importContext.getProgram()); } /** diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteIDValidation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteIDValidation.java index 64f471b88..a2edbdf65 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteIDValidation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteIDValidation.java @@ -18,25 +18,70 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware; import io.micronaut.context.annotation.Prototype; -import io.micronaut.http.exceptions.HttpStatusException; import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.EntityNotFoundException; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPIReadFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowReadInitialization; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException; +import org.breedinginsight.services.exceptions.ValidatorException; + +import javax.inject.Inject; +import java.util.Optional; +import java.util.Set; @Slf4j @Prototype public class AppendOverwriteIDValidation extends AppendOverwriteMiddleware { + WorkflowReadInitialization brAPIObservationUnitReadWorkflowInitialization; + BrAPIReadFactory brAPIReadFactory; + + @Inject + public AppendOverwriteIDValidation(BrAPIReadFactory brAPIReadFactory) { + this.brAPIReadFactory = brAPIReadFactory; + } @Override public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { + brAPIObservationUnitReadWorkflowInitialization = brAPIReadFactory.observationUnitWorkflowReadInitializationBean(context); + // Initialize the validation error collection + Optional.ofNullable(context.getAppendOverwriteWorkflowContext().getValidationErrors()).orElseGet(() -> { + context.getAppendOverwriteWorkflowContext().setValidationErrors(new ValidationErrors()); + return new ValidationErrors(); + }); + ValidationErrors validationErrors = context.getAppendOverwriteWorkflowContext().getValidationErrors(); + ExperimentUtilities.validateReferenceOUIdValues(context); // Check for missing or duplicate OU ids + Set uniqueOUIds = ExperimentUtilities.collateUniqueOUIds(context); + context.getAppendOverwriteWorkflowContext().setReferenceOUIds(uniqueOUIds); try { - context.getAppendOverwriteWorkflowContext().setReferenceOUIds(ExperimentUtilities.collateReferenceOUIds(context)); - } catch (HttpStatusException | IllegalStateException e) { + brAPIObservationUnitReadWorkflowInitialization.execute(); // Fetch the obs units from the BrAPi service + if (validationErrors.hasErrors()) { + throw new ValidatorException(validationErrors); + } + return processNext(context); + } catch (EntityNotFoundException e) { + /** + * Return an error response with a list of rows where the unique OU id was not found in the BrAPI service in + * addition to rows where there are missing or duplicate OU ids + */ + ExperimentUtilities.addValidationErrorsForObsUnitsNotFound(e, context); + context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(new ValidatorException(validationErrors))); + return this.compensate(context); + } catch (ApiException | ValidatorException e) { + /** + * If OUs were fetched for all unique reference ids but some of the reference ids failed validation, + * return an error response and a list of rows with duplicate or missing ids + * + * Return an error response if there was a problem connecting to the BrAPI service + */ context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(e)); return this.compensate(context); } - return processNext(context); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java index e65d11d15..2aa889fc2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java @@ -30,14 +30,15 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.EntityNotFoundException; import org.breedinginsight.model.ProgramLocation; +import org.breedinginsight.services.exceptions.ValidatorException; import javax.inject.Inject; @Slf4j @Prototype public class WorkflowInitialization extends AppendOverwriteMiddleware { - WorkflowReadInitialization brAPIObservationUnitReadWorkflowInitialization; WorkflowReadInitialization brAPITrialReadWorkflowInitialization; WorkflowReadInitialization brAPIStudyReadWorkflowInitialization; WorkflowReadInitialization locationReadWorkflowInitialization; @@ -51,7 +52,6 @@ public WorkflowInitialization(BrAPIReadFactory brAPIReadFactory) { } @Override public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { - brAPIObservationUnitReadWorkflowInitialization = brAPIReadFactory.observationUnitWorkflowReadInitializationBean(context); brAPITrialReadWorkflowInitialization = brAPIReadFactory.trialWorkflowReadInitializationBean(context); brAPIStudyReadWorkflowInitialization = brAPIReadFactory.studyWorkflowReadInitializationBean(context); locationReadWorkflowInitialization = brAPIReadFactory.locationWorkflowReadInitializationBean(context); @@ -60,7 +60,6 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext log.debug("reading required BrAPI data from BrAPI service"); try { - brAPIObservationUnitReadWorkflowInitialization.execute(); brAPITrialReadWorkflowInitialization.execute(); brAPIStudyReadWorkflowInitialization.execute(); locationReadWorkflowInitialization.execute(); @@ -69,6 +68,9 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext } catch (ApiException e) { context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(e)); return this.compensate(context); + } catch (EntityNotFoundException e) { + // TODO: handle edge cases of missing brapi entities as needed + return this.compensate(context); } return processNext(context); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index 7ee29e51f..770576b67 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -225,6 +225,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext for (int i = 0; i < context.getImportContext().getImportRows().size(); i++) { Integer rowNum = i; ExperimentObservation row = (ExperimentObservation) context.getImportContext().getImportRows().get(rowNum); + VisitedObservationData processedData = null; // Construct the pending import for the row Optional.ofNullable(context.getImportContext().getMappedBrAPIImport()).orElseGet(() -> { @@ -233,18 +234,28 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext }); PendingImport mappedImportRow = context.getImportContext().getMappedBrAPIImport().getOrDefault(rowNum, new PendingImport()); String unitId = row.getObsUnitID(); + String studyName = context.getAppendOverwriteWorkflowContext().getPendingStudyByOUId().get(unitId).getBrAPIObject().getStudyName(); mappedImportRow.setTrial(context.getAppendOverwriteWorkflowContext().getPendingTrialByOUId().get(unitId)); mappedImportRow.setLocation(context.getAppendOverwriteWorkflowContext().getPendingLocationByOUId().get(unitId)); mappedImportRow.setStudy(context.getAppendOverwriteWorkflowContext().getPendingStudyByOUId().get(unitId)); mappedImportRow.setObservationUnit(context.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().get(unitId)); mappedImportRow.setGermplasm(context.getAppendOverwriteWorkflowContext().getPendingGermplasmByOUId().get(unitId)); + /** + * Handle the edge case where a user imports with the append/overwrite workflow for an experiment + * without a dataset defined (i.e. no observation variables headers) and the import does not + * actually have new data to append + */ + if (phenotypeCols.isEmpty()) { + processedData = processedDataFactory.undefinedDatasetBean(); + updatePreviewStatistics(processedData, context, studyName, unitId); + } + // Assemble the pending observation data for all phenotypes for (Column column : phenotypeCols) { String cellData = column.getString(rowNum); // Generate hash for looking up prior observation data - String studyName = context.getAppendOverwriteWorkflowContext().getPendingStudyByOUId().get(unitId).getBrAPIObject().getStudyName(); String unitName = context.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().get(unitId).getBrAPIObject().getObservationUnitName(); String phenoColumnName = column.name(); String observationHash = observationService.getObservationHash(unitName, phenoColumnName, studyName); @@ -266,8 +277,6 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext } - VisitedObservationData processedData = null; - // Is there prior observation data for this unit + var? if (observationByObsHash.containsKey(observationHash)) { @@ -275,9 +284,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext BrAPIObservation observation = gson.fromJson(gson.toJson(observationByObsHash.get(observationHash)), BrAPIObservation.class); // Is there a change to the prior data? - if ( - isChanged(cellData, observation, cell.timestamp) - ) { + if (isChanged(cellData, observation, cell.timestamp)) { // Is prior data protected? /** @@ -356,13 +363,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext processedData.getValidationErrors().ifPresent(errList -> errList.forEach(e -> validationErrors.addError(rowNum + 2, e))); // +2 to account for header row and excel file 1-based row index // Update import preview statistics and set in the context - processedData.updateTally(statistic); - statistic.addEnvironmentName(studyName); - // TODO: change null values to actual data - // TODO: change signature to take two args, studyName and unitName - statistic.addObservationUnitId(null); - statistic.addGid(context.getAppendOverwriteWorkflowContext().getPendingGermplasmByOUId().get(unitId).getBrAPIObject().getAccessionNumber()); - context.getAppendOverwriteWorkflowContext().setStatistic(statistic); + updatePreviewStatistics(processedData, context, studyName, unitId); // Construct a pending observation Optional> pendingProcessedData = Optional.ofNullable(processedData.constructPendingObservation()); @@ -402,4 +403,34 @@ private boolean isChanged(String cellData, BrAPIObservation observation, String } return !observationService.parseDateTime(newTimestamp).equals(observation.getObservationTimeStamp()); } + + /** + * Updates the preview statistics for processed observation data. + * + * This method updates various statistical metrics related to the processed + * observation data and stores them in the provided context. + * + * @param processedData The VisitedObservationData object containing the processed observation data. + * @param context The AppendOverwriteMiddlewareContext object where the updated statistics will be stored. + * @param studyName The name of the study associated with the observation data. + * @param unitId The identifier of the observation unit. + * + * @implNote This method performs the following operations: + * 1. Updates the tally in the processedData object. + * 2. Adds the study name to the statistics. + * 3. Adds the observation unit ID to the statistics. + * 4. Adds the germplasm ID (GID) to the statistics, retrieved from the pending germplasm data in the context. + * 5. Sets the updated statistics in the context. + */ + private void updatePreviewStatistics(VisitedObservationData processedData, + AppendOverwriteMiddlewareContext context, + String studyName, + String unitId) { + // Update import preview statistics and set in the context + processedData.updateTally(statistic); + statistic.addEnvironmentName(studyName); + statistic.addObservationUnitId(unitId); + statistic.addGid(context.getAppendOverwriteWorkflowContext().getPendingGermplasmByOUId().get(unitId).getBrAPIObject().getAccessionNumber()); + context.getAppendOverwriteWorkflowContext().setStatistic(statistic); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java index ab4233b0a..b6bca19d0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java @@ -383,7 +383,7 @@ private Map> initializeObsVarDatas try { List existingDatasets = brAPIListDAO - .getListByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, + .getListsByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, program.getId(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName()), UUID.fromString(datasetId)); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ValidatePendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ValidatePendingImportObjectsStep.java index b1391c483..ee570c238 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ValidatePendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ValidatePendingImportObjectsStep.java @@ -363,7 +363,7 @@ private void validateGeoCoordinates(ValidationErrors validationErrors, int rowNu } if (elevationBadValue) { - ExperimentUtilities.addRowError(ExperimentObservation.Columns.LONG, "Invalid Elevation value (numerals expected)", validationErrors, rowNum); + ExperimentUtilities.addRowError(ExperimentObservation.Columns.ELEVATION, "Invalid Elevation value (numerals expected)", validationErrors, rowNum); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/EntityNotFoundException.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/EntityNotFoundException.java new file mode 100644 index 000000000..c6a915833 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/EntityNotFoundException.java @@ -0,0 +1,28 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.model; + +import lombok.Getter; + +import java.util.Set; +@Getter +public class EntityNotFoundException extends Throwable { + private Set missingEntityIds; + + public EntityNotFoundException(Set missingEntityIds) { this.missingEntityIds = missingEntityIds; } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java index b4f7caf90..2c0ae2e53 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java @@ -1,12 +1,26 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.model; import com.fasterxml.jackson.annotation.JsonValue; import io.micronaut.context.annotation.Property; -import io.micronaut.context.annotation.Value; import lombok.extern.slf4j.Slf4j; -import javax.annotation.PostConstruct; - @Slf4j public class ExpImportProcessConstants { @@ -18,8 +32,11 @@ public class ExpImportProcessConstants { public enum ErrMessage { MULTIPLE_EXP_TITLES("File contains more than one Experiment Title"), - MISSING_OBS_UNIT_ID_ERROR("Required field is blank"), - PREEXISTING_EXPERIMENT_TITLE("Experiment Title already exists"); + MISSING_OBS_UNIT_ID("Invalid ObsUnitID"), + PREEXISTING_EXPERIMENT_TITLE("Experiment Title already exists"), + UNMATCHED_COLUMN("Ontology term(s) not found: "), + OBS_UNIT_NOT_FOUND("Invalid ObsUnitID"), + DUPLICATE_OBS_UNIT_ID("ObsUnitId is repeated"); private String value; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java index f85137e15..68aca9e2b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java @@ -23,21 +23,17 @@ import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.BrAPIListSummary; import org.brapi.v2.model.core.BrAPIListTypes; -import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.core.response.BrAPIListDetails; -import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.dao.BrAPIListDAO; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.model.Program; -import org.breedinginsight.model.Trait; import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; import javax.inject.Singleton; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.UUID; @@ -74,7 +70,7 @@ public Optional fetchDatasetById(String id, Program program) t // Retrieve existing dataset summaries based on program ID and external reference List existingDatasets = brAPIListDAO - .getListByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, + .getListsByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, program.getId(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName()), UUID.fromString(id)); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java index e363f9d46..d66507558 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java @@ -22,11 +22,11 @@ import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; -import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.EntityNotFoundException; import org.breedinginsight.model.Program; import org.breedinginsight.utilities.Utilities; @@ -35,8 +35,6 @@ import java.util.*; import java.util.stream.Collectors; -import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.COMMA_DELIMITER; - @Singleton public class ObservationUnitService { private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; @@ -49,33 +47,38 @@ public ObservationUnitService(BrAPIObservationUnitDAO brAPIObservationUnitDAO) { } /** - * Retrieves a list of BrAPI (Breeding API) observation units by their database IDs for a given set of experimental unit IDs and program. + * Retrieves a list of BrAPI (Breeding API) observation units by their database IDs for a given set of observation unit IDs and program. * - * This method queries the BrAPIObservationUnitDAO to retrieve BrAPI observation units based on the provided experimental unit IDs and program. - * If the database IDs of the retrieved BrAPI observation units do not match the provided experimental unit IDs, an IllegalStateException is thrown. + * This method queries the BrAPIObservationUnitDAO to retrieve BrAPI observation units based on the provided observation unit IDs and program. + * If the database IDs of the retrieved BrAPI observation units do not match the provided observation unit IDs, an IllegalStateException is thrown. * The exception includes information on the missing observation unit database IDs. * - * @param expUnitIds a set of experimental unit IDs for which to retrieve BrAPI observation units + * @param obsUnitIds a set of observation unit IDs for which to retrieve BrAPI observation units * @param program the program for which to retrieve BrAPI observation units - * @return a list of BrAPIObservationUnit objects corresponding to the provided experimental unit IDs + * @return a list of BrAPIObservationUnit objects corresponding to the provided observation unit IDs * @throws ApiException if an error occurs during the retrieval of observation units - * @throws IllegalStateException if the retrieved observation units do not match the provided experimental unit IDs + * @throws IllegalStateException if the retrieved observation units do not match the provided observation unit IDs */ - public List getObservationUnitsByDbId(Set expUnitIds, Program program) throws ApiException, IllegalStateException { + public List getObservationUnitsById(Set obsUnitIds, Program program) throws ApiException, IllegalStateException, EntityNotFoundException { List brapiUnits = null; // Retrieve reference Observation Units based on IDs - brapiUnits = brAPIObservationUnitDAO.getObservationUnitsById(expUnitIds, program); + brapiUnits = brAPIObservationUnitDAO.getObservationUnitsById(obsUnitIds, program); - // If no BrAPI units are found, throw an IllegalStateException with an error message - if (expUnitIds.size() != brapiUnits.size()) { - Set missingIds = new HashSet<>(expUnitIds); + // If no BrAPI units are found, throw an EntityNotFoundException with an error message + if (obsUnitIds.size() != brapiUnits.size()) { + Set missingIds = new HashSet<>(obsUnitIds); // Calculate missing IDs based on retrieved BrAPI units - missingIds.removeAll(brapiUnits.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toSet())); + //missingIds.removeAll(brapiUnits.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toSet())); + missingIds.removeAll(brapiUnits.stream() + .map(unit -> Utilities.getExternalReference(unit.getExternalReferences(), BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS)) + .filter(Optional::isPresent) + .map(Optional::get) + .map(BrAPIExternalReference::getReferenceId).collect(Collectors.toSet())); // Throw exception with missing IDs information - throw new IllegalStateException(ExperimentUtilities.UNMATCHED_COLUMN + String.join(COMMA_DELIMITER, missingIds)); + throw new EntityNotFoundException(missingIds); } return brapiUnits; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmImportIdUtils.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmImportIdUtils.java new file mode 100644 index 000000000..5c695aaee --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmImportIdUtils.java @@ -0,0 +1,110 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.breedinginsight.brapps.importer.services.processors.germplasm; + +import com.google.gson.JsonElement; +import org.apache.commons.lang3.StringUtils; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; + +/** + * Utility class for managing germplasm import identifiers and pedigree relationships. + */ +public class GermplasmImportIdUtils { + + private GermplasmImportIdUtils() { + // Private constructor to prevent instantiation + } + + /** + * Generates an import ID for a germplasm based on its GID or entry number. + * @param gid The germplasm ID + * @param entryNo The entry number + * @return The generated import ID or null if both parameters are null + */ + public static String generateImportId(String gid, String entryNo) { + if (gid == null && entryNo == null) return null; + return StringUtils.isNotBlank(gid) ? "GID " + gid : "ENTRY NO " + entryNo; + } + + /** + * Gets the import ID for a germplasm. + * @param germplasm The germplasm object + * @return The import ID + */ + public static String getImportId(BrAPIGermplasm germplasm) { + String gid = germplasm.getAccessionNumber(); + String entryNo = germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_IMPORT_ENTRY_NUMBER).getAsString(); + return generateImportId(gid, entryNo); + } + + /** + * Gets the import ID for the mother/female parent of a germplasm. + * @param germplasm The germplasm object + * @return The import ID of the mother/female parent + */ + public static String getMotherImportId(BrAPIGermplasm germplasm) { + JsonElement motherGidElement = germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_FEMALE_PARENT_GID); + JsonElement motherEntryNoElement = germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_FEMALE_PARENT_ENTRY_NO); + String motherGid = !motherGidElement.isJsonNull() ? motherGidElement.getAsString() : null; + String motherEntryNo = !motherEntryNoElement.isJsonNull() ? motherEntryNoElement.getAsString() : null; + return generateImportId(motherGid, motherEntryNo); + } + + /** + * Gets the import ID for the father/male parent of a germplasm. + * @param germplasm The germplasm object + * @return The import ID of the father/male parent + */ + public static String getFatherImportId(BrAPIGermplasm germplasm) { + JsonElement fatherGidElement = germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_MALE_PARENT_GID); + JsonElement fatherEntryNoElement = germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_MALE_PARENT_ENTRY_NO); + String fatherGid = !fatherGidElement.isJsonNull() ? fatherGidElement.getAsString() : null; + String fatherEntryNo = !fatherEntryNoElement.isJsonNull() ? fatherEntryNoElement.getAsString() : null; + return generateImportId(fatherGid, fatherEntryNo); + } + + /** + * Checks if a male parent is present for a germplasm. + * @param germplasm The germplasm object + * @return true if a male parent is present, false otherwise + */ + public static boolean maleParentPresent(BrAPIGermplasm germplasm) { + boolean fatherGidNull = germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_MALE_PARENT_GID).isJsonNull(); + boolean fatherEntryNoNull = germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_MALE_PARENT_ENTRY_NO).isJsonNull(); + return !fatherGidNull || !fatherEntryNoNull; + } + + /** + * Checks if the female parent is unknown for a germplasm. + * @param germplasm The germplasm object + * @return true if the female parent is unknown, false otherwise + */ + public static boolean femaleParentUnknown(BrAPIGermplasm germplasm) { + return germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.FEMALE_PARENT_UNKNOWN).getAsBoolean(); + } + + /** + * Checks if the male parent is unknown for a germplasm. + * @param germplasm The germplasm object + * @return true if the male parent is unknown, false otherwise + */ + public static boolean maleParentUnknown(BrAPIGermplasm germplasm) { + return germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.MALE_PARENT_UNKNOWN).getAsBoolean(); + } + +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java similarity index 96% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java index 25555b28e..d4d2389a8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java @@ -14,9 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.breedinginsight.brapps.importer.services.processors; +package org.breedinginsight.brapps.importer.services.processors.germplasm; import com.google.gson.Gson; +import com.google.gson.JsonElement; import io.micronaut.context.annotation.Property; import io.micronaut.context.annotation.Prototype; import io.micronaut.http.HttpStatus; @@ -42,6 +43,7 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.Processor; import org.breedinginsight.dao.db.tables.pojos.ProgramBreedingMethodEntity; import org.breedinginsight.daos.BreedingMethodDAO; import org.breedinginsight.model.Program; @@ -115,6 +117,17 @@ public GermplasmProcessor(BrAPIGermplasmService brAPIGermplasmService, DSLContex public void getExistingBrapiData(List importRows, Program program) throws ApiException { + // BI-2573 - sort by entry no here so ordering is consistent everywhere in processor + importRows.sort((left, right) -> { + if (left.getGermplasm().getEntryNo() == null || right.getGermplasm().getEntryNo() == null) { + return 0; + } else { + Integer leftEntryNo = Integer.parseInt(left.getGermplasm().getEntryNo()); + Integer rightEntryNo = Integer.parseInt(right.getGermplasm().getEntryNo()); + return leftEntryNo.compareTo(rightEntryNo); + } + }); + // Get all of our objects specified in the data file by their unique attributes Map germplasmAccessionNumbers = new HashMap<>(); for (int i = 0; i < importRows.size(); i++) { @@ -178,7 +191,7 @@ public void getExistingBrapiData(List importRows, Program program) try { Germplasm row = importRows.get(0).getGermplasm(); String listName = Germplasm.constructGermplasmListName(row.getListName(), program); - List existingLists = brAPIListDAO.getListByName(List.of(listName), program.getId()); + List existingLists = brAPIListDAO.getListsByName(List.of(listName), program.getId()); for (BrAPIListSummary existingList: existingLists) { if (existingList.getListName().equals(listName)) { listNameDup = true; @@ -265,16 +278,7 @@ public Map process(ImportUpload upload, List
entryNumberCounts = new HashMap<>(); List userProvidedEntryNumbers = new ArrayList<>(); ValidationErrors validationErrors = new ValidationErrors(); - // Sort importRows by entry number (if present). - importRows.sort((left, right) -> { - if (left.getGermplasm().getEntryNo() == null || right.getGermplasm().getEntryNo() == null) { - return 0; - } else { - Integer leftEntryNo = Integer.parseInt(left.getGermplasm().getEntryNo()); - Integer rightEntryNo = Integer.parseInt(right.getGermplasm().getEntryNo()); - return leftEntryNo.compareTo(rightEntryNo); - } - }); + for (int i = 0; i < importRows.size(); i++) { log.debug("processing germplasm row: " + (i+1)); BrAPIImport brapiImport = importRows.get(i); @@ -325,11 +329,7 @@ public Map process(ImportUpload upload, List
[ - ]) - if (commit) { - createPostOrder(); - } + createPostOrder(); // Construct our response object return getStatisticsMap(importRows); @@ -556,9 +556,14 @@ private void validatePedigree(Germplasm germplasm, Integer rowNumber, Validation } } + /* + This will set the postOrder and validate for circular pedigree dependencies. + */ private void createPostOrder() { + + Set created = null; // Construct a dependency tree for POSTing order - Set created = existingGermplasm.stream().map(BrAPIGermplasm::getGermplasmName).collect(Collectors.toSet()); + created = existingGermplasm.stream().map(GermplasmImportIdUtils::getImportId).collect(Collectors.toSet()); //todo this gets messy @@ -569,7 +574,7 @@ private void createPostOrder() { for (BrAPIGermplasm germplasm : newGermplasmList) { // If we've already planned this germplasm, skip - if (created.contains(germplasm.getGermplasmName())) { + if (created.contains(GermplasmImportIdUtils.getImportId(germplasm))) { continue; } @@ -579,20 +584,20 @@ private void createPostOrder() { continue; } - // If both parents have been created already, add it - List pedigreeArray = List.of(germplasm.getPedigree().split("/")); - String femaleParent = pedigreeArray.get(0); - String maleParent = pedigreeArray.size() > 1 ? pedigreeArray.get(1) : null; - if (created.contains(femaleParent) || germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.FEMALE_PARENT_UNKNOWN).getAsBoolean()) { - if (maleParent == null || created.contains(maleParent) || germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.MALE_PARENT_UNKNOWN).getAsBoolean()) { + String femaleImportId = GermplasmImportIdUtils.getMotherImportId(germplasm); + String maleImportId = GermplasmImportIdUtils.getFatherImportId(germplasm); + + if (created.contains(femaleImportId) || GermplasmImportIdUtils.femaleParentUnknown(germplasm)) { + if (!GermplasmImportIdUtils.maleParentPresent(germplasm) || created.contains(maleImportId) || GermplasmImportIdUtils.maleParentUnknown(germplasm)) { createList.add(germplasm); } } + } totalRecorded += createList.size(); if (createList.size() > 0) { - created.addAll(createList.stream().map(BrAPIGermplasm::getGermplasmName).collect(Collectors.toList())); + created.addAll(createList.stream().map(GermplasmImportIdUtils::getImportId).collect(Collectors.toList())); postOrder.add(createList); } else if (totalRecorded < newGermplasmList.size()) { // We ran into circular dependencies, throw an error diff --git a/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java b/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java index 983c739c2..618d1239d 100644 --- a/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java +++ b/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java @@ -109,7 +109,7 @@ public TraitDAOImpl(Configuration config, this.runScheduledTasks = runScheduledTasks; } - @Scheduled(initialDelay = "2s") + @Scheduled(initialDelay = "${startup.delay.trait}") public void setup() { if(!runScheduledTasks) { return; diff --git a/src/main/java/org/breedinginsight/db/migration/V1_32_0__Set_Dev_Admin_Email.java b/src/main/java/org/breedinginsight/db/migration/V1_32_0__Set_Dev_Admin_Email.java new file mode 100644 index 000000000..8ffcc02e7 --- /dev/null +++ b/src/main/java/org/breedinginsight/db/migration/V1_32_0__Set_Dev_Admin_Email.java @@ -0,0 +1,87 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.db.migration; + +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.daos.UserDAO; +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.flywaydb.core.api.migration.Context; +import org.jooq.DSLContext; + +import javax.inject.Inject; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.*; + +@Slf4j +public class V1_32_0__Set_Dev_Admin_Email extends BaseJavaMigration { + + @Inject + private DSLContext dsl; + @Inject + private UserDAO userDAO; + + final private String ORCID_SANDBOX_AUTHENTICATION = "orcid-sandbox-authentication"; + + public void migrate(Context context) throws Exception { + Map placeholders = context.getConfiguration().getPlaceholders(); + boolean isOrcidSandboxAuthentication = Boolean.parseBoolean( placeholders.get(ORCID_SANDBOX_AUTHENTICATION) ); + updateDevAdminUser(context, isOrcidSandboxAuthentication); + //Must delete user "Chris Tucker" before adding the new Constraint (User "Chris Tucker" is added by migration V0.5.2__populate-user-data.sql. + // It would violate the new constraint) + // PS -- We miss Chris Tucker. + deleteUserChris(context); + addConstraint(context); + } + + private void updateDevAdminUser(Context context, boolean isOrcidSandboxAuthentication) throws SQLException { + String biDevAdminUserEmail = isOrcidSandboxAuthentication ? + "bidevteam@mailinator.com" : "bidevteam@cornell.edu"; + try (Statement update = context.getConnection().createStatement()) { + String sql = "UPDATE bi_user SET email='" + biDevAdminUserEmail + "' WHERE NULLIF(email,'') IS NULL AND bi_user.name='BI-DEV Admin'"; + log.debug(sql); + update.executeUpdate(sql); + } + } + + private void deleteUserChris(Context context) throws SQLException { + try (Statement delete = context.getConnection().createStatement()) { + String sql = "DELETE FROM bi_user WHERE name = 'Chris Tucker' AND email IS NULL"; + log.debug(sql); + delete.executeUpdate(sql); + } + } + + private void addConstraint(Context context) throws SQLException { + final String CONSTRAINT_NAME = "email_orcid"; + // First, drop the constraint if it already exist. + try (Statement altTable = context.getConnection().createStatement()) { + String sql = "ALTER TABLE bi_user DROP CONSTRAINT IF EXISTS "+ CONSTRAINT_NAME; + log.debug(sql); + altTable.executeUpdate(sql); + } + + // Add new constraint + try (Statement altTable = context.getConnection().createStatement()) { + String sql = "ALTER TABLE bi_user\n" + + "ADD CONSTRAINT " +CONSTRAINT_NAME+ " CHECK ( (email IS NOT NULL ) OR (orcid IS NULL) ) ;"; + log.debug(sql); + altTable.executeUpdate(sql); + } + } +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/model/SampleSubmission.java b/src/main/java/org/breedinginsight/model/SampleSubmission.java index 6202a5a6f..020f66352 100644 --- a/src/main/java/org/breedinginsight/model/SampleSubmission.java +++ b/src/main/java/org/breedinginsight/model/SampleSubmission.java @@ -17,6 +17,7 @@ package org.breedinginsight.model; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; @@ -79,6 +80,15 @@ private void parseShipmentForms(JSONB shipmentforms) { } } + /** + * Should only be deleted when status is not submitted and has no vendor status + */ + @JsonIgnore + public boolean isDeletable() { + return (this.getSubmitted() == null || (this.getSubmitted() != null && !this.getSubmitted())) + && this.getVendorStatus() == null; + } + public enum Status { NOT_SUBMITTED("NOT SUBMITTED"), SUBMITTED("SUBMITTED"), diff --git a/src/main/java/org/breedinginsight/model/delta/DeltaEntity.java b/src/main/java/org/breedinginsight/model/delta/DeltaEntity.java index 11646f9f8..4f5b8ead9 100644 --- a/src/main/java/org/breedinginsight/model/delta/DeltaEntity.java +++ b/src/main/java/org/breedinginsight/model/delta/DeltaEntity.java @@ -7,7 +7,7 @@ public abstract class DeltaEntity { - protected final Gson gson; + protected static final Gson gson = new GsonBuilder().registerTypeAdapterFactory(new GeometryAdapterFactory()).create(); @NonNull protected final T entity; @@ -15,7 +15,6 @@ public abstract class DeltaEntity { // Note: do not use @Inject, DeltaEntity are always constructed by DeltaEntityFactory. protected DeltaEntity(@NonNull T entity) { this.entity = entity; - this.gson = new GsonBuilder().registerTypeAdapterFactory(new GeometryAdapterFactory()).create(); } protected T getEntity() { diff --git a/src/main/java/org/breedinginsight/model/delta/DeltaEntityFactory.java b/src/main/java/org/breedinginsight/model/delta/DeltaEntityFactory.java index 3787c5d71..059f91e66 100644 --- a/src/main/java/org/breedinginsight/model/delta/DeltaEntityFactory.java +++ b/src/main/java/org/breedinginsight/model/delta/DeltaEntityFactory.java @@ -4,14 +4,13 @@ import io.micronaut.context.annotation.Factory; import io.micronaut.context.annotation.Prototype; import lombok.NonNull; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; - import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.brapi.v2.model.pheno.BrAPIObservationVariable; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; import org.breedinginsight.model.ProgramLocation; import javax.inject.Inject; diff --git a/src/main/java/org/breedinginsight/services/SampleSubmissionService.java b/src/main/java/org/breedinginsight/services/SampleSubmissionService.java index 2c629da55..3309dd0dc 100644 --- a/src/main/java/org/breedinginsight/services/SampleSubmissionService.java +++ b/src/main/java/org/breedinginsight/services/SampleSubmissionService.java @@ -427,4 +427,28 @@ public Optional updateSubmissionStatus(Program program, UUID s return submissionOptional; } + + /** + * Deletes BrAPI plates and submission objects as well as sample submission record in bidb + * We do not currently cache plates or samples so don't need to worry about that + * @param submissionId sample submission UUID to delete + * @exception ApiException if a BrAPI call fails + */ + public void deleteSampleSubmission(Program program, UUID submissionId) throws ApiException { + // create a batch of sampleIds and plateIds to delete + // get samples with the sample submission xref + List samples = sampleDAO.readSamplesBySubmissionIds(program, List.of(submissionId.toString())); + + // extract sampleDbIds and plateDbIds to include in batches + List sampleDbIds = samples.stream().map(BrAPISample::getSampleDbId).distinct().collect(Collectors.toList()); + List platesDbIds = samples.stream().map(BrAPISample::getPlateDbId).distinct().collect(Collectors.toList()); + + // delete samples and plates BrAPI objects in brapi server + sampleDAO.deleteSamples(program, sampleDbIds); + sampleDAO.deletePlates(program, platesDbIds); + + // delete sample submission record from bidb + submissionDAO.deleteById(submissionId); + } + } diff --git a/src/main/java/org/breedinginsight/services/TraitService.java b/src/main/java/org/breedinginsight/services/TraitService.java index 8394acd28..cdedac3ba 100644 --- a/src/main/java/org/breedinginsight/services/TraitService.java +++ b/src/main/java/org/breedinginsight/services/TraitService.java @@ -491,4 +491,29 @@ public List getByName(UUID programId, List names) throws DoesNotE return traitDAO.getTraitsByTraitName(programId, names.stream().map(name -> Trait.builder().observationVariableName(name).build()).collect(Collectors.toList())); } + + public Trait getByObservationVariableDbId(UUID programId, String observationVariableDbId) throws DoesNotExistException { + if (!programService.exists(programId)) { + throw new DoesNotExistException("Program does not exist"); + } + + return traitDAO.getTraitsFullByProgramId(programId).stream() + .filter(t -> t.getObservationVariableDbId().equals(observationVariableDbId)) + .findFirst().orElseThrow(() -> new DoesNotExistException("Trait not found for observationVariableDbId: " + observationVariableDbId)); + + } + + public HashMap getIdsByObservationVariableDbIds(UUID programId, List observationVariableDbIds) throws DoesNotExistException { + if (!programService.exists(programId)) { + throw new DoesNotExistException("Program does not exist"); + } + return traitDAO.getTraitsFullByProgramId(programId).stream() + .filter(t -> observationVariableDbIds.contains(t.getObservationVariableDbId())) + .collect(Collectors.toMap( + Trait::getObservationVariableDbId, + (t) -> t.getId().toString(), + (existing, replacement) -> existing, + HashMap::new + )); + } } diff --git a/src/main/java/org/breedinginsight/services/UserService.java b/src/main/java/org/breedinginsight/services/UserService.java index 7502ff8b2..b78fdf8a6 100644 --- a/src/main/java/org/breedinginsight/services/UserService.java +++ b/src/main/java/org/breedinginsight/services/UserService.java @@ -390,6 +390,8 @@ private void sendAccountSignUpEmail(BiUserEntity user, SignedJWT jwtToken) { String filledBody = emailTemplate.render(); String subject = "Activate DeltaBreed Account"; + log.debug(filledBody); + // Send email emailUtil.sendEmail(user.getEmail(), subject, filledBody); } diff --git a/src/main/java/org/breedinginsight/services/validators/TraitValidatorService.java b/src/main/java/org/breedinginsight/services/validators/TraitValidatorService.java index aab931921..34d9eaa8a 100644 --- a/src/main/java/org/breedinginsight/services/validators/TraitValidatorService.java +++ b/src/main/java/org/breedinginsight/services/validators/TraitValidatorService.java @@ -228,6 +228,7 @@ public ValidationErrors checkTraitFieldsFormat(List traits, TraitValidato } return errors; } + public List checkDuplicateTraitsExistingByName(UUID programId, List traits){ List duplicates = new ArrayList<>(); diff --git a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java index e03a67017..b730660e3 100644 --- a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java +++ b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java @@ -18,25 +18,37 @@ package org.breedinginsight.utilities; import io.micronaut.context.annotation.Property; + +import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.exceptions.HttpStatusException; import io.micronaut.http.server.exceptions.InternalServerException; import io.reactivex.functions.*; import lombok.extern.slf4j.Slf4j; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.brapi.client.v2.ApiResponse; import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.client.v2.modules.germplasm.GermplasmApi; import org.brapi.v2.model.*; -import org.brapi.v2.model.germ.BrAPIGermplasm; -import org.brapi.v2.model.germ.response.BrAPIGermplasmSingleResponse; +import org.breedinginsight.api.model.v1.response.DataResponse; +import org.breedinginsight.brapi.v1.controller.BrapiVersion; import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.model.ProgramBrAPIEndpoints; +import org.breedinginsight.services.ProgramService; +import org.breedinginsight.services.exceptions.DoesNotExistException; import javax.inject.Inject; import javax.inject.Singleton; +import java.io.IOException; import java.time.Duration; import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.TimeUnit; import static org.brapi.v2.model.BrAPIWSMIMEDataTypes.APPLICATION_JSON; @@ -48,16 +60,19 @@ public class BrAPIDAOUtil { private final Duration searchTimeout; private final int pageSize; private final int postGroupSize; + private final ProgramService programService; @Inject public BrAPIDAOUtil(@Property(name = "brapi.search.wait-time") int searchWaitTime, @Property(name = "brapi.read-timeout") Duration searchTimeout, @Property(name = "brapi.page-size") int pageSize, - @Property(name = "brapi.post-group-size") int postGroupSize) { + @Property(name = "brapi.post-group-size") int postGroupSize, + ProgramService programService) { this.searchWaitTime = searchWaitTime; this.searchTimeout = searchTimeout; this.pageSize = pageSize; this.postGroupSize = postGroupSize; + this.programService = programService; } public List search(Function, Optional>>> searchMethod, @@ -366,4 +381,64 @@ public List post(List brapiObjects, return post(brapiObjects, null, postMethod, null); } + /** + * TODO: replace with brapi client methods when available, will do timeout spec from config at that point + * @param brapiRequest + * @return + * @throws ApiException + */ + public String makeCallWithResponse(Request brapiRequest) throws ApiException { + OkHttpClient client = new OkHttpClient.Builder() + .readTimeout(5, TimeUnit.MINUTES) + .build(); + + // autoclose Response + try (Response response = client.newCall(brapiRequest).execute()) { + if (!response.isSuccessful()) { + throw new ApiException("Request failed with status code: " + response.code()); + } + return response.body().string(); + } catch (IOException e) { + throw new ApiException(e); + } + } + + public HttpResponse makeCall(Request brapiRequest) { + // Create OkHttpClient with timeout + OkHttpClient client = new OkHttpClient.Builder() + .readTimeout(5, TimeUnit.MINUTES) + .build(); + + try (Response brapiResponse = client.newCall(brapiRequest).execute()) { + int statusCode = brapiResponse.code(); + + if (!brapiResponse.isSuccessful()) { + return HttpResponse.status(HttpStatus.valueOf(statusCode)); + } + + String responseBody = brapiResponse.body() != null ? brapiResponse.body().string() : ""; + return HttpResponse.status(HttpStatus.valueOf(statusCode), responseBody); + + } catch (IOException e) { + log.error("Error calling BrAPI Service", e); + throw new HttpStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Error calling BrAPI Service"); + } + } + + public String getProgramBrAPIBaseUrl(UUID programId) { + ProgramBrAPIEndpoints programBrAPIEndpoints; + try { + programBrAPIEndpoints = programService.getBrapiEndpoints(programId); + } catch (DoesNotExistException e) { + throw new HttpStatusException(HttpStatus.NOT_FOUND, "Program does not exist"); + } + + if(programBrAPIEndpoints.getCoreUrl().isEmpty()) { + log.error("Program: " + programId + " is missing BrAPI URL config"); + throw new HttpStatusException(HttpStatus.INTERNAL_SERVER_ERROR, ""); + } + var programBrAPIBaseUrl = programBrAPIEndpoints.getCoreUrl().get(); + programBrAPIBaseUrl = programBrAPIBaseUrl.endsWith("/") ? programBrAPIBaseUrl.substring(0, programBrAPIBaseUrl.length() - 1) : programBrAPIBaseUrl; + return programBrAPIBaseUrl.endsWith(BrapiVersion.BRAPI_V2) ? programBrAPIBaseUrl : programBrAPIBaseUrl + BrapiVersion.BRAPI_V2; + } } diff --git a/src/main/java/org/breedinginsight/utilities/Utilities.java b/src/main/java/org/breedinginsight/utilities/Utilities.java index 20f3254d6..31990928c 100644 --- a/src/main/java/org/breedinginsight/utilities/Utilities.java +++ b/src/main/java/org/breedinginsight/utilities/Utilities.java @@ -29,6 +29,7 @@ import java.sql.Statement; import java.util.*; import java.util.function.Function; +import java.util.regex.Matcher; import java.util.regex.Pattern; public class Utilities { @@ -183,6 +184,37 @@ public static String removeProgramKeyAndUnknownAdditionalData(String original, S return stripped; } + /** + * Extracts the germplasm identifier (GID) from a germplasm name string that contains + * a key in the format "[PROGKEY-NUMBER]". + * + *

This method searches for a pattern matching "[anything-digits]" in the input string + * and returns the numeric portion if found. The prefix before the hyphen can be any sequence + * of characters.

+ * + * @param germplasmNameWithKey The germplasm name string containing the identifier in the format + * "[PROGKEY-NUMBER]", e.g., "TestDup [DEMO-12]" + * @return The numeric portion after the hyphen as a String if the pattern is found, + * or null if the pattern is not found in the input string + * @throws NullPointerException If the input string is null + * + * @example + * 
+     * String gid = extractGid("TestDup [DEMO-12]"); // Returns "12"
+     * String gid = extractGid("Wheat [BRC-789]");   // Returns "789"
+     * String gid = extractGid("NoPattern");         // Returns null
+     *
+ */ + public static String extractGid(String germplasmNameWithKey) { + Pattern pattern = Pattern.compile("\\[(.*?)-(\\d+)\\]"); + Matcher matcher = pattern.matcher(germplasmNameWithKey); + + if (matcher.find()) { + return matcher.group(2); + } + return null; + } + public static String generateApiExceptionLogMessage(ApiException e) { return new StringBuilder("BrAPI Exception: \n\t").append("message: ") .append(e.getMessage()) diff --git a/src/main/java/org/breedinginsight/utilities/email/EmailUtil.java b/src/main/java/org/breedinginsight/utilities/email/EmailUtil.java index a839715a1..7aef12550 100644 --- a/src/main/java/org/breedinginsight/utilities/email/EmailUtil.java +++ b/src/main/java/org/breedinginsight/utilities/email/EmailUtil.java @@ -19,6 +19,7 @@ import io.micronaut.context.annotation.Property; import io.micronaut.http.server.exceptions.HttpServerException; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import javax.inject.Singleton; @@ -29,6 +30,7 @@ import java.util.Date; import java.util.Properties; +@Slf4j @Singleton public class EmailUtil { @@ -66,6 +68,8 @@ protected PasswordAuthentication getPasswordAuthentication() { public void sendEmail(String toEmail, String subject, String body){ try { + log.debug("Sending email to: " + toEmail + " from: " + fromEmail + " with subject: " + subject); + Session session = getSmtpHost(); MimeMessage msg = new MimeMessage(session); //set message headers @@ -83,11 +87,16 @@ public void sendEmail(String toEmail, String subject, String body){ msg.setRecipients(Message.RecipientType.TO, InternetAddress.parse(toEmail, false)); Transport.send(msg); + + log.debug("Email sent to: " + toEmail + " from: " + fromEmail + " with subject: " + subject); } - catch (UnsupportedEncodingException | MessagingException e) { + catch (UnsupportedEncodingException e) { + log.debug("UnsupportedEncodingException " + e.getMessage()); + throw new HttpServerException(e.getMessage()); + } catch (MessagingException e) { + log.debug("MessagingException " + e.getMessage()); throw new HttpServerException(e.getMessage()); } } - } diff --git a/src/main/java/org/breedinginsight/utilities/response/ResponseUtils.java b/src/main/java/org/breedinginsight/utilities/response/ResponseUtils.java index 6b97984f7..cb8791fae 100644 --- a/src/main/java/org/breedinginsight/utilities/response/ResponseUtils.java +++ b/src/main/java/org/breedinginsight/utilities/response/ResponseUtils.java @@ -18,6 +18,7 @@ package org.breedinginsight.utilities.response; import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpResponseFactory; import io.micronaut.http.HttpStatus; import io.micronaut.http.MediaType; import io.micronaut.http.exceptions.HttpStatusException; @@ -193,6 +194,18 @@ private static List search(List data, SearchRequest searchRequest, Abstrac .collect(Collectors.toList()); } + //To enable checking for the case of Germplasm Search when the filter is accessionNumber or list entry number and thereby needs to do exact match + String accessionNumFilter; + if (mapper.exists("accessionNumber")) accessionNumFilter = mapper.getField("accessionNumber").toString(); + else { + accessionNumFilter = ""; + } + String entryNumFilter; + if (mapper.exists("importEntryNumber")) entryNumFilter = mapper.getField("importEntryNumber").toString(); + else { + entryNumFilter = ""; + } + if (filterFields.size() > 0){ // Apply filters List finalFilterFields = filterFields; @@ -211,6 +224,16 @@ private static List search(List data, SearchRequest searchRequest, Abstrac return recordList.stream() .anyMatch(listValue -> listValue.toString().toLowerCase().contains(filterField.getValue().toLowerCase())); + } + else if (!accessionNumFilter.isEmpty() && filterField.getField().toString().equals(accessionNumFilter)) { + //enable exact match in case of GID + return filterField.getField().apply(record).toString() + .toLowerCase().equalsIgnoreCase(filterField.getValue().toLowerCase()); + } + else if (!entryNumFilter.isEmpty() && filterField.getField().toString().equals(entryNumFilter)) { + //enable exact match in case of entry number + return filterField.getField().apply(record).toString() + .toLowerCase().equalsIgnoreCase(filterField.getValue().toLowerCase()); } else { return filterField.getField().apply(record).toString() .toLowerCase().contains(filterField.getValue().toLowerCase()); diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 10acaf1f0..3a98c3fb6 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -110,6 +110,7 @@ flyway: placeholders: default-url: ${brapi.server.default-url} brapi-reference-source: ${brapi.server.reference-source} + orcid-sandbox-authentication: ${ORCID_SANDBOX_AUTHENTICATION:false} out-of-order: true jooq: datasources: @@ -204,3 +205,11 @@ aws: buckets: genotype: bucket: ${AWS_GENO_BUCKET} +startup: + delay: + germplasm: ${GERMPLASM_START_DELAY:2s} + study: ${STUDY_START_DELAY:2s} + trial: ${TRIAL_START_DELAY:2s} + trait: ${TRAIT_START_DELAY:2s} + observation: ${OBSERVATION_START_DELAY:3s} + observation_unit: ${OBSERVATION_UNIT_START_DELAY:3s} diff --git a/src/main/resources/brapi/sql/R__create_indexes.sql b/src/main/resources/brapi/sql/R__create_indexes.sql index 59f3c32af..3b73ec3b6 100644 --- a/src/main/resources/brapi/sql/R__create_indexes.sql +++ b/src/main/resources/brapi/sql/R__create_indexes.sql @@ -15,7 +15,7 @@ -- Indexes to improve read performance of Germplasm operations. CREATE INDEX CONCURRENTLY IF NOT EXISTS "pedigree_edge_this_node_id" ON pedigree_edge (this_node_id); -CREATE INDEX CONCURRENTLY IF NOT EXISTS "pedigree_edge_connected_node_id" ON pedigree_edge (connceted_node_id); +CREATE INDEX CONCURRENTLY IF NOT EXISTS "pedigree_edge_connected_node_id" ON pedigree_edge (connected_node_id); CREATE INDEX CONCURRENTLY IF NOT EXISTS "pedigree_edge_edge_type" ON pedigree_edge (edge_type); CREATE INDEX CONCURRENTLY IF NOT EXISTS "program_external_references_program_entity_id" ON program_external_references (program_entity_id); CREATE INDEX CONCURRENTLY IF NOT EXISTS "external_reference_composite" ON external_reference (external_reference_source, external_reference_id); diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 7ea12fd42..a795160df 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,6 @@ # -version=v1.0.0 -versionInfo=https://github.com/Breeding-Insight/bi-api/releases/tag/v1.0.0 + +version=v1.1.0+951 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/4d8e00f84bd85e31d2ece453cf67c8d7fbe2a4d2 diff --git a/src/test/java/org/breedinginsight/BrAPITest.java b/src/test/java/org/breedinginsight/BrAPITest.java index 8c5c35bad..e6e6072d6 100644 --- a/src/test/java/org/breedinginsight/BrAPITest.java +++ b/src/test/java/org/breedinginsight/BrAPITest.java @@ -49,7 +49,7 @@ public class BrAPITest extends DatabaseTest { public BrAPITest() { super(); - brapiContainer = new GenericContainer<>("breedinginsight/brapi-java-server:develop") + brapiContainer = new GenericContainer<>("breedinginsight/brapi-java-server:rc") .withNetwork(super.getNetwork()) .withImagePullPolicy(PullPolicy.ageBased(Duration.ofMinutes(60))) .withExposedPorts(8080) diff --git a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java index e2961bb9f..a3c47e573 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java @@ -73,6 +73,8 @@ public class ExperimentControllerIntegrationTest extends BrAPITest { private List traits; private User testUser; private User otherTestUser; + private String mappingId; + private final String experimentTitle = "Test Exp"; @Property(name = "brapi.server.reference-source") private String BRAPI_REFERENCE_SOURCE; @@ -145,7 +147,7 @@ void setup() throws Exception { .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class ); HttpResponse response = call.blockingFirst(); - String mappingId = JsonParser.parseString(Objects.requireNonNull(response.body())).getAsJsonObject() + mappingId = JsonParser.parseString(Objects.requireNonNull(response.body())).getAsJsonObject() .getAsJsonObject("result") .getAsJsonArray("data") .get(0).getAsJsonObject().get("id").getAsString(); @@ -171,8 +173,8 @@ void setup() throws Exception { germplasmDAO.createBrAPIGermplasm(germplasm, program.getId(), null); // Make test experiment import - Map row1 = makeExpImportRow("Env1"); - Map row2 = makeExpImportRow("Env2"); + Map row1 = makeExpImportRow(experimentTitle, "Env1"); + Map row2 = makeExpImportRow(experimentTitle, "Env2"); // Add test observation data for (Trait trait : traits) { @@ -209,6 +211,37 @@ void setup() throws Exception { envIds.add(getEnvId(importResult, 1)); } + // Create an experiment with no observations. + private String uploadExperimentWithoutObs() throws Exception { + ImportTestUtils importTestUtils = new ImportTestUtils(); + List> expRows = new ArrayList<>(); + + // Make test experiment import. + Map row1 = makeExpImportRow("Without Obs", "NewEnv1"); + Map row2 = makeExpImportRow("Without Obs", "NewEnv2"); + + expRows.add(row1); + expRows.add(row2); + + // Import test experiment, environments. + JsonObject importResult = importTestUtils.uploadAndFetchWorkflow( + writeDataToFile(expRows, null), + null, + true, + client, + program, + mappingId, + newExperimentWorkflowId); + String expId = importResult + .get("preview").getAsJsonObject() + .get("rows").getAsJsonArray() + .get(0).getAsJsonObject() + .get("trial").getAsJsonObject() + .get("id").getAsString(); + + return expId; + } + /** * Tests all 18 permutations of * 3 formats: [CSV, XLS, XLSX], @@ -690,6 +723,92 @@ public void getExperimentalCollaboratorsDeactivated(boolean active) { dsl.execute(securityFp.get("DeleteProgramUser"), otherTestUser.getId().toString()); } + /** + * A delete request with an invalid trialDbId should result in a 404 Not Found response. + */ + @Test + @SneakyThrows + public void deleteExperimentInvalid() { + // A DELETE request endpoint with an invalid experimentId. + Flowable> invalidDeleteCall = client.exchange( + DELETE(String.format("/programs/%s/experiments/%s", program.getId().toString(), "00000000-1111-2222-3333-444444444444")) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + + // Ensure 404 NOT_FOUND response for requesting to delete a non-existant trial. + HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { + HttpResponse response = invalidDeleteCall.blockingFirst(); + }); + assertEquals(HttpStatus.NOT_FOUND, e.getStatus()); + } + + /* Test the following 4 Cases: + * 1. hard delete with obs - failure + * 2. soft delete with obs - success + * 3. hard delete without obs - success + * 4. soft delete without obs - success + */ + @ParameterizedTest + @CsvSource(value = {"true,true", "false,true", "true,false", "false,false"}) + @SneakyThrows + public void deleteExperimentSuccess(boolean hardDelete, boolean withObservations) { + // Set up a test trial and get the trialDbId. + String trialDbId; + if (withObservations) { + JsonArray beforeData = getProgramTrials(program.getId().toString()); + + // The trial created by setup has observations. + trialDbId = beforeData.get(0).getAsJsonObject().get("trialDbId").getAsString(); + } else { + // Create a trial without observations. + trialDbId = uploadExperimentWithoutObs(); + } + + // A DELETE request should delete an experiment with observations unless there are observations and hardDelete = true. + Flowable> deleteCall = client.exchange( + DELETE(String.format("/programs/%s/experiments/%s?hard=%s", program.getId().toString(), trialDbId, hardDelete)) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + + // Ensure 204 NO_CONTENT response after successfully deleting a trial, + // unless there are observations and hard delete is requested, then ensure 409 Conflict response. + if (hardDelete && withObservations) { + // Ensure 404 NOT_FOUND response for requesting to delete a non-existant trial. + HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { + HttpResponse response = deleteCall.blockingFirst(); + }); + assertEquals(HttpStatus.CONFLICT, e.getStatus()); + + // Check that the trial was not deleted. + JsonArray trials = getProgramTrials(program.getId().toString()); + assertEquals(1, trials.size()); + + // Check that the studies were not deleted. + JsonArray studies = getProgramStudies(program.getId().toString()); + assertEquals(2, studies.size()); + + // Check that lists were not deleted. + JsonArray lists = getProgramObsVarLists(program.getId().toString()); + assertEquals(1, lists.size()); + } else { + HttpResponse deleteResponse = deleteCall.blockingFirst(); + + assertEquals(HttpStatus.NO_CONTENT, deleteResponse.getStatus()); + + // Check that the trial was deleted. + JsonArray trials = getProgramTrials(program.getId().toString()); + assertEquals(0, trials.size()); + + // Check that the studies were deleted. + JsonArray studies = getProgramStudies(program.getId().toString()); + assertEquals(0, studies.size()); + + // Check that the BrAPI lists were deleted. + JsonArray lists = getProgramObsVarLists(program.getId().toString()); + assertEquals(0, lists.size()); + } + + } private List> buildSubEntityRows(List> topLevelRows, String entityName, int repeatedMeasures) { List> plantRows = new ArrayList<>(); @@ -726,11 +845,11 @@ private File writeDataToFile(List> data, List traits) return file; } - private Map makeExpImportRow(String environment) { + private Map makeExpImportRow(String title, String environment) { Map row = new HashMap<>(); row.put(ExperimentObservation.Columns.GERMPLASM_GID, "1"); row.put(ExperimentObservation.Columns.TEST_CHECK, "T"); - row.put(ExperimentObservation.Columns.EXP_TITLE, "Test Exp"); + row.put(ExperimentObservation.Columns.EXP_TITLE, title); row.put(ExperimentObservation.Columns.EXP_UNIT, "Plot"); //row.put(ExperimentObservation.Columns.SUB_OBS_UNIT, ""); row.put(ExperimentObservation.Columns.EXP_TYPE, "Phenotyping"); @@ -931,4 +1050,39 @@ private String getEnvId(JsonObject result, int index) { .get("referenceId").getAsString(); } + private JsonArray getProgramTrials(String programId) { + Flowable> getCall = client.exchange( + GET(String.format("/programs/%s/brapi/v2/trials", programId)) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = getCall.blockingFirst(); + + // Parse result. + JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); + return result.getAsJsonArray("data"); + } + + private JsonArray getProgramStudies(String programId) { + Flowable> getCall = client.exchange( + GET(String.format("/programs/%s/brapi/v2/studies", programId)) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = getCall.blockingFirst(); + + // Parse result. + JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); + return result.getAsJsonArray("data"); + } + + private JsonArray getProgramObsVarLists(String programId) { + Flowable> getCall = client.exchange( + GET(String.format("/programs/%s/brapi/v2/lists?listType=observationVariables", programId)) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = getCall.blockingFirst(); + + // Parse result. + JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); + return result.getAsJsonArray("data"); + } } diff --git a/src/test/java/org/breedinginsight/api/v1/controller/metadata/MetadataFilterIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/metadata/MetadataFilterIntegrationTest.java index 2d56ffb3b..4257f90d6 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/metadata/MetadataFilterIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/metadata/MetadataFilterIntegrationTest.java @@ -132,7 +132,7 @@ public void getDataResponseMetadataFilterSuccess() { // Check our page numbers weren't altered by the filtered assertEquals(1, data.getAsJsonPrimitive("totalPages").getAsInt(), "Default total pages is incorrect"); - assertEquals(38, data.getAsJsonPrimitive("totalCount").getAsInt(), "Default total count is incorrect"); + assertEquals(37, data.getAsJsonPrimitive("totalCount").getAsInt(), "Default total count is incorrect"); assertEquals(50, data.getAsJsonPrimitive("pageSize").getAsInt(), "Default page size is incorrect"); assertEquals(1, data.getAsJsonPrimitive("currentPage").getAsInt(), "Default current page is incorrect"); } diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java index b9ba07d43..27cf6defd 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java @@ -63,6 +63,7 @@ import java.util.*; import static io.micronaut.http.HttpRequest.*; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertEquals; @MicronautTest @@ -71,8 +72,10 @@ public class BrAPIObservationsControllerIntegrationTest extends BrAPITest { private Program program; - private String experimentId; + private String experimentId; private List envIds = new ArrayList<>(); + // Use hardcoded values for more deterministic test runs and easier assertions. + private List values = List.of(0.125F, 12.415F); private final List> rows = new ArrayList<>(); private final List columns = ExperimentFileColumns.getOrderedColumns(); private List traits; @@ -170,15 +173,12 @@ void setup() throws Exception { Map row2 = makeExpImportRow("Env2"); // Add test observation data - for (Trait trait : traits) { - Random random = new Random(); - + for (int i = 0; i < traits.size(); i++) { // TODO: test for sending obs data as double. // A float is returned from the backend instead of double. there is a separate card to fix this. - // Double val1 = Math.random(); - - Float val1 = random.nextFloat(); - row1.put(trait.getObservationVariableName(), val1); + int valueIndex = i % values.size(); // Prevent overflow. + row1.put(traits.get(i).getObservationVariableName(), values.get(valueIndex)); + row2.put(traits.get(i).getObservationVariableName(), values.get(valueIndex)); } rows.add(row1); @@ -273,7 +273,7 @@ public void testGetObsTableOK() { } @Test - @Disabled("Disabled until fetching of observations is implemented") + @Disabled("Disabled until fetching of observations is implemented in BI-2506.") public void testGetObsListByExpId() { Flowable> call = client.exchange( GET(String.format("/programs/%s/brapi/v2/observations?trialDbId=%s", program.getId(), experimentId)) @@ -286,7 +286,7 @@ public void testGetObsListByExpId() { } @Test - @Disabled("Disabled until fetching of observations is implemented") + @Disabled("Disabled until fetching of observations is implemented in BI-2506.") public void testGetOUById() { Flowable> call = client.exchange( GET(String.format("/programs/%s/brapi/v2/observations?trialDbId=%s", program.getId(), experimentId)) @@ -311,6 +311,99 @@ public void testGetOUById() { assertEquals(HttpStatus.OK, ouResponse.getStatus()); } + @Test + public void testGetObsByStudyDbId() { + // Make a GET request to the /observations endpoint with the studyDbId query parameter. + Flowable> call = client.exchange( + GET(String.format("/programs/%s/brapi/v2/observations?studyDbId=%s", program.getId(), envIds.get(0))) + .bearerAuth("test-registered-user"), + String.class + ); + + // Check for 200 OK response. + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + + // Check that two observations were returned. + JsonObject responseObj = gson.fromJson(response.body(), JsonObject.class); + JsonArray observations = responseObj.getAsJsonObject("result").getAsJsonArray("data"); + assertEquals(2, observations.size()); + + // Check the observation values, keep in mind the order of results is not guaranteed. + Float value1 = observations.get(0).getAsJsonObject().get("value").getAsFloat(); + Float value2 = observations.get(1).getAsJsonObject().get("value").getAsFloat(); + assertTrue(values.contains(value1), "Observation with value " + value1 + " not found."); + assertTrue(values.contains(value2), "Observation with value " + value2 + " not found."); + } + + @Test + public void testGetObsPagination() { + + // Check no pagination. + checkPagination(null, null, HttpStatus.OK, 4, 4, 1); + // Check page and pageSize defaults. + checkPagination(null, 2, HttpStatus.OK, 2, 4, 2); + checkPagination(0, null, HttpStatus.OK, 4, 4, 1); + // Check valid pagination, including last page edge case. + checkPagination(0, 1, HttpStatus.OK, 1, 4, 4); + checkPagination(1, 2, HttpStatus.OK, 2, 4, 2); + checkPagination(0, 3, HttpStatus.OK, 3, 4, 2); + checkPagination(1, 3, HttpStatus.OK, 1, 4, 2); + checkPagination(0, 100, HttpStatus.OK, 4, 4, 1); + // Check invalid pagination. + checkPagination(2, 2, HttpStatus.BAD_REQUEST, null, null, null); + checkPagination(0, 0, HttpStatus.BAD_REQUEST, null, null, null); + checkPagination(1, 0, HttpStatus.BAD_REQUEST, null, null, null); + checkPagination(10, 100, HttpStatus.BAD_REQUEST, null, null, null); + + } + + private void checkPagination(Integer page, Integer pageSize, HttpStatus expectedStatus, Integer expectedSize, Integer expectedTotalCount, Integer expectedTotalPages) { + // Build request URL. + String requestURL = String.format("/programs/%s/brapi/v2/observations", program.getId()); + if (page != null) { + requestURL = requestURL + "?page=" + page; + } + if (pageSize != null && page == null) { + requestURL = requestURL + "?pageSize=" + pageSize; + } else if (pageSize != null) { + requestURL = requestURL + "&pageSize=" + pageSize; + } + + // Make a GET request to the /observations endpoint with the supplied pagination parameters. + Flowable> call = client.exchange( + GET(requestURL).bearerAuth("test-registered-user"), + String.class + ); + + // Check for expected response. + try { + HttpResponse response = call.blockingFirst(); + assertEquals(expectedStatus, response.getStatus()); + + // If call.blockingFirst() doesn't throw, expect a 200 OK. + assertEquals(HttpStatus.OK, response.getStatus()); + + // Parse and check body and metadata. + JsonObject responseObj = gson.fromJson(response.body(), JsonObject.class); + // Get metadata. + JsonObject pagination = responseObj.getAsJsonObject("metadata").getAsJsonObject("pagination"); + assertEquals(expectedSize, pagination.get("pageSize").getAsInt()); + int expectedPage = page == null ? 0 : page; + assertEquals(expectedPage, pagination.get("currentPage").getAsInt()); + assertEquals(expectedTotalPages, pagination.get("totalPages").getAsInt()); + assertEquals(expectedTotalCount, pagination.get("totalCount").getAsInt()); + // Get observations. + JsonArray observations = responseObj.getAsJsonObject("result").getAsJsonArray("data"); + assertEquals(expectedSize, observations.size()); + + } catch (HttpClientResponseException e) { + // call.blockingFirst() will throw in the case of a non-200 code. + assertEquals(expectedStatus, e.getStatus()); + } + + } + private File writeDataToFile(List> data, List traits) throws IOException { File file = File.createTempFile("test", ".csv"); diff --git a/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java index 23e8f54d4..32781dd8c 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java @@ -245,7 +245,7 @@ public void getAllGermplasmByListSuccess() { String germplasmListDbId = fetchGermplasmListDbId(programId); // Build the endpoint to get germplasm by germplasm list. - String endpoint = String.format("/programs/%s/germplasm/lists/%s/records", programId, germplasmListDbId); + String endpoint = String.format("/programs/%s/brapi/v2/germplasm?listDbId=%s", programId, germplasmListDbId); // Get germplasm by list. Flowable> call = client.exchange( @@ -258,7 +258,7 @@ public void getAllGermplasmByListSuccess() { JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); JsonArray data = result.getAsJsonArray("data"); - assertEquals(data.size(), 3, "Wrong number of germplasm were returned"); + assertEquals(3, data.size(), "Wrong number of germplasm were returned"); } @Test diff --git a/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java index 9515b72a6..2d421d306 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java @@ -23,6 +23,7 @@ import io.micronaut.http.HttpStatus; import io.micronaut.http.client.RxHttpClient; import io.micronaut.http.client.annotation.Client; +import io.micronaut.http.client.exceptions.HttpClientResponseException; import io.micronaut.http.netty.cookies.NettyCookie; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import io.reactivex.Flowable; @@ -41,6 +42,7 @@ import org.breedinginsight.model.Program; import org.breedinginsight.services.SpeciesService; import org.jooq.DSLContext; +import org.junit.Rule; import org.junit.jupiter.api.*; import javax.inject.Inject; @@ -49,6 +51,7 @@ import java.time.OffsetDateTime; import java.util.*; +import static io.micronaut.http.HttpRequest.DELETE; import static io.micronaut.http.HttpRequest.GET; import static org.junit.jupiter.api.Assertions.*; @@ -145,13 +148,12 @@ public void setup() { newExp.put(traits.get(0).getObservationVariableName(), "1"); JsonObject result = importTestUtils.uploadAndFetchWorkflow( - importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId, newExperimentWorkflowId - ); - + importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId, newExperimentWorkflowId); } @Test @SneakyThrows + @Order(1) public void getAllListsSuccess() { // A GET request to the brapi/v2/lists endpoint with no query params should return all lists. Flowable> call = client.exchange( @@ -159,6 +161,12 @@ public void getAllListsSuccess() { .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class ); + // Collect all responses + List> responses = call.toList().blockingGet(); + + // Ensure we got a response + assertFalse(responses.isEmpty(), "No response received"); + // Ensure 200 OK response. HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.OK, response.getStatus()); @@ -190,4 +198,48 @@ public void getAllListsSuccess() { } } + + @Test + @SneakyThrows + @Order(2) + public void deleteListSuccess() { + // A GET request to the brapi/v2/lists endpoint with no query params should return all lists. + Flowable> getCall = client.exchange( + GET(String.format("/programs/%s/brapi/v2/lists", program.getId().toString())) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + + // Ensure 200 OK response for fetching lists. + HttpResponse beforeResponse = getCall.blockingFirst(); + assertEquals(HttpStatus.OK, beforeResponse.getStatus()); + + // Parse result. + JsonObject beforeResult = JsonParser.parseString(beforeResponse.body()).getAsJsonObject().getAsJsonObject("result"); + JsonArray beforeData = beforeResult.getAsJsonArray("data"); + + // A DELETE request to the brapi/v2/lists/ endpoint with no query params should delete the list. + String deleteListDbId = beforeData.get(0).getAsJsonObject().get("listDbId").getAsString(); + Flowable> deleteCall = client.exchange( + DELETE(String.format("/programs/%s/brapi/v2/lists/%s", program.getId().toString(), deleteListDbId)) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + + // Ensure 204 NO_CONTENT response for deleting a list. + HttpResponse deleteResponse = deleteCall.blockingFirst(); + assertEquals(HttpStatus.NO_CONTENT, deleteResponse.getStatus()); + + + // A DELETE request to the brapi/v2/lists/ endpoint with invalid dbId. + Flowable> invalidDeleteCall = client.exchange( + DELETE(String.format("/programs/%s/brapi/v2/lists/%s", program.getId().toString(), "NOT-VALID-DBID")) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + + // Ensure 404 NOT_FOUND response for requesting to delete a non-existant list. + try { + invalidDeleteCall.blockingFirst(); + } catch(HttpClientResponseException e) { + assertEquals(HttpStatus.NOT_FOUND, e.getStatus()); + } + } } diff --git a/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java index 9a58281bf..13d60a714 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java @@ -20,7 +20,7 @@ import org.breedinginsight.brapps.importer.model.imports.germplasm.GermplasmImportService; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; -import org.breedinginsight.brapps.importer.services.processors.GermplasmProcessor; +import org.breedinginsight.brapps.importer.services.processors.germplasm.GermplasmProcessor; import org.breedinginsight.dao.db.tables.pojos.BiUserEntity; import org.breedinginsight.dao.db.tables.pojos.ProgramBreedingMethodEntity; import org.breedinginsight.daos.BreedingMethodDAO; @@ -30,18 +30,18 @@ import org.breedinginsight.utilities.Utilities; import org.jooq.DSLContext; import org.junit.jupiter.api.*; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import tech.tablesaw.api.Table; import javax.inject.Inject; import java.io.File; import java.time.OffsetDateTime; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import static io.micronaut.http.HttpRequest.GET; import static io.micronaut.http.HttpRequest.POST; +import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.junit.jupiter.api.Assertions.*; @MicronautTest @@ -733,6 +733,223 @@ public void selfReferenceParentError() { assertEquals(GermplasmProcessor.circularDependency, result.getAsJsonObject("progress").get("message").getAsString()); } + /** + * Test preview and commit data when entry numbers are in ascending order in file. This is the normal case and here + * to catch any possible regressions + * + * @param commit controls whether import is preview or commit + */ + @ParameterizedTest + @ValueSource(booleans = {false, true}) + @SneakyThrows + public void entryNoAscending(boolean commit) { + String pathname = "src/test/resources/files/germplasm_import/entry_no_asc.csv"; + Table fileData = Table.read().file(pathname); + String listName = "EntryNoAsc"; + String listDescription = "Entry numbers in ascending order with pedigree"; + + JsonObject result = importGermplasm(pathname, listName, listDescription, commit); + assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt()); + + // preview table is sorted by entry number + fileData = fileData.sortAscendingOn("Entry No"); + + JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + checkEntryNoFields(fileData, previewRows, commit, listName, listDescription, "EntryNoAscGerm 2", "EntryNoAscGerm 3"); + } + + /** + * Prior to BI-2573 this file would result in a false positive circular dependency error. The reason is that when + * entry no order did not match germplasm file order, pedigree information was assigned to the wrong germplasm + * record due to inconsistencies in sorting during processing. + * + * Test preview and commit data when entry numbers are in descending order in file + * + * @param commit controls whether import is preview or commit + */ + @ParameterizedTest + @ValueSource(booleans = {false, true}) + @SneakyThrows + public void entryNoDescending(boolean commit) { + String pathname = "src/test/resources/files/germplasm_import/entry_no_desc.csv"; + Table fileData = Table.read().file(pathname); + String listName = "EntryNoDesc"; + String listDescription = "Entry numbers in descending order with pedigree"; + + JsonObject result = importGermplasm(pathname, listName, listDescription, commit); + assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt()); + + // preview table is sorted by entry number + fileData = fileData.sortAscendingOn("Entry No"); + + JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + checkEntryNoFields(fileData, previewRows, commit, listName, listDescription, "EntryNoDescGerm 2", "EntryNoDescGerm 1"); + } + + /** + * Prior to BI-2593 this file would result in a false positive circular dependency error. This was due to germplasm + * references not being unique in the preview and commit phases of the postOrder method. + * + * @param commit controls whether import is preview or commit + */ + @Order(7) // want some existing gids to reference + @ParameterizedTest + @ValueSource(booleans = {false, true}) + @SneakyThrows + public void duplicateNames(boolean commit) { + String pathname = "src/test/resources/files/germplasm_import/duplicate_names_circular_dependency.csv"; + Table fileData = Table.read().file(pathname); + String listName = "DuplicateNames"; + String listDescription = "Duplicate names with pedigree"; + + JsonObject result = importGermplasm(pathname, listName, listDescription, commit); + assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt()); + + // preview table is sorted by entry number + fileData = fileData.sortAscendingOn("Entry No"); + + JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + checkEntryNoFields(fileData, previewRows, commit, listName, listDescription, "TestDup", "TestDup"); + } + + /** + * Shared method to perform entry number order tests for preview and commit + * + * @param fileData raw file data + * @param previewRows processed preview rows + * @param commit whether checking preview or commit fields + * @param listName name of list when committing data + * @param listDescription list description when committing data + * @param motherName name of mother for pedigree check (from file) + * @param fatherName name of father for pedigree check (from file) + */ + private void checkEntryNoFields(Table fileData, JsonArray previewRows, boolean commit, + String listName, String listDescription, + String motherName, String fatherName) { + List germplasmNames = new ArrayList<>(); + + for (int i = 0; i < previewRows.size(); i++) { + JsonObject germplasm = previewRows.get(i).getAsJsonObject().getAsJsonObject("germplasm").getAsJsonObject("brAPIObject"); + germplasmNames.add(germplasm.get("germplasmName").getAsString()); + checkBasicResponse(germplasm, fileData, i); + + if (!commit) { + // preview checks + checkEntryNoPreviewFields(fileData, previewRows, i, motherName, fatherName); + } else { + // commit checks + checkEntryNoCommitFields(fileData, previewRows, i, motherName, fatherName); + } + } + + if (commit) { + // Check the germplasm list + checkGermplasmList(Germplasm.constructGermplasmListName(listName, validProgram), listDescription, germplasmNames); + } + } + + /** + * Check important field in preview data beyond basic info + * - entry number assignment + * - pedigree assignment + * + * @param fileData raw file data + * @param previewRows processed preview rows + */ + private void checkEntryNoPreviewFields(Table fileData, JsonArray previewRows, int i, String motherName, String fatherName) { + JsonObject germplasm = previewRows.get(i).getAsJsonObject().getAsJsonObject("germplasm").getAsJsonObject("brAPIObject"); + + // Check preview specific items + // Germplasm name (display name) + assertEquals(fileData.getString(i, "Germplasm Name"), germplasm.get("germplasmName").getAsString()); + JsonObject additionalInfo = germplasm.getAsJsonObject("additionalInfo"); + + // check entry number assignment + assertEquals(fileData.getString(i, "Entry No"), additionalInfo.get("importEntryNumber").getAsString(), "Wrong entry number"); + // check pedigree entry number assignment + String fileFemaleEntryNo = fileData.getString(i, "Female Parent Entry No"); + if (isNotBlank(fileFemaleEntryNo)) { + assertEquals(fileFemaleEntryNo, additionalInfo.get("femaleParentEntryNo").getAsString(), "Wrong female parent entry number"); + } + String fileMaleEntryNo = fileData.getString(i, "Male Parent Entry No"); + if (isNotBlank(fileMaleEntryNo)) { + assertEquals(fileMaleEntryNo, additionalInfo.get("maleParentEntryNo").getAsString(), "Wrong male parent entry number"); + } + + // check preview pedigree values + // only care about entry nos for this test case, not using GIDs + if (isNotBlank(fileFemaleEntryNo) && isNotBlank(fileMaleEntryNo)) { + String pedigree = germplasm.get("pedigree").getAsString(); + String[] pedigreeParts = pedigree.split("/"); + String mother = pedigreeParts[0]; + String father = pedigreeParts[1]; + assertEquals(motherName, mother, "Wrong mother"); + assertEquals(fatherName, father, "Wrong father"); + } + } + + /** + * Check properties of processed and committed germplasm objects match what would be expected based on file contents. + * Of particular importance are: + * - germplasm are in entry no order + * - gids are assigned in entry no order + * - correct pedigree was assigned to germplasm based on file specification + * + * @param fileData raw file data + * @param previewRows processed preview rows + * @param i row index + */ + private void checkEntryNoCommitFields(Table fileData, JsonArray previewRows, int i, String motherName, String fatherName) { + JsonObject germplasm = previewRows.get(i).getAsJsonObject().getAsJsonObject("germplasm").getAsJsonObject("brAPIObject"); + // Check commit specific items + // Germplasm name (display name) + String expectedGermplasmName = String.format("%s [%s-%s]", fileData.getString(i, "Germplasm Name"), validProgram.getKey(), germplasm.get("accessionNumber").getAsString()); + assertEquals(expectedGermplasmName, germplasm.get("germplasmName").getAsString()); + // Created Date + JsonObject additionalInfo = germplasm.getAsJsonObject("additionalInfo"); + assertTrue(additionalInfo.has(BrAPIAdditionalInfoFields.CREATED_DATE), "createdDate is missing"); + // Accession Number + assertTrue(germplasm.has("accessionNumber"), "accessionNumber missing"); + + // check that gids are assigned in entry no order + if (i > 0) { + JsonObject previousGermplasm = previewRows.get(i-1).getAsJsonObject().getAsJsonObject("germplasm").getAsJsonObject("brAPIObject"); + int lastEntryNo = previousGermplasm.getAsJsonObject("additionalInfo").get("importEntryNumber").getAsInt(); + int lastGid = previousGermplasm.get("accessionNumber").getAsInt(); + int currentEntryNo = additionalInfo.get("importEntryNumber").getAsInt(); + int currentGid = germplasm.get("accessionNumber").getAsInt(); + assertEquals(1, currentEntryNo-lastEntryNo, "Expected entry number to be monotonically increasing"); + assertEquals(1, currentGid-lastGid, "Expected GID to be monotonically increasing"); + } + + // pedigree check just for single germplasm in file with pedigree info + String fileFemaleEntryNo = fileData.getString(i, "Female Parent Entry No"); + String fileMaleEntryNo = fileData.getString(i, "Male Parent Entry No"); + + // only care about entry nos for this test case, not using GIDs + if (isNotBlank(fileFemaleEntryNo) && isNotBlank(fileMaleEntryNo)) { + String pedigree = germplasm.get("pedigree").getAsString(); + String[] pedigreeParts = pedigree.split("/"); + String mother = pedigreeParts[0]; + String father = pedigreeParts[1]; + String regexMatcher = "^(.*\\b) \\[([A-Z]{2,6})-(\\d+)\\]$"; + assertTrue(mother.matches(String.format(regexMatcher, motherName)), "Wrong mother"); + assertTrue(father.matches(String.format(regexMatcher, fatherName)), "Wrong father"); + } + + // External Reference germplasm + JsonArray externalReferences = germplasm.getAsJsonArray("externalReferences"); + boolean referenceFound = false; + for (JsonElement reference: externalReferences) { + String referenceSource = reference.getAsJsonObject().get("referenceSource").getAsString(); + if (referenceSource.equals(BRAPI_REFERENCE_SOURCE)) { + referenceFound = true; + break; + } + } + assertTrue(referenceFound, "Germplasm UUID reference not found"); + } + private JsonObject importGermplasm(String pathname, String listName, String listDescription, Boolean commit) throws InterruptedException { File file = new File(pathname); diff --git a/src/test/java/org/breedinginsight/daos/BrAPIDAOUtilUnitTest.java b/src/test/java/org/breedinginsight/daos/BrAPIDAOUtilUnitTest.java index 0cb51387e..427ed6d88 100644 --- a/src/test/java/org/breedinginsight/daos/BrAPIDAOUtilUnitTest.java +++ b/src/test/java/org/breedinginsight/daos/BrAPIDAOUtilUnitTest.java @@ -1,6 +1,7 @@ package org.breedinginsight.daos; import com.google.gson.JsonObject; +import io.micronaut.test.annotation.MockBean; import lombok.SneakyThrows; import org.apache.commons.lang3.tuple.Pair; import org.brapi.client.v2.ApiResponse; @@ -11,6 +12,7 @@ import org.brapi.v2.model.germ.response.BrAPIGermplasmListResponseResult; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.model.Program; +import org.breedinginsight.services.ProgramService; import org.breedinginsight.utilities.BrAPIDAOUtil; import org.breedinginsight.utilities.Utilities; import org.junit.jupiter.api.BeforeEach; @@ -21,6 +23,8 @@ import java.time.temporal.ChronoUnit; import java.util.*; +import static org.mockito.Mockito.mock; + @TestInstance(TestInstance.Lifecycle.PER_CLASS) public class BrAPIDAOUtilUnitTest { @@ -30,6 +34,10 @@ public class BrAPIDAOUtilUnitTest { private Program testProgram; private List paginatedGermplasm; private BrAPIGermplasmSearchRequest germplasmSearch; + @MockBean(ProgramService.class) + ProgramService programService() { + return mock(ProgramService.class); + } public Integer fetchPaginatedGermplasm(int page, int pageSize) { paginatedGermplasm = new ArrayList<>(); @@ -62,7 +70,7 @@ public ApiResponse, Optional