From ed97babcab4a62e268819bf4cae7921a07406c57 Mon Sep 17 00:00:00 2001 From: Reingold Shekhtel <13565058+raikbitters@users.noreply.github.com> Date: Thu, 13 Jul 2023 15:40:29 +0400 Subject: [PATCH 01/27] Improve docker image build (#79) Update dockerfile and gradle.properties --- Dockerfile | 32 +++++++++++++++++++++++--------- docker/Dockerfile-release | 26 -------------------------- gradle.properties | 4 ++-- 3 files changed, 25 insertions(+), 37 deletions(-) delete mode 100644 docker/Dockerfile-release diff --git a/Dockerfile b/Dockerfile index dcdfde6..efadd2c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,25 @@ -FROM amazoncorretto:11.0.17 -LABEL version=5.7.4 description="EPAM Report portal. Service jobs" maintainer="Andrei Varabyeu , Hleb Kanonik " -ARG GH_TOKEN -ARG GH_URL=https://__:$GH_TOKEN@maven.pkg.github.com/reportportal/service-jobs/com/epam/reportportal/service-jobs/5.7.4/service-jobs-5.7.4-exec.jar -RUN curl -O -L $GH_URL \ - --output service-jobs-5.7.3-exec.jar && \ - echo 'exec java ${JAVA_OPTS} -jar service-jobs-5.7.4-exec.jar' > /start.sh && chmod +x /start.sh -ENV JAVA_OPTS="-Xmx512m -XX:+UseG1GC -XX:InitiatingHeapOccupancyPercent=70 -Djava.security.egd=file:/dev/./urandom" +FROM gradle:6.8.3-jdk11 AS build +ARG BOM_VERSION MIGRATION_VERSION GITHUB_USER GITHUB_TOKEN RELEASE_MODE SCRIPTS_VERSION APP_VERSION +WORKDIR /usr/app +COPY . /usr/app +RUN if [ ${RELEASE_MODE} = true ]; then \ + gradle build --exclude-task test \ + -PreleaseMode=true \ + -PgithubUserName=${GITHUB_USER} \ + -PgithubToken=${GITHUB_TOKEN} \ + -Pscripts.version=${SCRIPTS_VERSION} \ + -Pmigrations.version=${MIGRATION_VERSION} \ + -Pbom.version=${BOM_VERSION} \ + -Dorg.gradle.project.version=${APP_VERSION}; \ + else gradle build --exclude-task test -Dorg.gradle.project.version=${APP_VERSION}; fi + +# For ARM build use flag: `--platform linux/arm64` +FROM --platform=$BUILDPLATFORM amazoncorretto:11.0.19 +LABEL version=${APP_VERSION} description="EPAM Report portal. Main API Service" maintainer="Andrei Varabyeu , Hleb Kanonik " +ARG APP_VERSION=${APP_VERSION} +ENV APP_DIR=/usr/app JAVA_OPTS="-Xmx1g -XX:+UseG1GC -XX:InitiatingHeapOccupancyPercent=70 -Djava.security.egd=file:/dev/./urandom" +WORKDIR $APP_DIR +COPY --from=build $APP_DIR/build/libs/service-jobs-*exec.jar . VOLUME ["/tmp"] EXPOSE 8080 -ENTRYPOINT ./start.sh +ENTRYPOINT exec java ${JAVA_OPTS} -jar ${APP_DIR}/service-jobs-*exec.jar \ No newline at end of file diff --git a/docker/Dockerfile-release b/docker/Dockerfile-release deleted file mode 100644 index 0119040..0000000 --- a/docker/Dockerfile-release +++ /dev/null @@ -1,26 +0,0 @@ -FROM openjdk:11-jre-slim - -LABEL version="@version@" -LABEL description="@description@" -LABEL maintainer="Andrei Varabyeu " - -ENV APP_FILE=@name@-@version@-exec.jar -ENV APP_DOWNLOAD_URL=https://dl.bintray.com/epam/reportportal/com/epam/reportportal/@name@/@version@/${APP_FILE} - -RUN apt-get update && \ - apt-get install wget unzip openssl -y && \ - # Create start.sh script - echo '#!/bin/sh \n exec java ${JAVA_OPTS} -jar ${APP_FILE}' > /start.sh && \ - chmod +x /start.sh && \ - # Download application - wget -O /${APP_FILE} ${APP_DOWNLOAD_URL} && \ - # Remove JOBS cache - rm -rf /var/lib/apt/lists/* - -# Set default JAVA_OPTS -ENV JAVA_OPTS="-Xmx512m -Djava.security.egd=file:/dev/./urandom" - -VOLUME ["/tmp"] - -EXPOSE 8080 -ENTRYPOINT ["/start.sh"] diff --git a/gradle.properties b/gradle.properties index eda4b33..d25a90b 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ -version=5.8.1 +version=develop description=EPAM Report portal. Service jobs dockerServerUrl=unix:///var/run/docker.sock dockerPrepareEnvironment= -dockerJavaOpts=-Xmx512m -XX:+UseG1GC -XX:InitiatingHeapOccupancyPercent=70 -Djava.security.egd=file:/dev/./urandom \ No newline at end of file +dockerJavaOpts=-Xmx512m -XX:+UseG1GC -XX:InitiatingHeapOccupancyPercent=70 -Djava.security.egd=file:/dev/./urandom From 8d831754c002fe97275a2a4668c9c7e8ef5d776f Mon Sep 17 00:00:00 2001 From: APiankouski <109206864+APiankouski@users.noreply.github.com> Date: Mon, 24 Jul 2023 10:31:05 +0300 Subject: [PATCH 02/27] EPMRPP-82486 || Update personal data retention policy (#82) * EPMRPP-82486 || Update personal data retention policy * EPMRPP-82486 || Add schedulerLock * EPMRPP-82486 || Remove dao * EPMRPP-82486 || Fix typo --------- Co-authored-by: Andrei Piankouski --- build.gradle | 1 + .../reportportal/ServiceJobApplication.java | 4 +- .../reportportal/analyzer/AnalyzerUtils.java | 73 ++++++ .../analyzer/index/IndexerServiceClient.java | 14 ++ .../index/IndexerServiceClientImpl.java | 44 ++++ .../com/epam/reportportal/jobs/BaseJob.java | 12 - .../jobs/clean/CleanAttachmentJob.java | 2 - .../jobs/clean/CleanLaunchJob.java | 2 - .../reportportal/jobs/clean/CleanLogJob.java | 3 - .../jobs/clean/CleanMaterializedViewJob.java | 4 - .../jobs/clean/CleanStorageJob.java | 3 - .../jobs/clean/DeleteExpiredUsersJob.java | 207 ++++++++++++++++++ .../storage/CalculateAllocatedStorageJob.java | 2 - .../logging/ExecutionTimeAspect.java | 47 ++++ src/main/resources/application.yml | 9 +- 15 files changed, 396 insertions(+), 31 deletions(-) create mode 100644 src/main/java/com/epam/reportportal/analyzer/AnalyzerUtils.java create mode 100644 src/main/java/com/epam/reportportal/jobs/clean/DeleteExpiredUsersJob.java create mode 100644 src/main/java/com/epam/reportportal/logging/ExecutionTimeAspect.java diff --git a/build.gradle b/build.gradle index 4b4ad80..9cffe81 100644 --- a/build.gradle +++ b/build.gradle @@ -68,6 +68,7 @@ dependencies { implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0' + implementation 'org.springframework.boot:spring-boot-starter-aop' implementation 'org.springframework.boot:spring-boot-starter-actuator' implementation 'org.springframework.boot:spring-boot-starter-jdbc' implementation 'org.springframework.boot:spring-boot-starter-web' diff --git a/src/main/java/com/epam/reportportal/ServiceJobApplication.java b/src/main/java/com/epam/reportportal/ServiceJobApplication.java index e79fe9f..75bf53a 100644 --- a/src/main/java/com/epam/reportportal/ServiceJobApplication.java +++ b/src/main/java/com/epam/reportportal/ServiceJobApplication.java @@ -18,8 +18,10 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.flyway.FlywayAutoConfiguration; -@SpringBootApplication(scanBasePackages = { "com.epam.reportportal" }) +@SpringBootApplication(scanBasePackages = { "com.epam.reportportal" }, exclude = { + FlywayAutoConfiguration.class}) public class ServiceJobApplication { public static void main(String[] args) { diff --git a/src/main/java/com/epam/reportportal/analyzer/AnalyzerUtils.java b/src/main/java/com/epam/reportportal/analyzer/AnalyzerUtils.java new file mode 100644 index 0000000..f1393d3 --- /dev/null +++ b/src/main/java/com/epam/reportportal/analyzer/AnalyzerUtils.java @@ -0,0 +1,73 @@ +/* + * Copyright 2023 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.analyzer; + +import static java.util.Optional.ofNullable; + +import com.rabbitmq.http.client.domain.ExchangeInfo; +import java.util.function.Predicate; +import java.util.function.ToIntFunction; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.math.NumberUtils; + +/** + * @author Andrei Piankouski + */ +public final class AnalyzerUtils { + + static final String ANALYZER_KEY = "analyzer"; + static final String ANALYZER_PRIORITY = "analyzer_priority"; + static final String ANALYZER_INDEX = "analyzer_index"; + static final String ANALYZER_LOG_SEARCH = "analyzer_log_search"; + static final String ANALYZER_SUGGEST = "analyzer_suggest"; + static final String ANALYZER_CLUSTER = "analyzer_cluster"; + + /** + * Comparing by client service priority + */ + public static final ToIntFunction EXCHANGE_PRIORITY = it -> ofNullable(it.getArguments() + .get(ANALYZER_PRIORITY)).map(val -> NumberUtils.toInt(val.toString(), Integer.MAX_VALUE)).orElse(Integer.MAX_VALUE); + + /** + * Checks if service support items indexing. false + * by default + */ + public static final Predicate DOES_SUPPORT_INDEX = it -> ofNullable(it.getArguments() + .get(ANALYZER_INDEX)).map(val -> BooleanUtils.toBoolean(val.toString())).orElse(false); + + /** + * Checks if service support logs searching. false + * by default + */ + public static final Predicate DOES_SUPPORT_SEARCH = it -> ofNullable(it.getArguments() + .get(ANALYZER_LOG_SEARCH)).map(val -> BooleanUtils.toBoolean(val.toString())).orElse(false); + + /** + * Checks if service support logs searching. false + * by default + */ + public static final Predicate DOES_SUPPORT_SUGGEST = it -> ofNullable(it.getArguments() + .get(ANALYZER_SUGGEST)).map(val -> BooleanUtils.toBoolean(val.toString())).orElse(false); + + /** + * Checks if service support logs cluster creation. false + * by default + */ + public static final Predicate DOES_SUPPORT_CLUSTER = it -> ofNullable(it.getArguments() + .get(ANALYZER_CLUSTER)).map(val -> BooleanUtils.toBoolean(val.toString())).orElse(false); + +} diff --git a/src/main/java/com/epam/reportportal/analyzer/index/IndexerServiceClient.java b/src/main/java/com/epam/reportportal/analyzer/index/IndexerServiceClient.java index b0e1940..9e11932 100644 --- a/src/main/java/com/epam/reportportal/analyzer/index/IndexerServiceClient.java +++ b/src/main/java/com/epam/reportportal/analyzer/index/IndexerServiceClient.java @@ -45,4 +45,18 @@ public interface IndexerServiceClient { */ void removeFromIndexLessThanLaunchDate(Long index, LocalDateTime lessThanDate); + /** + * Delete index + * + * @param index Index to be deleted + */ + void deleteIndex(Long index); + + /** + * Removes suggest index + * + * @param projectId Project/index id + */ + void removeSuggest(Long projectId); + } diff --git a/src/main/java/com/epam/reportportal/analyzer/index/IndexerServiceClientImpl.java b/src/main/java/com/epam/reportportal/analyzer/index/IndexerServiceClientImpl.java index fe17e0e..363dd5c 100644 --- a/src/main/java/com/epam/reportportal/analyzer/index/IndexerServiceClientImpl.java +++ b/src/main/java/com/epam/reportportal/analyzer/index/IndexerServiceClientImpl.java @@ -3,6 +3,12 @@ import com.epam.reportportal.analyzer.RabbitMqManagementClient; import com.epam.reportportal.model.index.CleanIndexByDateRangeRq; import com.epam.reportportal.model.index.CleanIndexRq; +import com.rabbitmq.http.client.domain.ExchangeInfo; +import java.util.Comparator; +import java.util.Optional; +import java.util.function.Predicate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -15,6 +21,7 @@ import java.util.Map; import java.util.stream.Collectors; +import static com.epam.reportportal.analyzer.AnalyzerUtils.DOES_SUPPORT_SUGGEST; import static com.epam.reportportal.analyzer.RabbitMqManagementClientTemplate.EXCHANGE_PRIORITY; /** @@ -23,10 +30,14 @@ @Service public class IndexerServiceClientImpl implements IndexerServiceClient { + private static final Logger LOGGER = LoggerFactory.getLogger(IndexerServiceClient.class); private static final String CLEAN_ROUTE = "clean"; private static final String CLEAN_BY_LOG_DATE_ROUTE = "remove_by_log_time"; private static final String CLEAN_BY_LAUNCH_DATE_ROUTE = "remove_by_launch_start_time"; private static final String EXCHANGE_NAME = "analyzer-default"; + private static final String REMOVE_SUGGEST_ROUTE = "remove_suggest_info"; + static final String DELETE_ROUTE = "delete"; + private static final Integer DELETE_INDEX_SUCCESS_CODE = 1; // need to be in line with analyzer API, better to fix api and remove it in future. private static final LocalDateTime OLDEST_DATE = LocalDateTime.now().minusYears(10L); @@ -69,9 +80,42 @@ public void removeFromIndexLessThanLaunchDate(Long index, LocalDateTime lessThan sendRangeRemovingMessageToRoute(index, lessThanDate, CLEAN_BY_LAUNCH_DATE_ROUTE); } + @Override + public void deleteIndex(Long index) { + rabbitMqManagementClient.getAnalyzerExchangesInfo() + .stream() + .map(exchange -> rabbitTemplate.convertSendAndReceiveAsType(exchange.getName(), + DELETE_ROUTE, + index, + new ParameterizedTypeReference() { + } + )) + .forEach(it -> { + if (DELETE_INDEX_SUCCESS_CODE.equals(it)) { + LOGGER.info("Successfully deleted index '{}'", index); + } else { + LOGGER.error("Error deleting index '{}'", index); + } + }); + } + private void sendRangeRemovingMessageToRoute(Long index, LocalDateTime lessThanDate, String route) { CleanIndexByDateRangeRq message = new CleanIndexByDateRangeRq(index, OLDEST_DATE, lessThanDate); rabbitTemplate.convertAndSend(EXCHANGE_NAME, route, message); } + @Override + public void removeSuggest(Long projectId) { + resolveExchangeName(DOES_SUPPORT_SUGGEST) + .ifPresent(suggestExchange -> rabbitTemplate.convertAndSend(suggestExchange, REMOVE_SUGGEST_ROUTE, projectId)); + } + + private Optional resolveExchangeName(Predicate supportCondition) { + return rabbitMqManagementClient.getAnalyzerExchangesInfo() + .stream() + .filter(supportCondition) + .min(Comparator.comparingInt(EXCHANGE_PRIORITY)) + .map(ExchangeInfo::getName); + } + } diff --git a/src/main/java/com/epam/reportportal/jobs/BaseJob.java b/src/main/java/com/epam/reportportal/jobs/BaseJob.java index 98fc4ad..76da27d 100644 --- a/src/main/java/com/epam/reportportal/jobs/BaseJob.java +++ b/src/main/java/com/epam/reportportal/jobs/BaseJob.java @@ -11,16 +11,4 @@ public abstract class BaseJob { public BaseJob(JdbcTemplate jdbcTemplate) { this.jdbcTemplate = jdbcTemplate; } - - protected void logStart() { - LOGGER.info("Job {} has been started.", this.getClass().getSimpleName()); - } - - protected void logFinish(Object result) { - LOGGER.info("Job {} has been finished. Result {}", this.getClass().getSimpleName(), result); - } - - protected void logFinish() { - logFinish(null); - } } diff --git a/src/main/java/com/epam/reportportal/jobs/clean/CleanAttachmentJob.java b/src/main/java/com/epam/reportportal/jobs/clean/CleanAttachmentJob.java index 34f9bc8..9c1be1a 100644 --- a/src/main/java/com/epam/reportportal/jobs/clean/CleanAttachmentJob.java +++ b/src/main/java/com/epam/reportportal/jobs/clean/CleanAttachmentJob.java @@ -34,7 +34,6 @@ public void execute() { } void moveAttachments() { - logStart(); AtomicInteger counter = new AtomicInteger(0); getProjectsWithAttribute(KEEP_SCREENSHOTS).forEach((projectId, duration) -> { LocalDateTime lessThanDate = LocalDateTime.now(ZoneOffset.UTC).minus(duration); @@ -42,6 +41,5 @@ void moveAttachments() { counter.addAndGet(movedCount); LOGGER.info("Moved {} attachments to the deletion table for project {}, lessThanDate {} ", movedCount, projectId, lessThanDate); }); - logFinish(counter.get()); } } diff --git a/src/main/java/com/epam/reportportal/jobs/clean/CleanLaunchJob.java b/src/main/java/com/epam/reportportal/jobs/clean/CleanLaunchJob.java index 2904a19..2b193bf 100644 --- a/src/main/java/com/epam/reportportal/jobs/clean/CleanLaunchJob.java +++ b/src/main/java/com/epam/reportportal/jobs/clean/CleanLaunchJob.java @@ -61,7 +61,6 @@ public void execute() { } private void removeLaunches() { - logStart(); AtomicInteger counter = new AtomicInteger(0); getProjectsWithAttribute(KEEP_LAUNCHES).forEach((projectId, duration) -> { final LocalDateTime lessThanDate = LocalDateTime.now(ZoneOffset.UTC).minus(duration); @@ -84,7 +83,6 @@ private void removeLaunches() { } } }); - logFinish(counter.get()); } private void deleteLogsFromElasticsearchByLaunchIdsAndProjectId(List launchIds, Long projectId) { diff --git a/src/main/java/com/epam/reportportal/jobs/clean/CleanLogJob.java b/src/main/java/com/epam/reportportal/jobs/clean/CleanLogJob.java index 5428d24..a93f626 100644 --- a/src/main/java/com/epam/reportportal/jobs/clean/CleanLogJob.java +++ b/src/main/java/com/epam/reportportal/jobs/clean/CleanLogJob.java @@ -53,7 +53,6 @@ public void execute() { } void removeLogs() { - logStart(); AtomicInteger counter = new AtomicInteger(0); // TODO: Need to refactor Logs to keep real it's launchId and combine code with // CleanLaunch to avoid duplication @@ -76,8 +75,6 @@ void removeLogs() { // LOGGER.info("Send event with elements deleted number {} for project {}", deleted, projectId); } }); - - logFinish(counter.get()); } private void deleteLogsFromElasticsearchByLaunchIdsAndProjectId(List launchIds, Long projectId) { diff --git a/src/main/java/com/epam/reportportal/jobs/clean/CleanMaterializedViewJob.java b/src/main/java/com/epam/reportportal/jobs/clean/CleanMaterializedViewJob.java index db4d963..0ce6f53 100644 --- a/src/main/java/com/epam/reportportal/jobs/clean/CleanMaterializedViewJob.java +++ b/src/main/java/com/epam/reportportal/jobs/clean/CleanMaterializedViewJob.java @@ -54,7 +54,6 @@ public CleanMaterializedViewJob(JdbcTemplate jdbcTemplate, @Value("${rp.environm @Scheduled(cron = "${rp.environment.variable.clean.view.cron}") @SchedulerLock(name = "cleanMaterializedView", lockAtMostFor = "24h") public void execute() { - logStart(); final AtomicInteger existingCounter = new AtomicInteger(0); final AtomicInteger staleCounter = new AtomicInteger(0); @@ -75,9 +74,6 @@ public void execute() { staleViews = getStaleViews(timeBound); } - - logFinish(String.format("Stale removed: %d, Existing removed: %d", staleCounter.get(), existingCounter.get())); - } private List getStaleViews(LocalDateTime timeBound) { diff --git a/src/main/java/com/epam/reportportal/jobs/clean/CleanStorageJob.java b/src/main/java/com/epam/reportportal/jobs/clean/CleanStorageJob.java index fb61a06..5fefef7 100644 --- a/src/main/java/com/epam/reportportal/jobs/clean/CleanStorageJob.java +++ b/src/main/java/com/epam/reportportal/jobs/clean/CleanStorageJob.java @@ -58,7 +58,6 @@ public CleanStorageJob(JdbcTemplate jdbcTemplate, DataStorageService storageServ @SchedulerLock(name = "cleanStorage", lockAtMostFor = "24h") @Transactional public void execute() { - logStart(); AtomicInteger counter = new AtomicInteger(0); int batchNumber = 1; @@ -97,8 +96,6 @@ public void execute() { LOGGER.info("Iteration {}, deleted {} attachments", batchNumber, attachmentsSize); batchNumber++; } - - logFinish(counter.get()); } private String decode(String data) { diff --git a/src/main/java/com/epam/reportportal/jobs/clean/DeleteExpiredUsersJob.java b/src/main/java/com/epam/reportportal/jobs/clean/DeleteExpiredUsersJob.java new file mode 100644 index 0000000..a89fcb9 --- /dev/null +++ b/src/main/java/com/epam/reportportal/jobs/clean/DeleteExpiredUsersJob.java @@ -0,0 +1,207 @@ +/* + * Copyright 2023 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.jobs.clean; + +import com.epam.reportportal.analyzer.index.IndexerServiceClient; +import com.epam.reportportal.jobs.BaseJob; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import net.javacrumbs.shedlock.spring.annotation.SchedulerLock; +import org.jclouds.blobstore.BlobStore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +/** + * Deleting Users and their personal project by retention policy. + * + * @author Andrei Piankouski + */ +@Service +public class DeleteExpiredUsersJob extends BaseJob { + + public static final Logger LOGGER = LoggerFactory.getLogger(DeleteExpiredUsersJob.class); + + private final NamedParameterJdbcTemplate namedParameterJdbcTemplate; + + private static final String RETENTION_PERIOD = "retentionPeriod"; + + private static final String SELECT_EXPIRED_USERS = "SELECT u.id AS user_id, p.id AS project_id " + + "FROM users u " + + "LEFT JOIN api_keys ak ON u.id = ak.user_id " + + "LEFT JOIN project p ON u.login || '_personal' = p.name AND p.project_type = 'PERSONAL' " + + "WHERE (u.metadata->'metadata'->>'last_login')::BIGINT <= :retentionPeriod " + + "AND ( " + + "ak.user_id IS NULL " + + "OR (EXTRACT(EPOCH FROM ak.last_used_at) * 1000)::BIGINT <= :retentionPeriod " + + "OR NOT EXISTS (SELECT 1 FROM api_keys WHERE user_id = u.id AND last_used_at IS NOT NULL) " + + ") " + + "AND u.role != 'ADMINISTRATOR' " + + "GROUP BY u.id, p.id"; + + private static final String MOVE_ATTACHMENTS_TO_DELETE = + "WITH moved_rows AS (DELETE FROM attachment " + + "WHERE project_id = :projectId RETURNING id, file_id, thumbnail_id, creation_date) " + + "INSERT INTO attachment_deletion " + + "(id, file_id, thumbnail_id, creation_attachment_date, deletion_date) " + + "SELECT id, file_id, thumbnail_id, creation_date, NOW() FROM moved_rows"; + + private static final String DELETE_PROJECT_ISSUE_TYPES = + "DELETE FROM issue_type " + + "WHERE id IN (" + + " SELECT it.id " + + " FROM issue_type it " + + " JOIN issue_type_project itp ON it.id = itp.issue_type_id " + + " WHERE itp.project_id = :projectId " + + " AND it.locator NOT IN ('pb001', 'ab001', 'si001', 'ti001', 'nd001'))"; + + private static final String DELETE_USERS = "DELETE FROM users WHERE id IN (:userIds)"; + + private static final String DELETE_PROJECTS_BY_ID_LIST = + "DELETE FROM project WHERE id IN (:projectIds)"; + + @Value("${rp.environment.variable.clean.expiredUser.retentionPeriod}") + private long retentionPeriod; + + private final BlobStore blobStore; + + private final IndexerServiceClient indexerServiceClient; + + @Value("${datastore.bucketPrefix}") + private String bucketPrefix; + + @Autowired + public DeleteExpiredUsersJob(JdbcTemplate jdbcTemplate, + NamedParameterJdbcTemplate namedParameterJdbcTemplate, + BlobStore blobStore, IndexerServiceClient indexerServiceClient) { + super(jdbcTemplate); + this.namedParameterJdbcTemplate = namedParameterJdbcTemplate; + this.blobStore = blobStore; + this.indexerServiceClient = indexerServiceClient; + } + + @Scheduled(cron = "${rp.environment.variable.clean.expiredUser.cron}") + @SchedulerLock(name = "deleteExpiredUsers", lockAtMostFor = "24h") + public void execute() { + List userProjects = findUsersAndPersonalProjects(); + List userIds = getUserIds(userProjects); + deleteUsersByIds(userIds); + getProjectIds(userProjects).forEach(this::deleteProjectAssociatedData); + deleteProjectsByIds(getProjectIds(userProjects)); + LOGGER.info("{} - users was deleted due to retention policy", userIds.size()); + } + + private List findUsersAndPersonalProjects() { + MapSqlParameterSource params = new MapSqlParameterSource(); + params.addValue("now_ms", System.currentTimeMillis()); + params.addValue(RETENTION_PERIOD, lastLoginBorder()); + + RowMapper rowMapper = (rs, rowNum) -> { + UserProject userProject = new UserProject(); + userProject.setUserId(rs.getLong("user_id")); + userProject.setProjectId(rs.getLong("project_id")); + return userProject; + }; + + return namedParameterJdbcTemplate.query(SELECT_EXPIRED_USERS, params, rowMapper); + } + + private void deleteProjectAssociatedData(Long projectId) { + deleteAttachmentsByProjectId(projectId); + deleteProjectIssueTypes(projectId); + indexerServiceClient.removeSuggest(projectId); + try { + blobStore.deleteContainer(bucketPrefix + projectId); + } catch (Exception e) { + LOGGER.warn("Cannot delete attachments bucket " + bucketPrefix + projectId); + } + indexerServiceClient.deleteIndex(projectId); + } + + private void deleteUsersByIds(List userIds) { + if (!userIds.isEmpty()) { + MapSqlParameterSource params = new MapSqlParameterSource(); + params.addValue("userIds", userIds); + namedParameterJdbcTemplate.update(DELETE_USERS, params); + } + } + + private void deleteProjectIssueTypes(Long projectId) { + MapSqlParameterSource params = new MapSqlParameterSource(); + params.addValue("projectId", projectId); + namedParameterJdbcTemplate.update(DELETE_PROJECT_ISSUE_TYPES, params); + } + + private void deleteAttachmentsByProjectId(Long projectId) { + MapSqlParameterSource params = new MapSqlParameterSource(); + params.addValue("projectId", projectId); + namedParameterJdbcTemplate.update(MOVE_ATTACHMENTS_TO_DELETE, params); + } + + private void deleteProjectsByIds(List projectIds) { + if (!projectIds.isEmpty()) { + MapSqlParameterSource params = new MapSqlParameterSource(); + params.addValue("projectIds", projectIds); + namedParameterJdbcTemplate.update(DELETE_PROJECTS_BY_ID_LIST, params); + } + } + + private long lastLoginBorder() { + return LocalDateTime.now().minusDays(retentionPeriod).toInstant(ZoneOffset.UTC).toEpochMilli(); + } + + public List getUserIds(List userProjects) { + return userProjects.stream().map(UserProject::getUserId).collect(Collectors.toList()); + } + + public List getProjectIds(List userProjects) { + return userProjects.stream().filter(Objects::nonNull).map(UserProject::getProjectId) + .collect(Collectors.toList()); + } + + private static class UserProject { + + private long userId; + private long projectId; + + public long getUserId() { + return userId; + } + + public void setUserId(long userId) { + this.userId = userId; + } + + public long getProjectId() { + return projectId; + } + + public void setProjectId(long projectId) { + this.projectId = projectId; + } + } +} diff --git a/src/main/java/com/epam/reportportal/jobs/storage/CalculateAllocatedStorageJob.java b/src/main/java/com/epam/reportportal/jobs/storage/CalculateAllocatedStorageJob.java index 32bc71e..a6d4952 100644 --- a/src/main/java/com/epam/reportportal/jobs/storage/CalculateAllocatedStorageJob.java +++ b/src/main/java/com/epam/reportportal/jobs/storage/CalculateAllocatedStorageJob.java @@ -34,11 +34,9 @@ public CalculateAllocatedStorageJob(TaskExecutor projectAllocatedStorageExecutor @Scheduled(cron = "${rp.environment.variable.storage.project.cron}") @SchedulerLock(name = "calculateAllocatedStorage", lockAtMostFor = "24h") public void calculate() { - logStart(); CompletableFuture.allOf(getProjectIds().stream() .map(id -> CompletableFuture.runAsync(() -> updateAllocatedStorage(id), projectAllocatedStorageExecutor)) .toArray(CompletableFuture[]::new)).join(); - logFinish(); } private List getProjectIds() { diff --git a/src/main/java/com/epam/reportportal/logging/ExecutionTimeAspect.java b/src/main/java/com/epam/reportportal/logging/ExecutionTimeAspect.java new file mode 100644 index 0000000..d83f133 --- /dev/null +++ b/src/main/java/com/epam/reportportal/logging/ExecutionTimeAspect.java @@ -0,0 +1,47 @@ +/* + * Copyright 2023 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.logging; + +import net.javacrumbs.shedlock.spring.annotation.SchedulerLock; +import org.aspectj.lang.ProceedingJoinPoint; +import org.aspectj.lang.annotation.Around; +import org.aspectj.lang.annotation.Aspect; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +/** + * @author Andrei Piankouski + */ +@Aspect +@Component +public class ExecutionTimeAspect { + + public static final Logger LOGGER = LoggerFactory.getLogger(ExecutionTimeAspect.class); + + @Around("@annotation(annotation)") + public Object executionTime(ProceedingJoinPoint point, SchedulerLock annotation) throws Throwable { + String name = annotation.name(); + long startTime = System.currentTimeMillis(); + LOGGER.info("Job {} has been started.", name); + Object object = point.proceed(); + long endtime = System.currentTimeMillis(); + + LOGGER.info("Job {} has been finished. Time taken for Execution is : {} ms", name, (endtime-startTime)); + return object; + } +} \ No newline at end of file diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index c6aaff6..8cdcd94 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -8,12 +8,17 @@ rp: clean: storage: ## 30 seconds - cron: '*/30 * * * * *' + cron: '0 0 */24 * * *' chunkSize: 1000 batchSize: 100 - attachment: + expired: ## 2 minutes cron: '0 */2 * * * *' + ## Retentions period in days + retentionPeriod: 1 + attachment: + ## 2 minutes + cron: '0 0 */24 * * *' log: ## 5 minutes cron: '0 */5 * * * *' From ec72ad14bfd6e24001c1508737ad48b1bfa68899 Mon Sep 17 00:00:00 2001 From: Andrei Piankouski Date: Tue, 25 Jul 2023 11:57:14 +0300 Subject: [PATCH 03/27] EPMRPP-85017 || Postfix for bucket names in binary storage --- .../com/epam/reportportal/config/DataStorageConfig.java | 9 +++++++-- .../epam/reportportal/storage/S3DataStorageService.java | 9 ++++++--- src/main/resources/application.yml | 1 + 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/epam/reportportal/config/DataStorageConfig.java b/src/main/java/com/epam/reportportal/config/DataStorageConfig.java index 19cdc53..40894b6 100644 --- a/src/main/java/com/epam/reportportal/config/DataStorageConfig.java +++ b/src/main/java/com/epam/reportportal/config/DataStorageConfig.java @@ -166,6 +166,7 @@ public BlobStore minioBlobStore(@Value("${datastore.accessKey}") String accessKe * * @param blobStore {@link BlobStore} object * @param bucketPrefix Prefix for bucket name + * @param bucketPostfix Postfix for bucket name * @param defaultBucketName Name of default bucket to use * @param featureFlagHandler Instance of {@link FeatureFlagHandler} to check enabled features * @return {@link DataStorageService} object @@ -174,9 +175,11 @@ public BlobStore minioBlobStore(@Value("${datastore.accessKey}") String accessKe @ConditionalOnProperty(name = "datastore.type", havingValue = "minio") public DataStorageService minioDataStore(@Autowired BlobStore blobStore, @Value("${datastore.bucketPrefix}") String bucketPrefix, + @Value("${datastore.bucketPostfix}") String bucketPostfix, @Value("${datastore.defaultBucketName}") String defaultBucketName, FeatureFlagHandler featureFlagHandler) { - return new S3DataStorageService(blobStore, bucketPrefix, defaultBucketName, featureFlagHandler); + return new S3DataStorageService(blobStore, bucketPrefix, bucketPostfix, defaultBucketName, + featureFlagHandler); } /** @@ -206,8 +209,10 @@ public BlobStore blobStore(@Value("${datastore.accessKey}") String accessKey, @ConditionalOnProperty(name = "datastore.type", havingValue = "s3") public DataStorageService s3DataStore(@Autowired BlobStore blobStore, @Value("${datastore.bucketPrefix}") String bucketPrefix, + @Value("${datastore.bucketPostfix}") String bucketPostfix, @Value("${datastore.defaultBucketName}") String defaultBucketName, FeatureFlagHandler featureFlagHandler) { - return new S3DataStorageService(blobStore, bucketPrefix, defaultBucketName, featureFlagHandler); + return new S3DataStorageService(blobStore, bucketPrefix, bucketPostfix, defaultBucketName, + featureFlagHandler); } } diff --git a/src/main/java/com/epam/reportportal/storage/S3DataStorageService.java b/src/main/java/com/epam/reportportal/storage/S3DataStorageService.java index a235465..14163bb 100644 --- a/src/main/java/com/epam/reportportal/storage/S3DataStorageService.java +++ b/src/main/java/com/epam/reportportal/storage/S3DataStorageService.java @@ -38,6 +38,7 @@ public class S3DataStorageService implements DataStorageService { private final BlobStore blobStore; private final String bucketPrefix; + private final String bucketPostfix; private final String defaultBucketName; private final FeatureFlagHandler featureFlagHandler; @@ -47,13 +48,15 @@ public class S3DataStorageService implements DataStorageService { * * @param blobStore {@link BlobStore} * @param bucketPrefix Prefix for bucket name + * @param bucketPostfix Postfix for bucket name * @param defaultBucketName Name for the default bucket(plugins, etc.) * @param featureFlagHandler {@link FeatureFlagHandler} */ - public S3DataStorageService(BlobStore blobStore, String bucketPrefix, String defaultBucketName, - FeatureFlagHandler featureFlagHandler) { + public S3DataStorageService(BlobStore blobStore, String bucketPrefix, String bucketPostfix, + String defaultBucketName, FeatureFlagHandler featureFlagHandler) { this.blobStore = blobStore; this.bucketPrefix = bucketPrefix; + this.bucketPostfix = bucketPostfix; this.defaultBucketName = defaultBucketName; this.featureFlagHandler = featureFlagHandler; } @@ -81,7 +84,7 @@ public void deleteAll(List paths) throws Exception { } } for (Map.Entry> bucketPaths : bucketPathMap.entrySet()) { - removeFiles(bucketPrefix + bucketPaths.getKey(), bucketPaths.getValue()); + removeFiles(bucketPrefix + bucketPaths.getKey() + bucketPostfix, bucketPaths.getValue()); } } } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 8cdcd94..2d23959 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -89,6 +89,7 @@ datastore: accessKey: secretKey: bucketPrefix: prj- + bucketPostfix: defaultBucketName: rp-bucket region: #{null} From 372e1a3c003d7a1c93acf1fcc6ea805afe1e5eb4 Mon Sep 17 00:00:00 2001 From: APiankouski <109206864+APiankouski@users.noreply.github.com> Date: Wed, 2 Aug 2023 16:59:54 +0300 Subject: [PATCH 04/27] EPMRPP-83085 || Implement user notification about account deletion due to retention policy (#85) * EPMRPP-83085 || Implement user notification about account deletion due to retention policy * EPMRPP-83085 || Add user deletion notification * EPMRPP-83085 || Fix checkstyle * EPMRPP-83085 || Fix checkstyle --------- Co-authored-by: Andrei Piankouski --- .../index/IndexerServiceClientImpl.java | 2 +- .../config/rabbit/InternalConfiguration.java | 60 ++++++++ .../ProcessingRabbitMqConfiguration.java | 71 --------- .../config/rabbit/RabbitMqConfiguration.java | 95 ++++++++++++ .../com/epam/reportportal/jobs/BaseJob.java | 2 + .../reportportal/jobs/clean/BaseCleanJob.java | 2 +- .../jobs/clean/CleanAttachmentJob.java | 1 + .../jobs/clean/CleanLaunchJob.java | 1 + .../reportportal/jobs/clean/CleanLogJob.java | 1 + .../jobs/clean/CleanMaterializedViewJob.java | 1 + .../jobs/clean/CleanStorageJob.java | 1 + .../jobs/clean/DeleteExpiredUsersJob.java | 46 +++++- .../notification/NotifyUserExpirationJob.java | 135 ++++++++++++++++++ .../jobs/processing/SaveLogMessageJob.java | 25 ++-- .../storage/CalculateAllocatedStorageJob.java | 2 +- .../model/EmailNotificationRequest.java | 71 +++++++++ src/main/resources/application.yml | 13 +- .../CalculateAllocatedStorageJobTest.java | 2 +- 18 files changed, 435 insertions(+), 96 deletions(-) create mode 100644 src/main/java/com/epam/reportportal/config/rabbit/InternalConfiguration.java delete mode 100644 src/main/java/com/epam/reportportal/config/rabbit/ProcessingRabbitMqConfiguration.java create mode 100644 src/main/java/com/epam/reportportal/config/rabbit/RabbitMqConfiguration.java create mode 100644 src/main/java/com/epam/reportportal/jobs/notification/NotifyUserExpirationJob.java create mode 100644 src/main/java/com/epam/reportportal/model/EmailNotificationRequest.java diff --git a/src/main/java/com/epam/reportportal/analyzer/index/IndexerServiceClientImpl.java b/src/main/java/com/epam/reportportal/analyzer/index/IndexerServiceClientImpl.java index 363dd5c..45e036d 100644 --- a/src/main/java/com/epam/reportportal/analyzer/index/IndexerServiceClientImpl.java +++ b/src/main/java/com/epam/reportportal/analyzer/index/IndexerServiceClientImpl.java @@ -46,7 +46,7 @@ public class IndexerServiceClientImpl implements IndexerServiceClient { @Autowired public IndexerServiceClientImpl(RabbitMqManagementClient rabbitMqManagementClient, - @Qualifier("analyzerRabbitTemplate") RabbitTemplate rabbitTemplate) { + @Qualifier("rabbitTemplate") RabbitTemplate rabbitTemplate) { this.rabbitMqManagementClient = rabbitMqManagementClient; this.rabbitTemplate = rabbitTemplate; } diff --git a/src/main/java/com/epam/reportportal/config/rabbit/InternalConfiguration.java b/src/main/java/com/epam/reportportal/config/rabbit/InternalConfiguration.java new file mode 100644 index 0000000..c72013e --- /dev/null +++ b/src/main/java/com/epam/reportportal/config/rabbit/InternalConfiguration.java @@ -0,0 +1,60 @@ +/* + * Copyright 2023 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.config.rabbit; + +import org.springframework.amqp.core.Binding; +import org.springframework.amqp.core.BindingBuilder; +import org.springframework.amqp.core.DirectExchange; +import org.springframework.amqp.core.Queue; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * RabbitMQ queue and exchange configuration. + * + * @author Andrei Piankouski + */ +@Configuration +public class InternalConfiguration { + + /** + * Exchanges. + */ + public static final String EXCHANGE_NOTIFICATION = "notification"; + + /** + * Queues. + */ + public static final String QUEUE_EMAIL = "notification.email"; + + @Bean + Queue emailNotificationQueue() { + return new Queue(QUEUE_EMAIL); + } + + @Bean + DirectExchange notificationExchange() { + return new DirectExchange(EXCHANGE_NOTIFICATION); + } + + @Bean + public Binding emailNotificationBinding() { + return BindingBuilder.bind(emailNotificationQueue()).to(notificationExchange()) + .with(QUEUE_EMAIL); + } + +} diff --git a/src/main/java/com/epam/reportportal/config/rabbit/ProcessingRabbitMqConfiguration.java b/src/main/java/com/epam/reportportal/config/rabbit/ProcessingRabbitMqConfiguration.java deleted file mode 100644 index 25bea9e..0000000 --- a/src/main/java/com/epam/reportportal/config/rabbit/ProcessingRabbitMqConfiguration.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright 2019 EPAM Systems - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.epam.reportportal.config.rabbit; - -import org.springframework.amqp.rabbit.annotation.EnableRabbit; -import org.springframework.amqp.rabbit.config.SimpleRabbitListenerContainerFactory; -import org.springframework.amqp.rabbit.connection.CachingConnectionFactory; -import org.springframework.amqp.rabbit.connection.ConnectionFactory; -import org.springframework.amqp.rabbit.core.RabbitAdmin; -import org.springframework.amqp.support.converter.MessageConverter; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.amqp.SimpleRabbitListenerContainerFactoryConfigurer; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -import java.net.URI; - -/** - * @author Pavel Bortnik - */ -@EnableRabbit -@Configuration -public class ProcessingRabbitMqConfiguration { - - @Bean(name = "processingConnectionFactory") - public ConnectionFactory processingConnectionFactory(@Value("${rp.amqp.addresses}") URI addresses, - @Value("${rp.amqp.base-vhost}") String virtualHost) { - CachingConnectionFactory factory = new CachingConnectionFactory(addresses); - factory.setVirtualHost(virtualHost); - return factory; - } - - @Bean - public SimpleRabbitListenerContainerFactory processingRabbitListenerContainerFactory( - @Qualifier("processingConnectionFactory") ConnectionFactory connectionFactory, MessageConverter jsonMessageConverter, - @Value("${rp.amqp.maxLogConsumer}") int maxLogConsumer) { - SimpleRabbitListenerContainerFactory factory = new SimpleRabbitListenerContainerFactory(); - factory.setConnectionFactory(connectionFactory); - factory.setMaxConcurrentConsumers(maxLogConsumer); - factory.setMessageConverter(jsonMessageConverter); - return factory; - } - - @Bean - public RabbitAdmin processingRabbitAdmin(@Qualifier("processingConnectionFactory") ConnectionFactory connectionFactory) { - return new RabbitAdmin(connectionFactory); - } - - @Bean - SimpleRabbitListenerContainerFactory rabbitListenerContainerFactory( - SimpleRabbitListenerContainerFactoryConfigurer configurer, @Qualifier("processingConnectionFactory") ConnectionFactory connectionFactory) { - SimpleRabbitListenerContainerFactory factory = new SimpleRabbitListenerContainerFactory(); - configurer.configure(factory, connectionFactory); - return factory; - } -} diff --git a/src/main/java/com/epam/reportportal/config/rabbit/RabbitMqConfiguration.java b/src/main/java/com/epam/reportportal/config/rabbit/RabbitMqConfiguration.java new file mode 100644 index 0000000..7c4c1f4 --- /dev/null +++ b/src/main/java/com/epam/reportportal/config/rabbit/RabbitMqConfiguration.java @@ -0,0 +1,95 @@ +/* + * Copyright 2019 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.config.rabbit; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.net.URI; +import org.springframework.amqp.rabbit.annotation.EnableRabbit; +import org.springframework.amqp.rabbit.config.SimpleRabbitListenerContainerFactory; +import org.springframework.amqp.rabbit.connection.CachingConnectionFactory; +import org.springframework.amqp.rabbit.connection.ConnectionFactory; +import org.springframework.amqp.rabbit.core.RabbitAdmin; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.amqp.support.converter.Jackson2JsonMessageConverter; +import org.springframework.amqp.support.converter.MessageConverter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.amqp.SimpleRabbitListenerContainerFactoryConfigurer; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + + +/** + * @author Pavel Bortnik + */ +@EnableRabbit +@Configuration +public class RabbitMqConfiguration { + + private final ObjectMapper objectMapper; + + @Autowired + public RabbitMqConfiguration(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + } + + @Bean(name = "connectionFactory") + public ConnectionFactory connectionFactory(@Value("${rp.amqp.addresses}") URI addresses, + @Value("${rp.amqp.base-vhost}") String virtualHost) { + CachingConnectionFactory factory = new CachingConnectionFactory(addresses); + factory.setVirtualHost(virtualHost); + return factory; + } + + @Bean + public SimpleRabbitListenerContainerFactory rabbitListenerContainerFactory( + @Qualifier("connectionFactory") ConnectionFactory connectionFactory, + MessageConverter jsonMessageConverter, + @Value("${rp.amqp.maxLogConsumer}") int maxLogConsumer) { + SimpleRabbitListenerContainerFactory factory = new SimpleRabbitListenerContainerFactory(); + factory.setConnectionFactory(connectionFactory); + factory.setMaxConcurrentConsumers(maxLogConsumer); + factory.setMessageConverter(jsonMessageConverter); + return factory; + } + + @Bean + public RabbitAdmin rabbitAdmin( + @Qualifier("connectionFactory") ConnectionFactory connectionFactory) { + return new RabbitAdmin(connectionFactory); + } + + @Bean + SimpleRabbitListenerContainerFactory rabbitListenerContainerFactory( + SimpleRabbitListenerContainerFactoryConfigurer configurer, + @Qualifier("connectionFactory") ConnectionFactory connectionFactory) { + SimpleRabbitListenerContainerFactory factory = new SimpleRabbitListenerContainerFactory(); + configurer.configure(factory, connectionFactory); + return factory; + } + + @Bean(name = "rabbitTemplate") + public RabbitTemplate rabbitTemplate( + @Autowired @Qualifier("connectionFactory") ConnectionFactory connectionFactory, + @Value("${rp.amqp.reply-timeout}") long replyTimeout) { + RabbitTemplate rabbitTemplate = new RabbitTemplate(connectionFactory); + rabbitTemplate.setMessageConverter(new Jackson2JsonMessageConverter(objectMapper)); + rabbitTemplate.setReplyTimeout(replyTimeout); + return rabbitTemplate; + } +} diff --git a/src/main/java/com/epam/reportportal/jobs/BaseJob.java b/src/main/java/com/epam/reportportal/jobs/BaseJob.java index 76da27d..8f5ad0f 100644 --- a/src/main/java/com/epam/reportportal/jobs/BaseJob.java +++ b/src/main/java/com/epam/reportportal/jobs/BaseJob.java @@ -11,4 +11,6 @@ public abstract class BaseJob { public BaseJob(JdbcTemplate jdbcTemplate) { this.jdbcTemplate = jdbcTemplate; } + + public abstract void execute(); } diff --git a/src/main/java/com/epam/reportportal/jobs/clean/BaseCleanJob.java b/src/main/java/com/epam/reportportal/jobs/clean/BaseCleanJob.java index aa775be..af9ba03 100644 --- a/src/main/java/com/epam/reportportal/jobs/clean/BaseCleanJob.java +++ b/src/main/java/com/epam/reportportal/jobs/clean/BaseCleanJob.java @@ -12,7 +12,7 @@ /** * @author Pavel Bortnik */ -public class BaseCleanJob extends BaseJob { +public abstract class BaseCleanJob extends BaseJob { protected static final String KEEP_LAUNCHES = "job.keepLaunches"; protected static final String KEEP_LOGS = "job.keepLogs"; diff --git a/src/main/java/com/epam/reportportal/jobs/clean/CleanAttachmentJob.java b/src/main/java/com/epam/reportportal/jobs/clean/CleanAttachmentJob.java index 9c1be1a..b070f4b 100644 --- a/src/main/java/com/epam/reportportal/jobs/clean/CleanAttachmentJob.java +++ b/src/main/java/com/epam/reportportal/jobs/clean/CleanAttachmentJob.java @@ -27,6 +27,7 @@ public CleanAttachmentJob(JdbcTemplate jdbcTemplate) { super(jdbcTemplate); } + @Override @Scheduled(cron = "${rp.environment.variable.clean.attachment.cron}") @SchedulerLock(name = "cleanAttachment", lockAtMostFor = "24h") public void execute() { diff --git a/src/main/java/com/epam/reportportal/jobs/clean/CleanLaunchJob.java b/src/main/java/com/epam/reportportal/jobs/clean/CleanLaunchJob.java index 2b193bf..13da6e8 100644 --- a/src/main/java/com/epam/reportportal/jobs/clean/CleanLaunchJob.java +++ b/src/main/java/com/epam/reportportal/jobs/clean/CleanLaunchJob.java @@ -53,6 +53,7 @@ public CleanLaunchJob(@Value("${rp.environment.variable.elements-counter.batch-s this.elasticSearchClient = elasticSearchClient; } + @Override @Scheduled(cron = "${rp.environment.variable.clean.launch.cron}") @SchedulerLock(name = "cleanLaunch", lockAtMostFor = "24h") public void execute() { diff --git a/src/main/java/com/epam/reportportal/jobs/clean/CleanLogJob.java b/src/main/java/com/epam/reportportal/jobs/clean/CleanLogJob.java index a93f626..4564435 100644 --- a/src/main/java/com/epam/reportportal/jobs/clean/CleanLogJob.java +++ b/src/main/java/com/epam/reportportal/jobs/clean/CleanLogJob.java @@ -45,6 +45,7 @@ public CleanLogJob(JdbcTemplate jdbcTemplate, CleanAttachmentJob cleanAttachment this.namedParameterJdbcTemplate = namedParameterJdbcTemplate; } + @Override @Scheduled(cron = "${rp.environment.variable.clean.log.cron}") @SchedulerLock(name = "cleanLog", lockAtMostFor = "24h") public void execute() { diff --git a/src/main/java/com/epam/reportportal/jobs/clean/CleanMaterializedViewJob.java b/src/main/java/com/epam/reportportal/jobs/clean/CleanMaterializedViewJob.java index 0ce6f53..7caa11b 100644 --- a/src/main/java/com/epam/reportportal/jobs/clean/CleanMaterializedViewJob.java +++ b/src/main/java/com/epam/reportportal/jobs/clean/CleanMaterializedViewJob.java @@ -51,6 +51,7 @@ public CleanMaterializedViewJob(JdbcTemplate jdbcTemplate, @Value("${rp.environm this.namedParameterJdbcTemplate = namedParameterJdbcTemplate; } + @Override @Scheduled(cron = "${rp.environment.variable.clean.view.cron}") @SchedulerLock(name = "cleanMaterializedView", lockAtMostFor = "24h") public void execute() { diff --git a/src/main/java/com/epam/reportportal/jobs/clean/CleanStorageJob.java b/src/main/java/com/epam/reportportal/jobs/clean/CleanStorageJob.java index 5fefef7..106e27c 100644 --- a/src/main/java/com/epam/reportportal/jobs/clean/CleanStorageJob.java +++ b/src/main/java/com/epam/reportportal/jobs/clean/CleanStorageJob.java @@ -54,6 +54,7 @@ public CleanStorageJob(JdbcTemplate jdbcTemplate, DataStorageService storageServ /** * Deletes attachments, which are set to be deleted. */ + @Override @Scheduled(cron = "${rp.environment.variable.clean.storage.cron}") @SchedulerLock(name = "cleanStorage", lockAtMostFor = "24h") @Transactional diff --git a/src/main/java/com/epam/reportportal/jobs/clean/DeleteExpiredUsersJob.java b/src/main/java/com/epam/reportportal/jobs/clean/DeleteExpiredUsersJob.java index a89fcb9..a77039d 100644 --- a/src/main/java/com/epam/reportportal/jobs/clean/DeleteExpiredUsersJob.java +++ b/src/main/java/com/epam/reportportal/jobs/clean/DeleteExpiredUsersJob.java @@ -16,18 +16,27 @@ package com.epam.reportportal.jobs.clean; +import static com.epam.reportportal.config.rabbit.InternalConfiguration.EXCHANGE_NOTIFICATION; +import static com.epam.reportportal.config.rabbit.InternalConfiguration.QUEUE_EMAIL; + import com.epam.reportportal.analyzer.index.IndexerServiceClient; import com.epam.reportportal.jobs.BaseJob; +import com.epam.reportportal.model.EmailNotificationRequest; +import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneOffset; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; import net.javacrumbs.shedlock.spring.annotation.SchedulerLock; import org.jclouds.blobstore.BlobStore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.RowMapper; @@ -48,9 +57,11 @@ public class DeleteExpiredUsersJob extends BaseJob { private final NamedParameterJdbcTemplate namedParameterJdbcTemplate; + public static final String USER_DELETION_TEMPLATE = "userDeletionNotification"; private static final String RETENTION_PERIOD = "retentionPeriod"; - private static final String SELECT_EXPIRED_USERS = "SELECT u.id AS user_id, p.id AS project_id " + private static final String SELECT_EXPIRED_USERS = "SELECT u.id AS user_id, " + + "p.id AS project_id, u.email as user_email " + "FROM users u " + "LEFT JOIN api_keys ak ON u.id = ak.user_id " + "LEFT JOIN project p ON u.login || '_personal' = p.name AND p.project_type = 'PERSONAL' " @@ -91,19 +102,24 @@ public class DeleteExpiredUsersJob extends BaseJob { private final IndexerServiceClient indexerServiceClient; + private final RabbitTemplate rabbitTemplate; + @Value("${datastore.bucketPrefix}") private String bucketPrefix; @Autowired public DeleteExpiredUsersJob(JdbcTemplate jdbcTemplate, NamedParameterJdbcTemplate namedParameterJdbcTemplate, - BlobStore blobStore, IndexerServiceClient indexerServiceClient) { + BlobStore blobStore, IndexerServiceClient indexerServiceClient, + @Qualifier("rabbitTemplate") RabbitTemplate rabbitTemplate) { super(jdbcTemplate); this.namedParameterJdbcTemplate = namedParameterJdbcTemplate; this.blobStore = blobStore; this.indexerServiceClient = indexerServiceClient; + this.rabbitTemplate = rabbitTemplate; } + @Override @Scheduled(cron = "${rp.environment.variable.clean.expiredUser.cron}") @SchedulerLock(name = "deleteExpiredUsers", lockAtMostFor = "24h") public void execute() { @@ -112,6 +128,7 @@ public void execute() { deleteUsersByIds(userIds); getProjectIds(userProjects).forEach(this::deleteProjectAssociatedData); deleteProjectsByIds(getProjectIds(userProjects)); + sendNotificationEmail(getUserEmails(userProjects)); LOGGER.info("{} - users was deleted due to retention policy", userIds.size()); } @@ -174,19 +191,32 @@ private long lastLoginBorder() { return LocalDateTime.now().minusDays(retentionPeriod).toInstant(ZoneOffset.UTC).toEpochMilli(); } - public List getUserIds(List userProjects) { + private List getUserIds(List userProjects) { return userProjects.stream().map(UserProject::getUserId).collect(Collectors.toList()); } - public List getProjectIds(List userProjects) { + private List getUserEmails(List userProjects) { + return userProjects.stream().map(UserProject::getEmail).collect(Collectors.toList()); + } + + private List getProjectIds(List userProjects) { return userProjects.stream().filter(Objects::nonNull).map(UserProject::getProjectId) .collect(Collectors.toList()); } + private void sendNotificationEmail(List recipients) { + for (String recipient : recipients) { + EmailNotificationRequest notification = + new EmailNotificationRequest(recipient, USER_DELETION_TEMPLATE); + rabbitTemplate.convertAndSend(EXCHANGE_NOTIFICATION, QUEUE_EMAIL, notification); + } + } + private static class UserProject { private long userId; private long projectId; + private String email; public long getUserId() { return userId; @@ -196,6 +226,14 @@ public void setUserId(long userId) { this.userId = userId; } + public String getEmail() { + return email; + } + + public void setEmail(String email) { + this.email = email; + } + public long getProjectId() { return projectId; } diff --git a/src/main/java/com/epam/reportportal/jobs/notification/NotifyUserExpirationJob.java b/src/main/java/com/epam/reportportal/jobs/notification/NotifyUserExpirationJob.java new file mode 100644 index 0000000..e98dc99 --- /dev/null +++ b/src/main/java/com/epam/reportportal/jobs/notification/NotifyUserExpirationJob.java @@ -0,0 +1,135 @@ +/* + * Copyright 2023 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.jobs.notification; + +import static com.epam.reportportal.config.rabbit.InternalConfiguration.EXCHANGE_NOTIFICATION; +import static com.epam.reportportal.config.rabbit.InternalConfiguration.QUEUE_EMAIL; + +import com.epam.reportportal.jobs.BaseJob; +import com.epam.reportportal.model.EmailNotificationRequest; +import java.time.LocalDate; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import net.javacrumbs.shedlock.spring.annotation.SchedulerLock; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +/** + * Notify users of oncoming deletion according to retention policy. + * + * @author Andrei Piankouski + */ +@Service +public class NotifyUserExpirationJob extends BaseJob { + + private static final String EMAIL = "email"; + private static final String USER_EXPIRATION_TEMPLATE = "userExpirationNotification"; + private static final String DAYS = " days"; + private static final String INACTIVITY_PERIOD = "inactivityPeriod"; + private static final String REMAINING_TIME = "remainingTime"; + private static final String DEADLINE_DATE = "deadlineDate"; + + private static final String SELECT_USERS_FOR_NOTIFY = "WITH user_last_action AS ( " + + "SELECT " + + " u.id as user_id, " + + " u.email as email, " + + "DATE_PART('day', NOW() - GREATEST(" + + " to_timestamp(CAST(u.metadata->'metadata'->>'last_login' AS bigint) / 1000), " + + "MAX(ak.last_used_at))) AS inactivityPeriod " + + "FROM " + + " users u " + + " LEFT JOIN api_keys ak ON u.id = ak.user_id " + + "WHERE " + + " u.role != 'ADMINISTRATOR' " + + "GROUP BY " + + " u.id " + + ") " + + "SELECT " + + " user_last_action.user_id, " + + " user_last_action.email, " + + " user_last_action.inactivityPeriod, " + + " :retentionPeriod - inactivityPeriod IN (1, 30, 60) as remainingTime " + + "FROM " + + " user_last_action " + + "WHERE " + + " :retentionPeriod - inactivityPeriod IN (1, 30, 60)"; + + private static final String RETENTION_PERIOD = "retentionPeriod"; + + @Value("${rp.environment.variable.clean.expiredUser.retentionPeriod}") + private long retentionPeriod; + + private final NamedParameterJdbcTemplate namedParameterJdbcTemplate; + + private final RabbitTemplate rabbitTemplate; + + @Autowired + public NotifyUserExpirationJob(JdbcTemplate jdbcTemplate, + NamedParameterJdbcTemplate namedParameterJdbcTemplate, + @Qualifier("rabbitTemplate") RabbitTemplate rabbitTemplate) { + super(jdbcTemplate); + this.namedParameterJdbcTemplate = namedParameterJdbcTemplate; + this.rabbitTemplate = rabbitTemplate; + } + + @Override + @Scheduled(cron = "${rp.environment.variable.notification.expiredUser.cron}") + @SchedulerLock(name = "notifyUserExpiration") + public void execute() { + List notifications = getUsersForNotify(); + notifications.forEach( + notification -> rabbitTemplate.convertAndSend(EXCHANGE_NOTIFICATION, QUEUE_EMAIL, + notification)); + } + + private String getRemainingTime(long remainingTime) { + if (remainingTime == 1) { + return "tomorrow"; + } else if (remainingTime == 30) { + return "1 month"; + } else if (remainingTime == 60) { + return "2 months"; + } else { + return remainingTime + DAYS; + } + } + + private List getUsersForNotify() { + MapSqlParameterSource parameters = new MapSqlParameterSource(); + parameters.addValue(RETENTION_PERIOD, retentionPeriod); + return namedParameterJdbcTemplate.query( + SELECT_USERS_FOR_NOTIFY, parameters, (rs, rowNum) -> { + Map params = new HashMap<>(); + params.put(INACTIVITY_PERIOD, rs.getLong(INACTIVITY_PERIOD) + DAYS); + params.put(REMAINING_TIME, getRemainingTime(rs.getLong(REMAINING_TIME))); + params.put(DEADLINE_DATE, + String.valueOf(LocalDate.now().plusDays(rs.getLong(REMAINING_TIME)))); + EmailNotificationRequest emailNotificationRequest = + new EmailNotificationRequest(rs.getString(EMAIL), USER_EXPIRATION_TEMPLATE); + emailNotificationRequest.setParams(params); + return emailNotificationRequest; + }); + } +} diff --git a/src/main/java/com/epam/reportportal/jobs/processing/SaveLogMessageJob.java b/src/main/java/com/epam/reportportal/jobs/processing/SaveLogMessageJob.java index e2c92fe..4a93e6b 100644 --- a/src/main/java/com/epam/reportportal/jobs/processing/SaveLogMessageJob.java +++ b/src/main/java/com/epam/reportportal/jobs/processing/SaveLogMessageJob.java @@ -2,13 +2,12 @@ import com.epam.reportportal.log.LogMessage; import com.epam.reportportal.log.LogProcessing; +import java.util.Objects; import org.springframework.amqp.rabbit.annotation.RabbitListener; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.messaging.handler.annotation.Payload; import org.springframework.stereotype.Service; -import java.util.Objects; - /** * Log consumer. * @@ -17,17 +16,19 @@ @Service @ConditionalOnProperty(prefix = "rp.elasticsearch", name = "host") public class SaveLogMessageJob { - public static final String LOG_MESSAGE_SAVING_QUEUE_NAME = "log_message_saving"; - private final LogProcessing logProcessing; - public SaveLogMessageJob(LogProcessing logProcessing) { - this.logProcessing = logProcessing; - } + public static final String LOG_MESSAGE_SAVING_QUEUE_NAME = "log_message_saving"; + private final LogProcessing logProcessing; + + public SaveLogMessageJob(LogProcessing logProcessing) { + this.logProcessing = logProcessing; + } - @RabbitListener(queues = LOG_MESSAGE_SAVING_QUEUE_NAME, containerFactory = "processingRabbitListenerContainerFactory") - public void execute(@Payload LogMessage logMessage) { - if (Objects.nonNull(logMessage)) { - this.logProcessing.add(logMessage); - } + @RabbitListener(queues = LOG_MESSAGE_SAVING_QUEUE_NAME, + containerFactory = "rabbitListenerContainerFactory") + public void execute(@Payload LogMessage logMessage) { + if (Objects.nonNull(logMessage)) { + this.logProcessing.add(logMessage); } + } } diff --git a/src/main/java/com/epam/reportportal/jobs/storage/CalculateAllocatedStorageJob.java b/src/main/java/com/epam/reportportal/jobs/storage/CalculateAllocatedStorageJob.java index a6d4952..c145872 100644 --- a/src/main/java/com/epam/reportportal/jobs/storage/CalculateAllocatedStorageJob.java +++ b/src/main/java/com/epam/reportportal/jobs/storage/CalculateAllocatedStorageJob.java @@ -33,7 +33,7 @@ public CalculateAllocatedStorageJob(TaskExecutor projectAllocatedStorageExecutor @Scheduled(cron = "${rp.environment.variable.storage.project.cron}") @SchedulerLock(name = "calculateAllocatedStorage", lockAtMostFor = "24h") - public void calculate() { + public void execute() { CompletableFuture.allOf(getProjectIds().stream() .map(id -> CompletableFuture.runAsync(() -> updateAllocatedStorage(id), projectAllocatedStorageExecutor)) .toArray(CompletableFuture[]::new)).join(); diff --git a/src/main/java/com/epam/reportportal/model/EmailNotificationRequest.java b/src/main/java/com/epam/reportportal/model/EmailNotificationRequest.java new file mode 100644 index 0000000..2aa4b8c --- /dev/null +++ b/src/main/java/com/epam/reportportal/model/EmailNotificationRequest.java @@ -0,0 +1,71 @@ +/* + * Copyright 2023 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.model; + +import java.util.Map; + +/** + * EmailNotification model for rabbitMq topic. + * + * @author Andrei Piankouski + */ +public class EmailNotificationRequest { + + private String recipient; + + private String template; + + private Map params; + + public EmailNotificationRequest(String recipient, String template) { + this.recipient = recipient; + this.template = template; + } + + public String getRecipient() { + return recipient; + } + + public void setRecipient(String recipient) { + this.recipient = recipient; + } + + public String getTemplate() { + return template; + } + + public void setTemplate(String template) { + this.template = template; + } + + public Map getParams() { + return params; + } + + public void setParams(Map params) { + this.params = params; + } + + @Override + public String toString() { + return "EmailNotificationRequest{" + + "recipient='" + recipient + '\'' + + ", template='" + template + '\'' + + ", params=" + params + + '}'; + } +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 2d23959..17cca7a 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -5,19 +5,23 @@ rp: variable: elements-counter: batch-size: 50 + notification: + expiredUser: + ## 24 hours + cron: '0 0 */24 * * *' clean: storage: - ## 30 seconds + ## 24 hours cron: '0 0 */24 * * *' chunkSize: 1000 batchSize: 100 expired: - ## 2 minutes - cron: '0 */2 * * * *' + ## 24 hours + cron: '0 0 */24 * * *' ## Retentions period in days retentionPeriod: 1 attachment: - ## 2 minutes + ## 24 hours cron: '0 0 */24 * * *' log: ## 5 minutes @@ -28,7 +32,6 @@ rp: view: ## 24 hours cron: '0 0 */24 * * *' - ## 2 hours liveTimeout: 7200 batch: 100 storage: diff --git a/src/test/java/com/epam/reportportal/jobs/storage/CalculateAllocatedStorageJobTest.java b/src/test/java/com/epam/reportportal/jobs/storage/CalculateAllocatedStorageJobTest.java index d98cc33..6ee3940 100644 --- a/src/test/java/com/epam/reportportal/jobs/storage/CalculateAllocatedStorageJobTest.java +++ b/src/test/java/com/epam/reportportal/jobs/storage/CalculateAllocatedStorageJobTest.java @@ -52,7 +52,7 @@ void shouldUpdateAllocatedStorageForAllProjects() { when(jdbcTemplate.queryForList(SELECT_PROJECT_IDS_QUERY, Long.class)).thenReturn(projectIds); when(jdbcTemplate.queryForObject(eq(SELECT_FILE_SIZE_SUM_BY_PROJECT_ID_QUERY), eq(Long.class), anyLong())).thenReturn(1000L); - calculateAllocatedStorageJob.calculate(); + calculateAllocatedStorageJob.execute(); verify(jdbcTemplate, times(2)).queryForObject(eq(SELECT_FILE_SIZE_SUM_BY_PROJECT_ID_QUERY), eq(Long.class), anyLong()); From 5cbbce7309091de50eeeb4dac9a322134d20c6ec Mon Sep 17 00:00:00 2001 From: Andrei Piankouski Date: Thu, 3 Aug 2023 17:29:39 +0300 Subject: [PATCH 05/27] NotifyUserExpiration || NotifyUserExpiration job fails when sending email --- .../reportportal/jobs/notification/NotifyUserExpirationJob.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/epam/reportportal/jobs/notification/NotifyUserExpirationJob.java b/src/main/java/com/epam/reportportal/jobs/notification/NotifyUserExpirationJob.java index e98dc99..2f1c62d 100644 --- a/src/main/java/com/epam/reportportal/jobs/notification/NotifyUserExpirationJob.java +++ b/src/main/java/com/epam/reportportal/jobs/notification/NotifyUserExpirationJob.java @@ -70,7 +70,7 @@ public class NotifyUserExpirationJob extends BaseJob { + " user_last_action.user_id, " + " user_last_action.email, " + " user_last_action.inactivityPeriod, " - + " :retentionPeriod - inactivityPeriod IN (1, 30, 60) as remainingTime " + + " :retentionPeriod - inactivityPeriod as remainingTime " + "FROM " + " user_last_action " + "WHERE " From eba68597639cc10daa2d87f9f80eec6c1245225a Mon Sep 17 00:00:00 2001 From: Andrei Piankouski Date: Fri, 4 Aug 2023 11:01:45 +0300 Subject: [PATCH 06/27] NotifyUserExpiration || NotifyUserExpiration job fails when sending email --- .../notification/NotifyUserExpirationJob.java | 47 ++++++++++++------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/src/main/java/com/epam/reportportal/jobs/notification/NotifyUserExpirationJob.java b/src/main/java/com/epam/reportportal/jobs/notification/NotifyUserExpirationJob.java index 2f1c62d..36e6e51 100644 --- a/src/main/java/com/epam/reportportal/jobs/notification/NotifyUserExpirationJob.java +++ b/src/main/java/com/epam/reportportal/jobs/notification/NotifyUserExpirationJob.java @@ -46,10 +46,10 @@ public class NotifyUserExpirationJob extends BaseJob { private static final String EMAIL = "email"; private static final String USER_EXPIRATION_TEMPLATE = "userExpirationNotification"; - private static final String DAYS = " days"; private static final String INACTIVITY_PERIOD = "inactivityPeriod"; private static final String REMAINING_TIME = "remainingTime"; private static final String DEADLINE_DATE = "deadlineDate"; + private static final String DAYS = " days"; private static final String SELECT_USERS_FOR_NOTIFY = "WITH user_last_action AS ( " + "SELECT " @@ -96,7 +96,7 @@ public NotifyUserExpirationJob(JdbcTemplate jdbcTemplate, @Override @Scheduled(cron = "${rp.environment.variable.notification.expiredUser.cron}") - @SchedulerLock(name = "notifyUserExpiration") + @SchedulerLock(name = "notifyUserExpiration", lockAtMostFor = "24h") public void execute() { List notifications = getUsersForNotify(); notifications.forEach( @@ -104,32 +104,43 @@ public void execute() { notification)); } - private String getRemainingTime(long remainingTime) { - if (remainingTime == 1) { - return "tomorrow"; - } else if (remainingTime == 30) { - return "1 month"; - } else if (remainingTime == 60) { - return "2 months"; - } else { - return remainingTime + DAYS; - } - } - private List getUsersForNotify() { MapSqlParameterSource parameters = new MapSqlParameterSource(); parameters.addValue(RETENTION_PERIOD, retentionPeriod); return namedParameterJdbcTemplate.query( SELECT_USERS_FOR_NOTIFY, parameters, (rs, rowNum) -> { Map params = new HashMap<>(); - params.put(INACTIVITY_PERIOD, rs.getLong(INACTIVITY_PERIOD) + DAYS); - params.put(REMAINING_TIME, getRemainingTime(rs.getLong(REMAINING_TIME))); - params.put(DEADLINE_DATE, - String.valueOf(LocalDate.now().plusDays(rs.getLong(REMAINING_TIME)))); + params.put(INACTIVITY_PERIOD, getInactivityPeriod(rs.getInt(INACTIVITY_PERIOD))); + params.put(REMAINING_TIME, getRemainingTime(rs.getInt(REMAINING_TIME))); + params.put(DEADLINE_DATE, getDeadlineDate(rs.getInt(REMAINING_TIME))); EmailNotificationRequest emailNotificationRequest = new EmailNotificationRequest(rs.getString(EMAIL), USER_EXPIRATION_TEMPLATE); emailNotificationRequest.setParams(params); return emailNotificationRequest; }); } + + private String getRemainingTime(int remainingTime) { + if (remainingTime == 1) { + return "tomorrow"; + } else if (remainingTime == 30) { + return "1 month"; + } else if (remainingTime == 60) { + return "2 months"; + } else { + return remainingTime + DAYS; + } + } + + private String getDeadlineDate(int remainingTime) { + return remainingTime == 1 + ? "today" + : "before " + LocalDate.now().plusDays(remainingTime) + ""; + } + + private String getInactivityPeriod(int inactivityPeriod) { + int inactivityMouths = inactivityPeriod / 30; + return retentionPeriod - inactivityPeriod == 1 ? "almost " + retentionPeriod / 30 + " months" + : inactivityMouths + " months"; + } } From b85a66b985825447a87af8b94cdf3dddc96e9845 Mon Sep 17 00:00:00 2001 From: Andrei Piankouski Date: Fri, 4 Aug 2023 12:37:13 +0300 Subject: [PATCH 07/27] EPMRPP-83085 || Fix date calculation --- .../reportportal/jobs/notification/NotifyUserExpirationJob.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/epam/reportportal/jobs/notification/NotifyUserExpirationJob.java b/src/main/java/com/epam/reportportal/jobs/notification/NotifyUserExpirationJob.java index 36e6e51..5ad5028 100644 --- a/src/main/java/com/epam/reportportal/jobs/notification/NotifyUserExpirationJob.java +++ b/src/main/java/com/epam/reportportal/jobs/notification/NotifyUserExpirationJob.java @@ -56,7 +56,7 @@ public class NotifyUserExpirationJob extends BaseJob { + " u.id as user_id, " + " u.email as email, " + "DATE_PART('day', NOW() - GREATEST(" - + " to_timestamp(CAST(u.metadata->'metadata'->>'last_login' AS bigint) / 1000), " + + " DATE(to_timestamp(CAST(u.metadata->'metadata'->>'last_login' AS bigint) / 1000)), " + "MAX(ak.last_used_at))) AS inactivityPeriod " + "FROM " + " users u " From 14f8bef554a658f5b5669b977a9854816b92d0aa Mon Sep 17 00:00:00 2001 From: Andrei Piankouski Date: Fri, 4 Aug 2023 15:33:13 +0300 Subject: [PATCH 08/27] EPMRPP-83085 || Fix delete-account-notification-template --- .../jobs/notification/NotifyUserExpirationJob.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/epam/reportportal/jobs/notification/NotifyUserExpirationJob.java b/src/main/java/com/epam/reportportal/jobs/notification/NotifyUserExpirationJob.java index 5ad5028..10775a3 100644 --- a/src/main/java/com/epam/reportportal/jobs/notification/NotifyUserExpirationJob.java +++ b/src/main/java/com/epam/reportportal/jobs/notification/NotifyUserExpirationJob.java @@ -124,9 +124,9 @@ private String getRemainingTime(int remainingTime) { if (remainingTime == 1) { return "tomorrow"; } else if (remainingTime == 30) { - return "1 month"; + return "in 1 month"; } else if (remainingTime == 60) { - return "2 months"; + return "in 2 months"; } else { return remainingTime + DAYS; } @@ -140,7 +140,7 @@ private String getDeadlineDate(int remainingTime) { private String getInactivityPeriod(int inactivityPeriod) { int inactivityMouths = inactivityPeriod / 30; - return retentionPeriod - inactivityPeriod == 1 ? "almost " + retentionPeriod / 30 + " months" - : inactivityMouths + " months"; + return retentionPeriod - inactivityPeriod == 1 ? "almost " + retentionPeriod / 30 + " months" + : "for " + inactivityMouths + " months"; } } From 70fd5ef13cd1dbe5db2457533911cf9d7a73d122 Mon Sep 17 00:00:00 2001 From: Andrei Piankouski Date: Fri, 4 Aug 2023 17:13:21 +0300 Subject: [PATCH 09/27] EPMRPP-83085 || Fix delete-account-notification-template --- .../reportportal/jobs/notification/NotifyUserExpirationJob.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/epam/reportportal/jobs/notification/NotifyUserExpirationJob.java b/src/main/java/com/epam/reportportal/jobs/notification/NotifyUserExpirationJob.java index 10775a3..97961b5 100644 --- a/src/main/java/com/epam/reportportal/jobs/notification/NotifyUserExpirationJob.java +++ b/src/main/java/com/epam/reportportal/jobs/notification/NotifyUserExpirationJob.java @@ -141,6 +141,6 @@ private String getDeadlineDate(int remainingTime) { private String getInactivityPeriod(int inactivityPeriod) { int inactivityMouths = inactivityPeriod / 30; return retentionPeriod - inactivityPeriod == 1 ? "almost " + retentionPeriod / 30 + " months" - : "for " + inactivityMouths + " months"; + : "the past " + inactivityMouths + " months"; } } From e2bacaa85147ff0c4acf2d9f2fcc03ca208d35a3 Mon Sep 17 00:00:00 2001 From: Andrei Piankouski Date: Mon, 7 Aug 2023 11:32:16 +0300 Subject: [PATCH 10/27] EPMRPP-85418 || Email is not sent to user when the user is deleted by the job --- .../com/epam/reportportal/jobs/clean/DeleteExpiredUsersJob.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/epam/reportportal/jobs/clean/DeleteExpiredUsersJob.java b/src/main/java/com/epam/reportportal/jobs/clean/DeleteExpiredUsersJob.java index a77039d..aef4d0b 100644 --- a/src/main/java/com/epam/reportportal/jobs/clean/DeleteExpiredUsersJob.java +++ b/src/main/java/com/epam/reportportal/jobs/clean/DeleteExpiredUsersJob.java @@ -141,6 +141,7 @@ private List findUsersAndPersonalProjects() { UserProject userProject = new UserProject(); userProject.setUserId(rs.getLong("user_id")); userProject.setProjectId(rs.getLong("project_id")); + userProject.setEmail(rs.getString("user_email")); return userProject; }; From 14e7fe13810e4db38acb587e5f0d6f108b85ba66 Mon Sep 17 00:00:00 2001 From: Andrei Piankouski Date: Mon, 7 Aug 2023 15:19:53 +0300 Subject: [PATCH 11/27] EPMRPP-85418 || Email is not sent to user when the user is deleted by the job --- .../reportportal/analyzer/index/IndexerServiceClientImpl.java | 2 +- .../com/epam/reportportal/jobs/clean/DeleteExpiredUsersJob.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/com/epam/reportportal/analyzer/index/IndexerServiceClientImpl.java b/src/main/java/com/epam/reportportal/analyzer/index/IndexerServiceClientImpl.java index 45e036d..363dd5c 100644 --- a/src/main/java/com/epam/reportportal/analyzer/index/IndexerServiceClientImpl.java +++ b/src/main/java/com/epam/reportportal/analyzer/index/IndexerServiceClientImpl.java @@ -46,7 +46,7 @@ public class IndexerServiceClientImpl implements IndexerServiceClient { @Autowired public IndexerServiceClientImpl(RabbitMqManagementClient rabbitMqManagementClient, - @Qualifier("rabbitTemplate") RabbitTemplate rabbitTemplate) { + @Qualifier("analyzerRabbitTemplate") RabbitTemplate rabbitTemplate) { this.rabbitMqManagementClient = rabbitMqManagementClient; this.rabbitTemplate = rabbitTemplate; } diff --git a/src/main/java/com/epam/reportportal/jobs/clean/DeleteExpiredUsersJob.java b/src/main/java/com/epam/reportportal/jobs/clean/DeleteExpiredUsersJob.java index aef4d0b..9556dd0 100644 --- a/src/main/java/com/epam/reportportal/jobs/clean/DeleteExpiredUsersJob.java +++ b/src/main/java/com/epam/reportportal/jobs/clean/DeleteExpiredUsersJob.java @@ -134,7 +134,6 @@ public void execute() { private List findUsersAndPersonalProjects() { MapSqlParameterSource params = new MapSqlParameterSource(); - params.addValue("now_ms", System.currentTimeMillis()); params.addValue(RETENTION_PERIOD, lastLoginBorder()); RowMapper rowMapper = (rs, rowNum) -> { From d9c30e750fe7f1af168aed71bdbee510b0bb12cc Mon Sep 17 00:00:00 2001 From: Reingold Shekhtel <13565058+raikbitters@users.noreply.github.com> Date: Tue, 8 Aug 2023 14:58:38 +0400 Subject: [PATCH 12/27] Patch Dockerfile (#92) * Update Dockerfile --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index efadd2c..52f1b1b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM gradle:6.8.3-jdk11 AS build ARG BOM_VERSION MIGRATION_VERSION GITHUB_USER GITHUB_TOKEN RELEASE_MODE SCRIPTS_VERSION APP_VERSION WORKDIR /usr/app COPY . /usr/app -RUN if [ ${RELEASE_MODE} = true ]; then \ +RUN if [ "${RELEASE_MODE}" = true ]; then \ gradle build --exclude-task test \ -PreleaseMode=true \ -PgithubUserName=${GITHUB_USER} \ @@ -22,4 +22,4 @@ WORKDIR $APP_DIR COPY --from=build $APP_DIR/build/libs/service-jobs-*exec.jar . VOLUME ["/tmp"] EXPOSE 8080 -ENTRYPOINT exec java ${JAVA_OPTS} -jar ${APP_DIR}/service-jobs-*exec.jar \ No newline at end of file +ENTRYPOINT exec java ${JAVA_OPTS} -jar ${APP_DIR}/service-jobs-*exec.jar From 868f59d3edb47a2b542bad139b0527ab810e13c4 Mon Sep 17 00:00:00 2001 From: rkukharenka Date: Thu, 10 Aug 2023 13:31:21 +0300 Subject: [PATCH 13/27] EPMRPP-84893 || added MessageBus to jobs-service --- .../epam/reportportal/service/MessageBus.java | 33 ++++++++ .../service/impl/MessageBusImpl.java | 77 +++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 src/main/java/com/epam/reportportal/service/MessageBus.java create mode 100644 src/main/java/com/epam/reportportal/service/impl/MessageBusImpl.java diff --git a/src/main/java/com/epam/reportportal/service/MessageBus.java b/src/main/java/com/epam/reportportal/service/MessageBus.java new file mode 100644 index 0000000..d7cc83f --- /dev/null +++ b/src/main/java/com/epam/reportportal/service/MessageBus.java @@ -0,0 +1,33 @@ +/* + * Copyright 2023 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.service; + +import com.epam.reportportal.model.activity.ActivityEvent; +import java.util.List; + +/** + * MessageBus is an abstraction for dealing with events over external event-streaming system. + * + * @author Ryhor_Kukharenka + */ +public interface MessageBus { + + void publishActivity(ActivityEvent event); + + void sendNotificationEmail(List recipients); + +} diff --git a/src/main/java/com/epam/reportportal/service/impl/MessageBusImpl.java b/src/main/java/com/epam/reportportal/service/impl/MessageBusImpl.java new file mode 100644 index 0000000..df1548c --- /dev/null +++ b/src/main/java/com/epam/reportportal/service/impl/MessageBusImpl.java @@ -0,0 +1,77 @@ +/* + * Copyright 2023 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.service.impl; + +import static com.epam.reportportal.config.rabbit.InternalConfiguration.EXCHANGE_NOTIFICATION; +import static com.epam.reportportal.config.rabbit.InternalConfiguration.QUEUE_EMAIL; + +import com.epam.reportportal.model.EmailNotificationRequest; +import com.epam.reportportal.model.activity.Activity; +import com.epam.reportportal.model.activity.ActivityEvent; +import com.epam.reportportal.service.MessageBus; +import java.util.List; +import java.util.Objects; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Service; + +/** + * MessageBus implementation using RabbitMQ for transfer message. + * + * @author Ryhor_Kukharenka + */ +@Service +public class MessageBusImpl implements MessageBus { + + public static final String USER_DELETION_TEMPLATE = "userDeletionNotification"; + + private final RabbitTemplate rabbitTemplate; + + public MessageBusImpl(@Qualifier("rabbitTemplate") RabbitTemplate rabbitTemplate) { + this.rabbitTemplate = rabbitTemplate; + } + + /** + * Publishes activity to the queue with the following routing key. + * + * @param event Activity event to be converted to Activity object + * @author Ryhor_Kuharenka + */ + @Override + public void publishActivity(ActivityEvent event) { + final Activity activity = event.toActivity(); + if (Objects.nonNull(activity)) { + rabbitTemplate.convertAndSend("activity", generateKeyForActivity(activity), activity); + } + } + + private String generateKeyForActivity(Activity activity) { + return String.format("activity.%d.%s.%s", + activity.getProjectId(), + activity.getObjectType(), + activity.getEventName()); + } + + @Override + public void sendNotificationEmail(List recipients) { + for (String recipient : recipients) { + EmailNotificationRequest notification = + new EmailNotificationRequest(recipient, USER_DELETION_TEMPLATE); + rabbitTemplate.convertAndSend(EXCHANGE_NOTIFICATION, QUEUE_EMAIL, notification); + } + } +} From 3ab31c7f75b335dee94fdf6347dd839e78eb9582 Mon Sep 17 00:00:00 2001 From: rkukharenka Date: Thu, 10 Aug 2023 13:35:12 +0300 Subject: [PATCH 14/27] EPMRPP-84893 || added activity events to jobs-service --- .../jobs/clean/DeleteExpiredUsersJob.java | 61 +++-- .../reportportal/model/activity/Activity.java | 238 ++++++++++++++++++ .../model/activity/ActivityEvent.java | 33 +++ .../model/activity/enums/ActivityAction.java | 42 ++++ .../model/activity/enums/EventAction.java | 29 +++ .../model/activity/enums/EventObject.java | 28 +++ .../model/activity/enums/EventPriority.java | 30 +++ .../model/activity/enums/EventSubject.java | 38 +++ .../activity/event/ProjectDeletedEvent.java | 60 +++++ .../activity/event/UnassignUserEvent.java | 55 ++++ .../activity/event/UserDeletedEvent.java | 60 +++++ 11 files changed, 651 insertions(+), 23 deletions(-) create mode 100644 src/main/java/com/epam/reportportal/model/activity/Activity.java create mode 100644 src/main/java/com/epam/reportportal/model/activity/ActivityEvent.java create mode 100644 src/main/java/com/epam/reportportal/model/activity/enums/ActivityAction.java create mode 100644 src/main/java/com/epam/reportportal/model/activity/enums/EventAction.java create mode 100644 src/main/java/com/epam/reportportal/model/activity/enums/EventObject.java create mode 100644 src/main/java/com/epam/reportportal/model/activity/enums/EventPriority.java create mode 100644 src/main/java/com/epam/reportportal/model/activity/enums/EventSubject.java create mode 100644 src/main/java/com/epam/reportportal/model/activity/event/ProjectDeletedEvent.java create mode 100644 src/main/java/com/epam/reportportal/model/activity/event/UnassignUserEvent.java create mode 100644 src/main/java/com/epam/reportportal/model/activity/event/UserDeletedEvent.java diff --git a/src/main/java/com/epam/reportportal/jobs/clean/DeleteExpiredUsersJob.java b/src/main/java/com/epam/reportportal/jobs/clean/DeleteExpiredUsersJob.java index 9556dd0..44a1703 100644 --- a/src/main/java/com/epam/reportportal/jobs/clean/DeleteExpiredUsersJob.java +++ b/src/main/java/com/epam/reportportal/jobs/clean/DeleteExpiredUsersJob.java @@ -16,16 +16,15 @@ package com.epam.reportportal.jobs.clean; -import static com.epam.reportportal.config.rabbit.InternalConfiguration.EXCHANGE_NOTIFICATION; -import static com.epam.reportportal.config.rabbit.InternalConfiguration.QUEUE_EMAIL; - import com.epam.reportportal.analyzer.index.IndexerServiceClient; import com.epam.reportportal.jobs.BaseJob; -import com.epam.reportportal.model.EmailNotificationRequest; -import java.time.LocalDate; +import com.epam.reportportal.model.activity.event.ProjectDeletedEvent; +import com.epam.reportportal.model.activity.event.UnassignUserEvent; +import com.epam.reportportal.model.activity.event.UserDeletedEvent; +import com.epam.reportportal.service.MessageBus; import java.time.LocalDateTime; import java.time.ZoneOffset; -import java.util.HashMap; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; @@ -34,9 +33,7 @@ import org.jclouds.blobstore.BlobStore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.RowMapper; @@ -44,6 +41,7 @@ import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; /** * Deleting Users and their personal project by retention policy. @@ -57,7 +55,6 @@ public class DeleteExpiredUsersJob extends BaseJob { private final NamedParameterJdbcTemplate namedParameterJdbcTemplate; - public static final String USER_DELETION_TEMPLATE = "userDeletionNotification"; private static final String RETENTION_PERIOD = "retentionPeriod"; private static final String SELECT_EXPIRED_USERS = "SELECT u.id AS user_id, " @@ -95,6 +92,11 @@ public class DeleteExpiredUsersJob extends BaseJob { private static final String DELETE_PROJECTS_BY_ID_LIST = "DELETE FROM project WHERE id IN (:projectIds)"; + private static final String FIND_NON_PERSONAL_PROJECTS_BY_USER_IDS = "SELECT p.id " + + "FROM project_user pu " + + "JOIN project p ON pu.project_id = p.id " + + "WHERE p.project_type != 'PERSONAL' AND pu.user_id IN (:userIds)"; + @Value("${rp.environment.variable.clean.expiredUser.retentionPeriod}") private long retentionPeriod; @@ -102,7 +104,7 @@ public class DeleteExpiredUsersJob extends BaseJob { private final IndexerServiceClient indexerServiceClient; - private final RabbitTemplate rabbitTemplate; + private final MessageBus messageBus; @Value("${datastore.bucketPrefix}") private String bucketPrefix; @@ -111,12 +113,12 @@ public class DeleteExpiredUsersJob extends BaseJob { public DeleteExpiredUsersJob(JdbcTemplate jdbcTemplate, NamedParameterJdbcTemplate namedParameterJdbcTemplate, BlobStore blobStore, IndexerServiceClient indexerServiceClient, - @Qualifier("rabbitTemplate") RabbitTemplate rabbitTemplate) { + MessageBus messageBus) { super(jdbcTemplate); this.namedParameterJdbcTemplate = namedParameterJdbcTemplate; this.blobStore = blobStore; this.indexerServiceClient = indexerServiceClient; - this.rabbitTemplate = rabbitTemplate; + this.messageBus = messageBus; } @Override @@ -125,13 +127,32 @@ public DeleteExpiredUsersJob(JdbcTemplate jdbcTemplate, public void execute() { List userProjects = findUsersAndPersonalProjects(); List userIds = getUserIds(userProjects); + + List personalProjectIds = getProjectIds(userProjects); + List nonPersonalProjectsByUserIds = findNonPersonalProjectIdsByUserIds(userIds); + deleteUsersByIds(userIds); - getProjectIds(userProjects).forEach(this::deleteProjectAssociatedData); - deleteProjectsByIds(getProjectIds(userProjects)); - sendNotificationEmail(getUserEmails(userProjects)); + publishUnassignUserEvents(nonPersonalProjectsByUserIds); + personalProjectIds.forEach(this::deleteProjectAssociatedData); + deleteProjectsByIds(personalProjectIds); + + messageBus.sendNotificationEmail(getUserEmails(userProjects)); + LOGGER.info("{} - users was deleted due to retention policy", userIds.size()); } + private void publishUnassignUserEvents(List nonPersonalProjectsByUserIds) { + nonPersonalProjectsByUserIds.forEach( + projectId -> messageBus.publishActivity(new UnassignUserEvent(projectId))); + } + + private List findNonPersonalProjectIdsByUserIds(List userIds) { + return CollectionUtils.isEmpty(userIds) + ? Collections.emptyList() + : namedParameterJdbcTemplate.queryForList(FIND_NON_PERSONAL_PROJECTS_BY_USER_IDS, + Map.of("userIds", userIds), Long.class); + } + private List findUsersAndPersonalProjects() { MapSqlParameterSource params = new MapSqlParameterSource(); params.addValue(RETENTION_PERIOD, lastLoginBorder()); @@ -164,6 +185,7 @@ private void deleteUsersByIds(List userIds) { MapSqlParameterSource params = new MapSqlParameterSource(); params.addValue("userIds", userIds); namedParameterJdbcTemplate.update(DELETE_USERS, params); + messageBus.publishActivity(new UserDeletedEvent(userIds.size())); } } @@ -184,6 +206,7 @@ private void deleteProjectsByIds(List projectIds) { MapSqlParameterSource params = new MapSqlParameterSource(); params.addValue("projectIds", projectIds); namedParameterJdbcTemplate.update(DELETE_PROJECTS_BY_ID_LIST, params); + messageBus.publishActivity(new ProjectDeletedEvent(projectIds.size())); } } @@ -204,14 +227,6 @@ private List getProjectIds(List userProjects) { .collect(Collectors.toList()); } - private void sendNotificationEmail(List recipients) { - for (String recipient : recipients) { - EmailNotificationRequest notification = - new EmailNotificationRequest(recipient, USER_DELETION_TEMPLATE); - rabbitTemplate.convertAndSend(EXCHANGE_NOTIFICATION, QUEUE_EMAIL, notification); - } - } - private static class UserProject { private long userId; diff --git a/src/main/java/com/epam/reportportal/model/activity/Activity.java b/src/main/java/com/epam/reportportal/model/activity/Activity.java new file mode 100644 index 0000000..0016b4e --- /dev/null +++ b/src/main/java/com/epam/reportportal/model/activity/Activity.java @@ -0,0 +1,238 @@ +/* + * Copyright 2023 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.model.activity; + +import com.epam.reportportal.model.activity.enums.EventAction; +import com.epam.reportportal.model.activity.enums.EventObject; +import com.epam.reportportal.model.activity.enums.EventPriority; +import com.epam.reportportal.model.activity.enums.EventSubject; +import java.time.LocalDateTime; + +/** + * A model that represents the state of the Activity. + * + * @author Ryhor_Kukharenka + */ +public class Activity { + + private LocalDateTime createdAt; + private EventAction action; + private String eventName; + private EventPriority priority; + private Long objectId; + private String objectName; + private EventObject objectType; + private Long projectId; + private String projectName; + private Long subjectId; + private String subjectName; + private EventSubject subjectType; + private boolean isSavedEvent; + + public Activity() { + this.isSavedEvent = true; + } + + public LocalDateTime getCreatedAt() { + return createdAt; + } + + public void setCreatedAt(LocalDateTime createdAt) { + this.createdAt = createdAt; + } + + public EventAction getAction() { + return action; + } + + public void setAction(EventAction action) { + this.action = action; + } + + public String getEventName() { + return eventName; + } + + public void setEventName(String eventName) { + this.eventName = eventName; + } + + public EventPriority getPriority() { + return priority; + } + + public void setPriority(EventPriority priority) { + this.priority = priority; + } + + public Long getObjectId() { + return objectId; + } + + public void setObjectId(Long objectId) { + this.objectId = objectId; + } + + public String getObjectName() { + return objectName; + } + + public void setObjectName(String objectName) { + this.objectName = objectName; + } + + public EventObject getObjectType() { + return objectType; + } + + public void setObjectType(EventObject objectType) { + this.objectType = objectType; + } + + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public Long getSubjectId() { + return subjectId; + } + + public void setSubjectId(Long subjectId) { + this.subjectId = subjectId; + } + + public String getSubjectName() { + return subjectName; + } + + public void setSubjectName(String subjectName) { + this.subjectName = subjectName; + } + + public EventSubject getSubjectType() { + return subjectType; + } + + public void setSubjectType(EventSubject subjectType) { + this.subjectType = subjectType; + } + + public boolean isSavedEvent() { + return isSavedEvent; + } + + public void setSavedEvent(boolean savedEvent) { + isSavedEvent = savedEvent; + } + + public static ActivityBuilder builder() { + return new ActivityBuilder(); + } + + + /** + * Activity builder. + * + * @author Ryhor_Kukharenka + */ + public static class ActivityBuilder { + + private final Activity activity; + + private ActivityBuilder() { + this.activity = new Activity(); + } + + public ActivityBuilder addCreatedNow() { + activity.setCreatedAt(LocalDateTime.now()); + return this; + } + + public ActivityBuilder addAction(EventAction action) { + activity.setAction(action); + return this; + } + + public ActivityBuilder addEventName(String eventName) { + activity.setEventName(eventName); + return this; + } + + public ActivityBuilder addPriority(EventPriority priority) { + activity.setPriority(priority); + return this; + } + + public ActivityBuilder addObjectId(Long objectId) { + activity.setObjectId(objectId); + return this; + } + + public ActivityBuilder addObjectName(String objectName) { + activity.setObjectName(objectName); + return this; + } + + public ActivityBuilder addObjectType(EventObject objectType) { + activity.setObjectType(objectType); + return this; + } + + public ActivityBuilder addProjectId(Long projectId) { + activity.setProjectId(projectId); + return this; + } + + public ActivityBuilder addProjectName(String projectName) { + activity.setProjectName(projectName); + return this; + } + + public ActivityBuilder addSubjectId(Long subjectId) { + activity.setSubjectId(subjectId); + return this; + } + + public ActivityBuilder addSubjectName(String subjectName) { + activity.setSubjectName(subjectName); + return this; + } + + public ActivityBuilder addSubjectType(EventSubject subjectType) { + activity.setSubjectType(subjectType); + return this; + } + + public Activity build() { + return activity; + } + + } + +} diff --git a/src/main/java/com/epam/reportportal/model/activity/ActivityEvent.java b/src/main/java/com/epam/reportportal/model/activity/ActivityEvent.java new file mode 100644 index 0000000..3999535 --- /dev/null +++ b/src/main/java/com/epam/reportportal/model/activity/ActivityEvent.java @@ -0,0 +1,33 @@ +/* + * Copyright 2023 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.model.activity; + +/** + * Activity Event. + * + * @author Ryhor_Kukharenka + */ +public interface ActivityEvent { + + /** + * Method for transform Event to Activity. + * + * @return Activity entity + */ + Activity toActivity(); + +} diff --git a/src/main/java/com/epam/reportportal/model/activity/enums/ActivityAction.java b/src/main/java/com/epam/reportportal/model/activity/enums/ActivityAction.java new file mode 100644 index 0000000..62ed204 --- /dev/null +++ b/src/main/java/com/epam/reportportal/model/activity/enums/ActivityAction.java @@ -0,0 +1,42 @@ +/* + * Copyright 2023 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.model.activity.enums; + +/** + * Activity Action Type (event_name). + * + * @author Ryhor_Kukharenka + */ +public enum ActivityAction { + + UNASSIGN_USER("unassignUser"), + + BULK_DELETE_USERS("bulkDeleteUsers"), + + BULK_DELETE_PROJECT("bulkDeleteProject"); + + private final String value; + + ActivityAction(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + +} diff --git a/src/main/java/com/epam/reportportal/model/activity/enums/EventAction.java b/src/main/java/com/epam/reportportal/model/activity/enums/EventAction.java new file mode 100644 index 0000000..8565523 --- /dev/null +++ b/src/main/java/com/epam/reportportal/model/activity/enums/EventAction.java @@ -0,0 +1,29 @@ +/* + * Copyright 2023 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.model.activity.enums; + +/** + * Event Action Type. + * + * @author Ryhor_Kukharenka + */ +public enum EventAction { + + UNASSIGN, + BULK_DELETE + +} diff --git a/src/main/java/com/epam/reportportal/model/activity/enums/EventObject.java b/src/main/java/com/epam/reportportal/model/activity/enums/EventObject.java new file mode 100644 index 0000000..787449d --- /dev/null +++ b/src/main/java/com/epam/reportportal/model/activity/enums/EventObject.java @@ -0,0 +1,28 @@ +/* + * Copyright 2023 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.model.activity.enums; + +/** + * Event Object Type. + * + * @author Ryhor_Kukharenka + */ +public enum EventObject { + + USER + +} diff --git a/src/main/java/com/epam/reportportal/model/activity/enums/EventPriority.java b/src/main/java/com/epam/reportportal/model/activity/enums/EventPriority.java new file mode 100644 index 0000000..1260740 --- /dev/null +++ b/src/main/java/com/epam/reportportal/model/activity/enums/EventPriority.java @@ -0,0 +1,30 @@ +/* + * Copyright 2023 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.model.activity.enums; + +/** + * Event Priority Type. + * + * @author Ryhor_Kukharenka + */ +public enum EventPriority { + + CRITICAL, + HIGH, + MEDIUM + +} diff --git a/src/main/java/com/epam/reportportal/model/activity/enums/EventSubject.java b/src/main/java/com/epam/reportportal/model/activity/enums/EventSubject.java new file mode 100644 index 0000000..f03146f --- /dev/null +++ b/src/main/java/com/epam/reportportal/model/activity/enums/EventSubject.java @@ -0,0 +1,38 @@ +/* + * Copyright 2023 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.model.activity.enums; + +/** + * Event Subject Type. + * + * @author Ryhor_Kukharenka + */ +public enum EventSubject { + + APPLICATION("Job Service"); + + private final String applicationName; + + EventSubject(String applicationName) { + this.applicationName = applicationName; + } + + public String getApplicationName() { + return applicationName; + } + +} diff --git a/src/main/java/com/epam/reportportal/model/activity/event/ProjectDeletedEvent.java b/src/main/java/com/epam/reportportal/model/activity/event/ProjectDeletedEvent.java new file mode 100644 index 0000000..545d229 --- /dev/null +++ b/src/main/java/com/epam/reportportal/model/activity/event/ProjectDeletedEvent.java @@ -0,0 +1,60 @@ +/* + * Copyright 2023 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.model.activity.event; + +import com.epam.reportportal.model.activity.Activity; +import com.epam.reportportal.model.activity.ActivityEvent; +import com.epam.reportportal.model.activity.enums.ActivityAction; +import com.epam.reportportal.model.activity.enums.EventAction; +import com.epam.reportportal.model.activity.enums.EventObject; +import com.epam.reportportal.model.activity.enums.EventPriority; +import com.epam.reportportal.model.activity.enums.EventSubject; +import org.apache.commons.lang3.StringUtils; + +/** + * Publish an event when project is deleted. + * + * @author Ryhor_Kukharenka + */ +public class ProjectDeletedEvent implements ActivityEvent { + + private final int countProjectsDeleted; + + public ProjectDeletedEvent(int countProjectsDeleted) { + this.countProjectsDeleted = countProjectsDeleted; + } + + @Override + public Activity toActivity() { + return Activity.builder() + .addCreatedNow() + .addAction(EventAction.BULK_DELETE) + .addEventName(ActivityAction.BULK_DELETE_PROJECT.getValue()) + .addObjectName(getFormatText()) + .addObjectType(EventObject.USER) + .addSubjectName(EventSubject.APPLICATION.getApplicationName()) + .addSubjectType(EventSubject.APPLICATION) + .addPriority(EventPriority.CRITICAL) + .build(); + } + + private String getFormatText() { + String projectWord = "project" + (countProjectsDeleted == 1 ? StringUtils.EMPTY : "s"); + return String.format("%d personal %s", countProjectsDeleted, projectWord); + } + +} diff --git a/src/main/java/com/epam/reportportal/model/activity/event/UnassignUserEvent.java b/src/main/java/com/epam/reportportal/model/activity/event/UnassignUserEvent.java new file mode 100644 index 0000000..e0c4861 --- /dev/null +++ b/src/main/java/com/epam/reportportal/model/activity/event/UnassignUserEvent.java @@ -0,0 +1,55 @@ +/* + * Copyright 2023 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.model.activity.event; + +import com.epam.reportportal.model.activity.Activity; +import com.epam.reportportal.model.activity.ActivityEvent; +import com.epam.reportportal.model.activity.enums.ActivityAction; +import com.epam.reportportal.model.activity.enums.EventAction; +import com.epam.reportportal.model.activity.enums.EventObject; +import com.epam.reportportal.model.activity.enums.EventPriority; +import com.epam.reportportal.model.activity.enums.EventSubject; + +/** + * Event publish when user is unassigned to a project. + * + * @author Ryhor_Kukharenka + */ +public class UnassignUserEvent implements ActivityEvent { + + private final Long projectId; + + public UnassignUserEvent(Long projectId) { + this.projectId = projectId; + } + + @Override + public Activity toActivity() { + return Activity.builder() + .addCreatedNow() + .addAction(EventAction.UNASSIGN) + .addEventName(ActivityAction.UNASSIGN_USER.getValue()) + .addObjectType(EventObject.USER) + .addObjectName("deleted_user") + .addProjectId(projectId) + .addSubjectName(EventSubject.APPLICATION.getApplicationName()) + .addSubjectType(EventSubject.APPLICATION) + .addPriority(EventPriority.MEDIUM) + .build(); + } + +} diff --git a/src/main/java/com/epam/reportportal/model/activity/event/UserDeletedEvent.java b/src/main/java/com/epam/reportportal/model/activity/event/UserDeletedEvent.java new file mode 100644 index 0000000..0a97510 --- /dev/null +++ b/src/main/java/com/epam/reportportal/model/activity/event/UserDeletedEvent.java @@ -0,0 +1,60 @@ +/* + * Copyright 2023 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.model.activity.event; + +import com.epam.reportportal.model.activity.Activity; +import com.epam.reportportal.model.activity.ActivityEvent; +import com.epam.reportportal.model.activity.enums.ActivityAction; +import com.epam.reportportal.model.activity.enums.EventAction; +import com.epam.reportportal.model.activity.enums.EventObject; +import com.epam.reportportal.model.activity.enums.EventPriority; +import com.epam.reportportal.model.activity.enums.EventSubject; +import org.apache.commons.lang3.StringUtils; + +/** + * Publish an event when user is deleted. + * + * @author Ryhor_Kukharenka + */ +public class UserDeletedEvent implements ActivityEvent { + + private final int countUsersDeleted; + + public UserDeletedEvent(int countUsersDeleted) { + this.countUsersDeleted = countUsersDeleted; + } + + @Override + public Activity toActivity() { + return Activity.builder() + .addCreatedNow() + .addAction(EventAction.BULK_DELETE) + .addEventName(ActivityAction.BULK_DELETE_USERS.getValue()) + .addObjectName(getFormatText()) + .addObjectType(EventObject.USER) + .addSubjectName(EventSubject.APPLICATION.getApplicationName()) + .addSubjectType(EventSubject.APPLICATION) + .addPriority(EventPriority.HIGH) + .build(); + } + + private String getFormatText() { + String userWord = "user" + (countUsersDeleted == 1 ? StringUtils.EMPTY : "s"); + return String.format("%d deleted %s", countUsersDeleted, userWord); + } + +} From 85a00d63d000d3fbd4562aaa87c4c1c960bf7bbf Mon Sep 17 00:00:00 2001 From: rkukharenka Date: Thu, 10 Aug 2023 19:51:39 +0300 Subject: [PATCH 15/27] EPMRPP-85345 || rewrite email notification --- .../jobs/clean/DeleteExpiredUsersJob.java | 12 +++++++++++- .../notification/NotifyUserExpirationJob.java | 19 +++++++------------ .../epam/reportportal/service/MessageBus.java | 3 ++- .../service/impl/MessageBusImpl.java | 12 ++++-------- 4 files changed, 24 insertions(+), 22 deletions(-) diff --git a/src/main/java/com/epam/reportportal/jobs/clean/DeleteExpiredUsersJob.java b/src/main/java/com/epam/reportportal/jobs/clean/DeleteExpiredUsersJob.java index 44a1703..a435578 100644 --- a/src/main/java/com/epam/reportportal/jobs/clean/DeleteExpiredUsersJob.java +++ b/src/main/java/com/epam/reportportal/jobs/clean/DeleteExpiredUsersJob.java @@ -18,6 +18,7 @@ import com.epam.reportportal.analyzer.index.IndexerServiceClient; import com.epam.reportportal.jobs.BaseJob; +import com.epam.reportportal.model.EmailNotificationRequest; import com.epam.reportportal.model.activity.event.ProjectDeletedEvent; import com.epam.reportportal.model.activity.event.UnassignUserEvent; import com.epam.reportportal.model.activity.event.UserDeletedEvent; @@ -57,6 +58,8 @@ public class DeleteExpiredUsersJob extends BaseJob { private static final String RETENTION_PERIOD = "retentionPeriod"; + private static final String USER_DELETION_TEMPLATE = "userDeletionNotification"; + private static final String SELECT_EXPIRED_USERS = "SELECT u.id AS user_id, " + "p.id AS project_id, u.email as user_email " + "FROM users u " @@ -136,11 +139,18 @@ public void execute() { personalProjectIds.forEach(this::deleteProjectAssociatedData); deleteProjectsByIds(personalProjectIds); - messageBus.sendNotificationEmail(getUserEmails(userProjects)); + publishEmailNotificationEvents(getUserEmails(userProjects)); LOGGER.info("{} - users was deleted due to retention policy", userIds.size()); } + private void publishEmailNotificationEvents(List userEmails) { + List notifications = userEmails.stream() + .map(recipient -> new EmailNotificationRequest(recipient, USER_DELETION_TEMPLATE)) + .collect(Collectors.toList()); + messageBus.publishEmailNotificationEvents(notifications); + } + private void publishUnassignUserEvents(List nonPersonalProjectsByUserIds) { nonPersonalProjectsByUserIds.forEach( projectId -> messageBus.publishActivity(new UnassignUserEvent(projectId))); diff --git a/src/main/java/com/epam/reportportal/jobs/notification/NotifyUserExpirationJob.java b/src/main/java/com/epam/reportportal/jobs/notification/NotifyUserExpirationJob.java index 97961b5..89ff3a6 100644 --- a/src/main/java/com/epam/reportportal/jobs/notification/NotifyUserExpirationJob.java +++ b/src/main/java/com/epam/reportportal/jobs/notification/NotifyUserExpirationJob.java @@ -16,19 +16,15 @@ package com.epam.reportportal.jobs.notification; -import static com.epam.reportportal.config.rabbit.InternalConfiguration.EXCHANGE_NOTIFICATION; -import static com.epam.reportportal.config.rabbit.InternalConfiguration.QUEUE_EMAIL; - import com.epam.reportportal.jobs.BaseJob; import com.epam.reportportal.model.EmailNotificationRequest; +import com.epam.reportportal.service.MessageBus; import java.time.LocalDate; import java.util.HashMap; import java.util.List; import java.util.Map; import net.javacrumbs.shedlock.spring.annotation.SchedulerLock; -import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; @@ -83,15 +79,15 @@ public class NotifyUserExpirationJob extends BaseJob { private final NamedParameterJdbcTemplate namedParameterJdbcTemplate; - private final RabbitTemplate rabbitTemplate; + private final MessageBus messageBus; @Autowired public NotifyUserExpirationJob(JdbcTemplate jdbcTemplate, NamedParameterJdbcTemplate namedParameterJdbcTemplate, - @Qualifier("rabbitTemplate") RabbitTemplate rabbitTemplate) { + MessageBus messageBus) { super(jdbcTemplate); this.namedParameterJdbcTemplate = namedParameterJdbcTemplate; - this.rabbitTemplate = rabbitTemplate; + this.messageBus = messageBus; } @Override @@ -99,9 +95,7 @@ public NotifyUserExpirationJob(JdbcTemplate jdbcTemplate, @SchedulerLock(name = "notifyUserExpiration", lockAtMostFor = "24h") public void execute() { List notifications = getUsersForNotify(); - notifications.forEach( - notification -> rabbitTemplate.convertAndSend(EXCHANGE_NOTIFICATION, QUEUE_EMAIL, - notification)); + messageBus.publishEmailNotificationEvents(notifications); } private List getUsersForNotify() { @@ -140,7 +134,8 @@ private String getDeadlineDate(int remainingTime) { private String getInactivityPeriod(int inactivityPeriod) { int inactivityMouths = inactivityPeriod / 30; - return retentionPeriod - inactivityPeriod == 1 ? "almost " + retentionPeriod / 30 + " months" + return retentionPeriod - inactivityPeriod == 1 ? "almost " + retentionPeriod / 30 + + " months" : "the past " + inactivityMouths + " months"; } } diff --git a/src/main/java/com/epam/reportportal/service/MessageBus.java b/src/main/java/com/epam/reportportal/service/MessageBus.java index d7cc83f..ebfe191 100644 --- a/src/main/java/com/epam/reportportal/service/MessageBus.java +++ b/src/main/java/com/epam/reportportal/service/MessageBus.java @@ -16,6 +16,7 @@ package com.epam.reportportal.service; +import com.epam.reportportal.model.EmailNotificationRequest; import com.epam.reportportal.model.activity.ActivityEvent; import java.util.List; @@ -28,6 +29,6 @@ public interface MessageBus { void publishActivity(ActivityEvent event); - void sendNotificationEmail(List recipients); + void publishEmailNotificationEvents(List notifications); } diff --git a/src/main/java/com/epam/reportportal/service/impl/MessageBusImpl.java b/src/main/java/com/epam/reportportal/service/impl/MessageBusImpl.java index df1548c..f9882ac 100644 --- a/src/main/java/com/epam/reportportal/service/impl/MessageBusImpl.java +++ b/src/main/java/com/epam/reportportal/service/impl/MessageBusImpl.java @@ -37,8 +37,6 @@ @Service public class MessageBusImpl implements MessageBus { - public static final String USER_DELETION_TEMPLATE = "userDeletionNotification"; - private final RabbitTemplate rabbitTemplate; public MessageBusImpl(@Qualifier("rabbitTemplate") RabbitTemplate rabbitTemplate) { @@ -67,11 +65,9 @@ private String generateKeyForActivity(Activity activity) { } @Override - public void sendNotificationEmail(List recipients) { - for (String recipient : recipients) { - EmailNotificationRequest notification = - new EmailNotificationRequest(recipient, USER_DELETION_TEMPLATE); - rabbitTemplate.convertAndSend(EXCHANGE_NOTIFICATION, QUEUE_EMAIL, notification); - } + public void publishEmailNotificationEvents(List notifications) { + notifications.forEach(notification -> + rabbitTemplate.convertAndSend(EXCHANGE_NOTIFICATION, QUEUE_EMAIL, notification)); } + } From ada856c6a612264661dbdc3b20df06d7b1cd7243 Mon Sep 17 00:00:00 2001 From: rkukharenka Date: Fri, 11 Aug 2023 14:44:47 +0300 Subject: [PATCH 16/27] EPMRPP-85345 || updated retention policy for users --- .../jobs/clean/DeleteExpiredUsersJob.java | 11 +++- .../notification/NotifyUserExpirationJob.java | 14 ++++- .../reportportal/utils/ValidationUtil.java | 51 +++++++++++++++++++ src/main/resources/application.yml | 8 ++- 4 files changed, 77 insertions(+), 7 deletions(-) create mode 100644 src/main/java/com/epam/reportportal/utils/ValidationUtil.java diff --git a/src/main/java/com/epam/reportportal/jobs/clean/DeleteExpiredUsersJob.java b/src/main/java/com/epam/reportportal/jobs/clean/DeleteExpiredUsersJob.java index a435578..63668e9 100644 --- a/src/main/java/com/epam/reportportal/jobs/clean/DeleteExpiredUsersJob.java +++ b/src/main/java/com/epam/reportportal/jobs/clean/DeleteExpiredUsersJob.java @@ -23,6 +23,7 @@ import com.epam.reportportal.model.activity.event.UnassignUserEvent; import com.epam.reportportal.model.activity.event.UserDeletedEvent; import com.epam.reportportal.service.MessageBus; +import com.epam.reportportal.utils.ValidationUtil; import java.time.LocalDateTime; import java.time.ZoneOffset; import java.util.Collections; @@ -36,6 +37,7 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.RowMapper; import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; @@ -50,6 +52,8 @@ * @author Andrei Piankouski */ @Service +@ConditionalOnProperty(prefix = "rp.environment.variable", + name = "clean.expiredUser.retentionPeriod") public class DeleteExpiredUsersJob extends BaseJob { public static final Logger LOGGER = LoggerFactory.getLogger(DeleteExpiredUsersJob.class); @@ -101,7 +105,7 @@ public class DeleteExpiredUsersJob extends BaseJob { + "WHERE p.project_type != 'PERSONAL' AND pu.user_id IN (:userIds)"; @Value("${rp.environment.variable.clean.expiredUser.retentionPeriod}") - private long retentionPeriod; + private Long retentionPeriod; private final BlobStore blobStore; @@ -128,6 +132,11 @@ public DeleteExpiredUsersJob(JdbcTemplate jdbcTemplate, @Scheduled(cron = "${rp.environment.variable.clean.expiredUser.cron}") @SchedulerLock(name = "deleteExpiredUsers", lockAtMostFor = "24h") public void execute() { + if (ValidationUtil.isInvalidRetentionPeriod(retentionPeriod)) { + LOGGER.info("No users are deleted"); + return; + } + List userProjects = findUsersAndPersonalProjects(); List userIds = getUserIds(userProjects); diff --git a/src/main/java/com/epam/reportportal/jobs/notification/NotifyUserExpirationJob.java b/src/main/java/com/epam/reportportal/jobs/notification/NotifyUserExpirationJob.java index 89ff3a6..2889f29 100644 --- a/src/main/java/com/epam/reportportal/jobs/notification/NotifyUserExpirationJob.java +++ b/src/main/java/com/epam/reportportal/jobs/notification/NotifyUserExpirationJob.java @@ -19,13 +19,17 @@ import com.epam.reportportal.jobs.BaseJob; import com.epam.reportportal.model.EmailNotificationRequest; import com.epam.reportportal.service.MessageBus; +import com.epam.reportportal.utils.ValidationUtil; import java.time.LocalDate; import java.util.HashMap; import java.util.List; import java.util.Map; import net.javacrumbs.shedlock.spring.annotation.SchedulerLock; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; @@ -38,8 +42,12 @@ * @author Andrei Piankouski */ @Service +@ConditionalOnProperty(prefix = "rp.environment.variable", + name = "clean.expiredUser.retentionPeriod") public class NotifyUserExpirationJob extends BaseJob { + public static final Logger LOGGER = LoggerFactory.getLogger(NotifyUserExpirationJob.class); + private static final String EMAIL = "email"; private static final String USER_EXPIRATION_TEMPLATE = "userExpirationNotification"; private static final String INACTIVITY_PERIOD = "inactivityPeriod"; @@ -75,7 +83,7 @@ public class NotifyUserExpirationJob extends BaseJob { private static final String RETENTION_PERIOD = "retentionPeriod"; @Value("${rp.environment.variable.clean.expiredUser.retentionPeriod}") - private long retentionPeriod; + private Long retentionPeriod; private final NamedParameterJdbcTemplate namedParameterJdbcTemplate; @@ -94,6 +102,10 @@ public NotifyUserExpirationJob(JdbcTemplate jdbcTemplate, @Scheduled(cron = "${rp.environment.variable.notification.expiredUser.cron}") @SchedulerLock(name = "notifyUserExpiration", lockAtMostFor = "24h") public void execute() { + if (ValidationUtil.isInvalidRetentionPeriod(retentionPeriod)) { + LOGGER.info("No notifications are send to users."); + return; + } List notifications = getUsersForNotify(); messageBus.publishEmailNotificationEvents(notifications); } diff --git a/src/main/java/com/epam/reportportal/utils/ValidationUtil.java b/src/main/java/com/epam/reportportal/utils/ValidationUtil.java new file mode 100644 index 0000000..6229bf5 --- /dev/null +++ b/src/main/java/com/epam/reportportal/utils/ValidationUtil.java @@ -0,0 +1,51 @@ +/* + * Copyright 2023 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.utils; + +import java.util.Objects; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Utility class which validate some information. + * + * @author Ryhor_Kukharenka + */ +public final class ValidationUtil { + + public static final Logger LOGGER = LoggerFactory.getLogger(ValidationUtil.class); + + private ValidationUtil() { + } + + /** + * Check retention period. + * + * @param retentionPeriod retentionPeriod + * @return boolean value + */ + public static boolean isInvalidRetentionPeriod(Long retentionPeriod) { + if (Objects.isNull(retentionPeriod) || retentionPeriod <= 0) { + LOGGER.warn("The parameter 'retentionPeriod' is not specified correctly.\n" + + "Must be greater than 0.\n" + + "The current value of the parameter = {}.", retentionPeriod); + return true; + } + return false; + } + +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 17cca7a..5768f16 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -15,11 +15,6 @@ rp: cron: '0 0 */24 * * *' chunkSize: 1000 batchSize: 100 - expired: - ## 24 hours - cron: '0 0 */24 * * *' - ## Retentions period in days - retentionPeriod: 1 attachment: ## 24 hours cron: '0 0 */24 * * *' @@ -34,6 +29,9 @@ rp: cron: '0 0 */24 * * *' liveTimeout: 7200 batch: 100 + expiredUser: + ## 24 hours + cron: '0 0 */24 * * *' storage: project: ## 1 minute From 667bd5643e565e2d3f1e3f72af8d6aff52ff245c Mon Sep 17 00:00:00 2001 From: Andrei Piankouski Date: Tue, 15 Aug 2023 16:07:26 +0300 Subject: [PATCH 17/27] EPMRPP-85482 || Remove creating dockerfile after release --- build.gradle | 2 -- 1 file changed, 2 deletions(-) diff --git a/build.gradle b/build.gradle index 9cffe81..5b7516a 100644 --- a/build.gradle +++ b/build.gradle @@ -99,7 +99,5 @@ test { useJUnitPlatform() } -addDockerfileToGit.dependsOn createDockerfile -beforeReleaseBuild.dependsOn addDockerfileToGit publish.dependsOn build publish.mustRunAfter build From 1b0b390f7159a27ae2b9b2377d9a40a4fa3e6b72 Mon Sep 17 00:00:00 2001 From: rkukharenka Date: Wed, 16 Aug 2023 15:32:42 +0300 Subject: [PATCH 18/27] EPMRPP-84893 || added new EventObject type --- .../epam/reportportal/model/activity/enums/EventObject.java | 3 ++- .../reportportal/model/activity/event/ProjectDeletedEvent.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/epam/reportportal/model/activity/enums/EventObject.java b/src/main/java/com/epam/reportportal/model/activity/enums/EventObject.java index 787449d..ea4118b 100644 --- a/src/main/java/com/epam/reportportal/model/activity/enums/EventObject.java +++ b/src/main/java/com/epam/reportportal/model/activity/enums/EventObject.java @@ -23,6 +23,7 @@ */ public enum EventObject { - USER + USER, + PROJECT } diff --git a/src/main/java/com/epam/reportportal/model/activity/event/ProjectDeletedEvent.java b/src/main/java/com/epam/reportportal/model/activity/event/ProjectDeletedEvent.java index 545d229..19c6749 100644 --- a/src/main/java/com/epam/reportportal/model/activity/event/ProjectDeletedEvent.java +++ b/src/main/java/com/epam/reportportal/model/activity/event/ProjectDeletedEvent.java @@ -45,7 +45,7 @@ public Activity toActivity() { .addAction(EventAction.BULK_DELETE) .addEventName(ActivityAction.BULK_DELETE_PROJECT.getValue()) .addObjectName(getFormatText()) - .addObjectType(EventObject.USER) + .addObjectType(EventObject.PROJECT) .addSubjectName(EventSubject.APPLICATION.getApplicationName()) .addSubjectType(EventSubject.APPLICATION) .addPriority(EventPriority.CRITICAL) From d8ade748a87bb4b78cb38d866a37a592297b89cb Mon Sep 17 00:00:00 2001 From: raikbitters Date: Thu, 24 Aug 2023 19:18:39 +0400 Subject: [PATCH 19/27] Update Dockerfile for Release build --- Dockerfile | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index dcdfde6..6daeb40 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,26 @@ -FROM amazoncorretto:11.0.17 -LABEL version=5.7.4 description="EPAM Report portal. Service jobs" maintainer="Andrei Varabyeu , Hleb Kanonik " -ARG GH_TOKEN -ARG GH_URL=https://__:$GH_TOKEN@maven.pkg.github.com/reportportal/service-jobs/com/epam/reportportal/service-jobs/5.7.4/service-jobs-5.7.4-exec.jar -RUN curl -O -L $GH_URL \ - --output service-jobs-5.7.3-exec.jar && \ - echo 'exec java ${JAVA_OPTS} -jar service-jobs-5.7.4-exec.jar' > /start.sh && chmod +x /start.sh -ENV JAVA_OPTS="-Xmx512m -XX:+UseG1GC -XX:InitiatingHeapOccupancyPercent=70 -Djava.security.egd=file:/dev/./urandom" +FROM gradle:6.8.3-jdk11 AS build +ARG RELEASE_MODE +ARG APP_VERSION +ARG GITHUB_USER +ARG GITHUB_TOKEN +WORKDIR /usr/app +COPY . /usr/app +RUN if [ "${RELEASE_MODE}" = true ]; then \ + gradle build --exclude-task test \ + -PreleaseMode=true \ + -PgithubUserName=${GITHUB_USER} \ + -PgithubToken=${GITHUB_TOKEN} \ + -Dorg.gradle.project.version=${APP_VERSION}; \ + else gradle build --exclude-task test -Dorg.gradle.project.version=${APP_VERSION}; fi + +# For ARM build use flag: `--platform linux/arm64` +FROM --platform=$BUILDPLATFORM amazoncorretto:11.0.19 +LABEL version=${APP_VERSION} description="EPAM Report portal. Main API Service" maintainer="Andrei Varabyeu , Hleb Kanonik " +ARG APP_VERSION=${APP_VERSION} +ENV APP_DIR=/usr/app +ENV JAVA_OPTS="-Xmx1g -XX:+UseG1GC -XX:InitiatingHeapOccupancyPercent=70 -Djava.security.egd=file:/dev/./urandom" +WORKDIR $APP_DIR +COPY --from=build $APP_DIR/build/libs/service-api-*exec.jar . VOLUME ["/tmp"] EXPOSE 8080 -ENTRYPOINT ./start.sh +ENTRYPOINT exec java ${JAVA_OPTS} -jar ${APP_DIR}/service-api-*exec.jar From a0306d1a99cafb0007d080e220dc32c2b1ed6ed5 Mon Sep 17 00:00:00 2001 From: Reingold Shekhtel <13565058+raikbitters@users.noreply.github.com> Date: Thu, 24 Aug 2023 19:31:01 +0400 Subject: [PATCH 20/27] Update Dockerfile --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6daeb40..b23123c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,12 +15,12 @@ RUN if [ "${RELEASE_MODE}" = true ]; then \ # For ARM build use flag: `--platform linux/arm64` FROM --platform=$BUILDPLATFORM amazoncorretto:11.0.19 -LABEL version=${APP_VERSION} description="EPAM Report portal. Main API Service" maintainer="Andrei Varabyeu , Hleb Kanonik " +LABEL version=${APP_VERSION} description="EPAM Report portal. Jobs Service" maintainer="Andrei Varabyeu , Hleb Kanonik " ARG APP_VERSION=${APP_VERSION} ENV APP_DIR=/usr/app ENV JAVA_OPTS="-Xmx1g -XX:+UseG1GC -XX:InitiatingHeapOccupancyPercent=70 -Djava.security.egd=file:/dev/./urandom" WORKDIR $APP_DIR -COPY --from=build $APP_DIR/build/libs/service-api-*exec.jar . +COPY --from=build $APP_DIR/build/libs/service-jobs-*exec.jar . VOLUME ["/tmp"] EXPOSE 8080 -ENTRYPOINT exec java ${JAVA_OPTS} -jar ${APP_DIR}/service-api-*exec.jar +ENTRYPOINT exec java ${JAVA_OPTS} -jar ${APP_DIR}/service-jobs-*exec.jar From 898eb91acf25607559a66c30155816aabb147bad Mon Sep 17 00:00:00 2001 From: Reingold Shekhtel <13565058+raikbitters@users.noreply.github.com> Date: Fri, 25 Aug 2023 15:43:58 +0400 Subject: [PATCH 21/27] Update Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b23123c..ac5e088 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ RUN if [ "${RELEASE_MODE}" = true ]; then \ else gradle build --exclude-task test -Dorg.gradle.project.version=${APP_VERSION}; fi # For ARM build use flag: `--platform linux/arm64` -FROM --platform=$BUILDPLATFORM amazoncorretto:11.0.19 +FROM --platform=$BUILDPLATFORM amazoncorretto:11.0.20 LABEL version=${APP_VERSION} description="EPAM Report portal. Jobs Service" maintainer="Andrei Varabyeu , Hleb Kanonik " ARG APP_VERSION=${APP_VERSION} ENV APP_DIR=/usr/app From 8e8f5574a9c7aa717f3c25b2161037135e41012f Mon Sep 17 00:00:00 2001 From: hlebkanonik Date: Thu, 31 Aug 2023 16:30:17 +0200 Subject: [PATCH 22/27] added github action rc build images --- .github/workflows/rc.yaml | 91 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 .github/workflows/rc.yaml diff --git a/.github/workflows/rc.yaml b/.github/workflows/rc.yaml new file mode 100644 index 0000000..f38812c --- /dev/null +++ b/.github/workflows/rc.yaml @@ -0,0 +1,91 @@ +name: Build RC Docker image + +on: + push: + branches: + - "rc/*" + - "hotfix/*" + +env: + AWS_REGION: ${{ vars.AWS_REGION }} # set this to your preferred AWS region, e.g. us-west-1 + ECR_REPOSITORY: ${{ vars.ECR_REPOSITORY }} # set this to your Amazon ECR repository name + PLATFORMS: ${{ vars.BUILD_PLATFORMS }} # set target build platforms. By default linux/amd64 + RELEASE_MODE: ${{ vars.RELEASE_MODE }} + +jobs: + build-and-export: + name: Build and export to AWS ECR + runs-on: ubuntu-latest + environment: rc + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + # role-to-assume: arn:aws:iam::123456789012:role/my-github-actions-role + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + with: + mask-password: 'true' + + - name: Create variables + id: vars + run: | + echo "tag=$(echo ${{ github.ref_name }} | tr '/' '-')" >> $GITHUB_OUTPUT + echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + echo "version=$(echo '${{ github.ref_name }}' | sed -nE 's/.*([0-9]+\.[0-9]+\.[0-9]+).*/\1/p')" >> $GITHUB_OUTPUT + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build + uses: docker/build-push-action@v4 + env: + VERSION: ${{ steps.vars.outputs.version }} + DATE: ${{ steps.vars.outputs.date }} + IMAGE_TAG: ${{ steps.vars.outputs.tag }} + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + with: + context: . + push: true + build-args: | + APP_VERSION=${{ env.VERSION }} + BUILD_DATE=${{ env.DATE }} + GITHUB_USER=${{ secrets.GH_USER }} + GITHUB_TOKEN=${{ secrets.GH_TOKEN }} + RELEASE_MODE=${{ env.RELEASE_MODE }} + platforms: ${{ env.PLATFORMS }} + tags: ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} + + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@master + env: + IMAGE_TAG: ${{ steps.vars.outputs.tag }} + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + with: + image-ref: '${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }}' + format: 'table' + exit-code: '1' + ignore-unfixed: true + vuln-type: 'os,library' + severity: 'CRITICAL,HIGH' + + - name: Summarize + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + IMAGE_TAG: ${{ steps.vars.outputs.tag }} + run: | + echo "## General information about the build:" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "- :gift: Docker image in Amazon ECR: ecr/$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_STEP_SUMMARY + echo "- :octocat: The commit SHA from which the build was performed: [$GITHUB_SHA](https://github.com/$GITHUB_REPOSITORY/commit/$GITHUB_SHA)" >> $GITHUB_STEP_SUMMARY \ No newline at end of file From 7e744f85d17da2d3e7f820f67f2acc99debbcf27 Mon Sep 17 00:00:00 2001 From: hlebkanonik Date: Thu, 31 Aug 2023 16:49:21 +0200 Subject: [PATCH 23/27] changed exit-code --- .github/workflows/rc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rc.yaml b/.github/workflows/rc.yaml index f38812c..7491849 100644 --- a/.github/workflows/rc.yaml +++ b/.github/workflows/rc.yaml @@ -75,7 +75,7 @@ jobs: with: image-ref: '${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }}' format: 'table' - exit-code: '1' + exit-code: '0' ignore-unfixed: true vuln-type: 'os,library' severity: 'CRITICAL,HIGH' From 4d4034cffb20e4fa42830739407e99665a09604c Mon Sep 17 00:00:00 2001 From: Ivan_Kustau Date: Tue, 29 Aug 2023 17:17:44 +0300 Subject: [PATCH 24/27] Release 5.10.0 --- .github/workflows/manually-release.yml | 11 +---------- .github/workflows/release.yml | 5 +---- build.gradle | 6 +++--- 3 files changed, 5 insertions(+), 17 deletions(-) diff --git a/.github/workflows/manually-release.yml b/.github/workflows/manually-release.yml index 0af5b9f..9af4925 100644 --- a/.github/workflows/manually-release.yml +++ b/.github/workflows/manually-release.yml @@ -6,17 +6,9 @@ on: version: description: 'Release version' required: true - scripts_version: - description: 'Gradle scripts version' - required: true - bom_version: - description: 'Commons bom version' - required: true env: GH_USER_NAME: github.actor - SCRIPTS_VERSION: ${{ github.event.inputs.scripts_version }} - BOM_VERSION: ${{ github.event.inputs.bom_version }} RELEASE_VERSION: ${{ github.event.inputs.version }} REPOSITORY_URL: 'https://maven.pkg.github.com/' @@ -46,7 +38,6 @@ jobs: - name: Release with Gradle id: release run: | - ./gradlew release -PreleaseMode -Pscripts.version=${{env.SCRIPTS_VERSION}} \ - -Pbom.version=${{env.BOM_VERSION}} \ + ./gradlew release -PreleaseMode \ -PgithubUserName=${{env.GH_USER_NAME}} -PgithubToken=${{secrets.GITHUB_TOKEN}} \ -PgpgPassphrase=${{secrets.GPG_PASSPHRASE}} -PgpgPrivateKey="${{secrets.GPG_PRIVATE_KEY}}" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f8840b0..80cbee9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,8 +11,6 @@ on: env: GH_USER_NAME: github.actor - SCRIPTS_VERSION: 5.8.0 - BOM_VERSION: 5.7.6 REPOSITORY_URL: 'https://maven.pkg.github.com/' jobs: @@ -41,7 +39,6 @@ jobs: - name: Release with Gradle id: release run: | - ./gradlew release -PreleaseMode -Pscripts.version=${{env.SCRIPTS_VERSION}} \ - -Pbom.version=${{env.BOM_VERSION}} \ + ./gradlew release -PreleaseMode \ -PgithubUserName=${{env.GH_USER_NAME}} -PgithubToken=${{secrets.GITHUB_TOKEN}} \ -PgpgPassphrase=${{secrets.GPG_PASSPHRASE}} -PgpgPrivateKey="${{secrets.GPG_PRIVATE_KEY}}" diff --git a/build.gradle b/build.gradle index 40139e8..0e4fc16 100644 --- a/build.gradle +++ b/build.gradle @@ -9,7 +9,7 @@ project.ext { releaseMode = project.hasProperty("releaseMode") } -def scriptsUrl = 'https://raw.githubusercontent.com/reportportal/gradle-scripts/' + (releaseMode ? getProperty('scripts.version') : 'master') +def scriptsUrl = 'https://raw.githubusercontent.com/reportportal/gradle-scripts/' + (releaseMode ? '5.10.0' : 'master') apply from: "$scriptsUrl/build-docker.gradle" apply from: "$scriptsUrl/build-commons.gradle" @@ -54,9 +54,9 @@ ext['snakeyaml.version'] = '1.31' dependencies { if (releaseMode) { - implementation 'com.github.reportportal:commons-events:5.7.3' + implementation 'com.github.reportportal:commons-events:5.10.0' } else { - implementation 'com.github.reportportal:commons-events:35a1246f' + implementation 'com.github.reportportal:commons-events:e337f8b7be' } implementation group: 'org.json', name: 'json', version: '20220320' From ffdd7e789ba4e2ac16792027113d74adc83f3e6d Mon Sep 17 00:00:00 2001 From: Hleb Kanonik Date: Mon, 2 Oct 2023 14:47:52 +0200 Subject: [PATCH 25/27] Create dockerhub-release.yaml --- .github/workflows/dockerhub-release.yaml | 71 ++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 .github/workflows/dockerhub-release.yaml diff --git a/.github/workflows/dockerhub-release.yaml b/.github/workflows/dockerhub-release.yaml new file mode 100644 index 0000000..f874986 --- /dev/null +++ b/.github/workflows/dockerhub-release.yaml @@ -0,0 +1,71 @@ +name: Retag RC Docker image + +on: + pull_request_review: + types: [submitted] + +env: + AWS_REGION: ${{ vars.AWS_REGION }} # set this to your preferred AWS region, e.g. us-west-1 + ECR_REPOSITORY: ${{ vars.ECR_REPOSITORY }} # set this to your Amazon ECR repository name + TARGET_REGISTRY: ${{ vars.TARGET_REGISTRY }} # set to target regestry (DockerHub, GitHub & etc) + TARGET_REPOSITORY: ${{ vars.TARGET_REPOSITORY }} # set to target repository + PLATFORMS: ${{ vars.BUILD_PLATFORMS }} # set target build platforms. By default linux/amd64 + RELEASE_MODE: ${{ vars.RELEASE_MODE }} + +jobs: + retag-image: + name: Retag and push image + runs-on: ubuntu-latest + environment: rc + if: github.event.pull_request.base.ref == 'master' || github.event.pull_request.base.ref == 'main' + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + # role-to-assume: arn:aws:iam::123456789012:role/my-github-actions-role + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + with: + mask-password: 'true' + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.REGESTRY_USERNAME }} + password: ${{ secrets.REGESTRY_PASSWORD }} + + - name: Create variables + id: vars + run: | + echo "tag=$(echo '${{ github.event.pull_request.title }}' | sed -nE 's/.*([0-9]+\.[0-9]+\.[0-9]+).*/\1/p')" >> $GITHUB_OUTPUT + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Retag and Push Docker Image + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + IMAGE_TAG: ${{ steps.vars.outputs.tag }} + run: | + docker buildx imagetools create $ECR_REGISTRY/$ECR_REPOSITORY:latest --tag $TARGET_REGISTRY/$TARGET_REPOSITORY:$IMAGE_TAG --tag $TARGET_REGISTRY/$TARGET_REPOSITORY:latest + + - name: Summarize + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + IMAGE_TAG: ${{ steps.vars.outputs.tag }} + run: | + echo "## General information about the build:" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "- :whale: Docker image: $TARGET_REGISTRY/$TARGET_REPOSITORY:$IMAGE_TAG" >> $GITHUB_STEP_SUMMARY + echo "- :octocat: The commit SHA from which the build was performed: [$GITHUB_SHA](https://github.com/$GITHUB_REPOSITORY/commit/$GITHUB_SHA)" >> $GITHUB_STEP_SUMMARY From bde18cb4e54edf96d858f4e63d426710852fd7c3 Mon Sep 17 00:00:00 2001 From: Ivan_Kustau Date: Mon, 2 Oct 2023 16:34:22 +0300 Subject: [PATCH 26/27] Update app version --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index d25a90b..e5e7658 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -version=develop +version=5.10.0 description=EPAM Report portal. Service jobs dockerServerUrl=unix:///var/run/docker.sock dockerPrepareEnvironment= From d0f0dcd8bac276d57a5e53af8f84435a58039285 Mon Sep 17 00:00:00 2001 From: Hleb Kanonik Date: Mon, 2 Oct 2023 15:50:12 +0200 Subject: [PATCH 27/27] Update rc.yaml --- .github/workflows/rc.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rc.yaml b/.github/workflows/rc.yaml index 7491849..056c033 100644 --- a/.github/workflows/rc.yaml +++ b/.github/workflows/rc.yaml @@ -65,7 +65,9 @@ jobs: GITHUB_TOKEN=${{ secrets.GH_TOKEN }} RELEASE_MODE=${{ env.RELEASE_MODE }} platforms: ${{ env.PLATFORMS }} - tags: ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} + tags: | + ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} + ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:latest - name: Run Trivy vulnerability scanner uses: aquasecurity/trivy-action@master @@ -88,4 +90,4 @@ jobs: echo "## General information about the build:" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "- :gift: Docker image in Amazon ECR: ecr/$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_STEP_SUMMARY - echo "- :octocat: The commit SHA from which the build was performed: [$GITHUB_SHA](https://github.com/$GITHUB_REPOSITORY/commit/$GITHUB_SHA)" >> $GITHUB_STEP_SUMMARY \ No newline at end of file + echo "- :octocat: The commit SHA from which the build was performed: [$GITHUB_SHA](https://github.com/$GITHUB_REPOSITORY/commit/$GITHUB_SHA)" >> $GITHUB_STEP_SUMMARY