Add tests to make sure our test framework tolerates maven reruns even with test profiles

integration-tests/maven/src/test/java/io/quarkus/maven/it/FlakyDevMojoIT.java → integration-tests/maven/src/test/java/io/quarkus/maven/it/AuxiliaryTestsForDevMojoIT.java

@@ -19,7 +19,7 @@
  * This test has been isolated as it is very flaky and causing issues with Develocity PTS.
  */
 @DisableForNative
-public class FlakyDevMojoIT extends RunAndCheckMojoTestBase {
+public class AuxiliaryTestsForDevMojoIT extends RunAndCheckMojoTestBase {
 
     protected DevModeClient devModeClient = new DevModeClient(getPort());
 

integration-tests/maven/src/test/java/io/quarkus/maven/it/SurefireRerunFailedTestsIT.java

@@ -0,0 +1,65 @@
+package io.quarkus.maven.it;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.awaitility.Awaitility.await;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.maven.shared.invoker.MavenInvocationException;
+import org.junit.jupiter.api.Test;
+
+import io.quarkus.maven.it.verifier.MavenProcessInvocationResult;
+import io.quarkus.maven.it.verifier.RunningInvoker;
+
+/**
+ * Test that `mvn -Dsurefire.rerunFailingTestsCount=xx test` (where xx is more than one) works with QuarkusTests,
+ * win the case when a test fails.
+ */
+@DisableForNative
+public class SurefireRerunFailedTestsIT extends RunAndCheckMojoTestBase {
+
+    @Test
+    public void testFlakyTestsInSingleProfileCase() throws MavenInvocationException, IOException {
+        testDir = initProject("projects/test-flaky-test-single-profile", "projects/test-flaky-test-single-profile-processed");
+        final RunningInvoker running = new RunningInvoker(testDir, false);
+
+        // trigger mvn verify -Dsurefire.rerunFailingTestsCount=2
+        final String[] mvnArgs = new String[] { "verify", "-Dsurefire.rerunFailingTestsCount=3" };
+        final MavenProcessInvocationResult result = running.execute(Arrays.asList(mvnArgs), Collections.emptyMap());
+        await().atMost(5, TimeUnit.MINUTES)
+                .until(() -> result.getProcess() != null && !result.getProcess()
+                        .isAlive());
+        final String processLog = running.log();
+        assertEquals(running.getResult()
+                .getExitCode(), 0);
+        assertThat(processLog).containsIgnoringCase("BUILD SUCCESS");
+        // Also do a sense check that the test is functioning correctly and did have a failed execution first
+        assertThat(processLog).containsIgnoringCase("deliberate failure run 1");
+        assertThat(processLog).containsIgnoringCase("Run 2: PASS");
+    }
+
+    @Test
+    public void testFlakyTestsInMultipleProfileCase() throws MavenInvocationException, IOException {
+        testDir = initProject("projects/test-flaky-test-multiple-profiles",
+                "projects/test-flaky-test-multiple-profiles-processed");
+        final RunningInvoker running = new RunningInvoker(testDir, false);
+
+        // trigger mvn verify -Dsurefire.rerunFailingTestsCount=2
+        final String[] mvnArgs = new String[] { "verify", "-Dsurefire.rerunFailingTestsCount=3" };
+        final MavenProcessInvocationResult result = running.execute(Arrays.asList(mvnArgs), Collections.emptyMap());
+        await().atMost(5, TimeUnit.MINUTES)
+                .until(() -> result.getProcess() != null && !result.getProcess()
+                        .isAlive());
+        final String processLog = running.log();
+        assertEquals(running.getResult()
+                .getExitCode(), 0);
+        assertThat(processLog).containsIgnoringCase("BUILD SUCCESS");
+        // Also do a sense check that the test is functioning correctly and did have a failed execution first
+        assertThat(processLog).containsIgnoringCase("deliberate failure run 1");
+        assertThat(processLog).containsIgnoringCase("Run 2: PASS");
+    }
+}

integration-tests/maven/src/test/resources-filtered/projects/test-flaky-test-multiple-profiles/pom.xml

@@ -0,0 +1,108 @@
+<?xml version="1.0"?>
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>org.acme</groupId>
+  <artifactId>acme</artifactId>
+  <version>1.0-SNAPSHOT</version>
+  <properties>
+    <quarkus.platform.group-id>io.quarkus</quarkus.platform.group-id>
+    <quarkus.platform.artifact-id>quarkus-bom</quarkus.platform.artifact-id>
+    <quarkus.platform.version>@project.version@</quarkus.platform.version>
+    <quarkus-plugin.version>@project.version@</quarkus-plugin.version>
+    <compiler-plugin.version>${compiler-plugin.version}</compiler-plugin.version>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <maven.compiler.source>${maven.compiler.source}</maven.compiler.source>
+    <maven.compiler.target>${maven.compiler.target}</maven.compiler.target>
+
+    <!-- do not update this dependency, it needs to be stable for testing -->
+    <webjar.jquery-ui.version>1.13.0</webjar.jquery-ui.version>
+  </properties>
+  <dependencyManagement>
+    <dependencies>
+      <!-- insert managed dependencies here -->
+      <dependency>
+        <groupId>\${quarkus.platform.group-id}</groupId>
+        <artifactId>\${quarkus.platform.artifact-id}</artifactId>
+        <version>\${quarkus.platform.version}</version>
+        <type>pom</type>
+        <scope>import</scope>
+      </dependency>
+    </dependencies>
+  </dependencyManagement>
+  <dependencies>
+    <!-- insert test dependencies here -->
+    <dependency>
+      <groupId>io.quarkus</groupId>
+      <artifactId>quarkus-resteasy</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>io.quarkus</groupId>
+      <artifactId>quarkus-smallrye-context-propagation</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>io.quarkus</groupId>
+      <artifactId>quarkus-websockets</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>io.smallrye.common</groupId>
+      <artifactId>smallrye-common-vertx-context</artifactId>
+      <version>1.13.2</version>
+    </dependency>
+    <dependency>
+      <groupId>org.webjars</groupId>
+      <artifactId>jquery-ui</artifactId>
+      <version>\${webjar.jquery-ui.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>commons-io</groupId>
+      <artifactId>commons-io</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>io.quarkus</groupId>
+      <artifactId>quarkus-junit5</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>io.rest-assured</groupId>
+      <artifactId>rest-assured</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+  <build>
+    <plugins>
+      <plugin>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <version>\${compiler-plugin.version}</version>
+      </plugin>
+      <plugin>
+        <groupId>io.quarkus</groupId>
+        <artifactId>quarkus-maven-plugin</artifactId>
+        <version>\${quarkus-plugin.version}</version>
+        <executions>
+          <execution>
+            <goals>
+              <goal>generate-code</goal>
+              <goal>generate-code-tests</goal>
+              <goal>build</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+  <profiles>
+    <profile>
+      <id>native</id>
+      <properties>
+        <quarkus.native.enabled>true</quarkus.native.enabled>
+      </properties>
+    </profile>
+    <profile>
+      <id>customOutputDir</id>
+      <build>
+        <directory>target-other</directory>
+      </build>
+    </profile>
+  </profiles>
+</project>

integration-tests/maven/src/test/resources-filtered/projects/test-flaky-test-multiple-profiles/src/main/java/org/acme/HelloResource.java

@@ -0,0 +1,16 @@
+package org.acme;
+
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.core.MediaType;
+
+@Path("/hello")
+public class HelloResource {
+
+    @GET
+    @Produces(MediaType.TEXT_PLAIN)
+    public String hello() {
+        return "hello";
+    }
+}

integration-tests/maven/src/test/resources-filtered/projects/test-flaky-test-multiple-profiles/src/test/java/org/acme/Flaker.java

@@ -0,0 +1,13 @@
+package org.acme;
+
+public class Flaker {
+    private static int count = 0;
+
+    public static void flake() throws FlakingException {
+        count++;
+
+        if (count == 1) {
+            throw new FlakingException("deliberate failure run " + count);
+        }
+    }
+}

integration-tests/maven/src/test/resources-filtered/projects/test-flaky-test-multiple-profiles/src/test/java/org/acme/FlakingException.java

@@ -0,0 +1,7 @@
+package org.acme;
+
+public class FlakingException extends Exception {
+    public FlakingException(String message) {
+        super(message);
+    }
+}

integration-tests/maven/src/test/resources-filtered/projects/test-flaky-test-multiple-profiles/src/test/java/org/acme/FlakyHelloResourceWithAnotherProfileTest.java

@@ -0,0 +1,45 @@
+package org.acme;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.CoreMatchers.is;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+
+import org.junit.jupiter.api.Test;
+
+import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.QuarkusTestProfile;
+import io.quarkus.test.junit.TestProfile;
+
+@TestProfile(FlakyHelloResourceWithAnotherProfileTest.AnotherProfile.class)
+@QuarkusTest
+public class FlakyHelloResourceWithAnotherProfileTest {
+
+    @Test
+    public void testHelloEndpoint() throws Exception {
+        given()
+                .when().get("/hello")
+                .then()
+                .statusCode(200)
+                .body(is("hello"));
+
+        // Should fail the first invocation and pass subsequent ones
+        // Use the system classloader to give a consistent view of static variables
+        Class flakerClass = ClassLoader.getSystemClassLoader().loadClass(Flaker.class.getName());
+        Method flakeMethod = flakerClass.getMethod("flake");
+
+        try {
+            flakeMethod.invoke(null);
+        } catch (InvocationTargetException e) {
+            throw new FlakingException(e.getCause().getMessage());
+        }
+    }
+
+    public static class AnotherProfile implements QuarkusTestProfile {
+
+        public AnotherProfile() {
+        }
+    }
+
+}

integration-tests/maven/src/test/resources-filtered/projects/test-flaky-test-multiple-profiles/src/test/java/org/acme/FlakyHelloResourceWithSomeProfileTest.java

@@ -0,0 +1,44 @@
+package org.acme;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.CoreMatchers.is;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+
+import org.junit.jupiter.api.Test;
+
+import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.QuarkusTestProfile;
+import io.quarkus.test.junit.TestProfile;
+
+@TestProfile(FlakyHelloResourceWithSomeProfileTest.SomeProfile.class)
+@QuarkusTest
+public class FlakyHelloResourceWithSomeProfileTest {
+
+    @Test
+    public void testHelloEndpoint() throws Exception {
+        given()
+                .when().get("/hello")
+                .then()
+                .statusCode(200)
+                .body(is("hello"));
+
+        // Should fail the first invocation and pass subsequent ones
+        // Use the system classloader to give a consistent view of static variables
+        Class flakerClass = ClassLoader.getSystemClassLoader().loadClass(Flaker.class.getName());
+        Method flakeMethod = flakerClass.getMethod("flake");
+        try {
+            flakeMethod.invoke(null);
+        } catch (InvocationTargetException e) {
+            throw new FlakingException(e.getCause().getMessage());
+        }
+    }
+
+    public static class SomeProfile implements QuarkusTestProfile {
+
+        public SomeProfile() {
+        }
+    }
+
+}