MLE-24523 Getting reverse proxy tests running again

Lots of little improvements to Jenkinsfile too.

Author: rjrudin

.copyrightconfig

@@ -11,4 +11,4 @@ startyear: 2010
 #   - Dotfiles already skipped automatically
 # Enable by removing the leading '# ' from the next line and editing values.
 # filesexcluded: third_party/*, docs/generated/*.md, assets/*.png, scripts/temp_*.py, vendor/lib.js
-filesexcluded: .github/*, README.md, Jenkinsfile, gradle/*, docker-compose.yml, *.gradle, gradle.properties, gradlew, gradlew.bat, **/test/resources/**, *.md, pom.xml
+filesexcluded: .github/*, README.md, Jenkinsfile, gradle/*, docker-compose.yaml, docker-compose.yml, *.gradle, gradle.properties, gradlew, gradlew.bat, **/test/resources/**, *.md, pom.xml

Jenkinsfile

@@ -1,6 +1,6 @@
 @Library('shared-libraries') _
 
-def getJava() {
+def getJavaHomePath() {
 	if (env.JAVA_VERSION == "JAVA21") {
 		return "/home/builder/java/jdk-21.0.1"
 	} else {
@@ -38,9 +38,10 @@ def runTests(String image) {
 			export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR
 			export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH
 			cd java-client-api
-			// Ensure all modules can be built first.
+
+			echo "Ensure all subprojects can be built first."
       ./gradlew clean build -x test
-			mkdir -p marklogic-client-api/build/test-results/test
+
 			./gradlew marklogic-client-api:test  || true
 	'''
 
@@ -49,7 +50,6 @@ def runTests(String image) {
 			export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR
 			export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH
 			cd java-client-api
-			mkdir -p ml-development-tools/build/test-results/test
 			./gradlew ml-development-tools:test || true
 	'''
 
@@ -84,28 +84,41 @@ def runTests(String image) {
 def runTestsWithReverseProxy(String image) {
 	setupDockerMarkLogic(image)
 
+	sh label: 'run marklogic-client-api tests with reverse proxy', script: '''#!/bin/bash
+			export JAVA_HOME=$JAVA_HOME_DIR
+			export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR
+			export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH
+			cd java-client-api
+
+			echo "Ensure all subprojects can be built first."
+      ./gradlew clean build -x test
+
+      echo "Running marklogic-client-api tests with reverse proxy."
+			./gradlew -PtestUseReverseProxyServer=true runReverseProxyServer marklogic-client-api:test || true
+	'''
+
 	sh label: 'run fragile functional tests with reverse proxy', script: '''#!/bin/bash
 			export JAVA_HOME=$JAVA_HOME_DIR
 			export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR
 			export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH
 			cd java-client-api
-			./gradlew -PtestUseReverseProxyServer=true test-app:runReverseProxyServer marklogic-client-api-functionaltests:runFragileTests || true
+			./gradlew -PtestUseReverseProxyServer=true runReverseProxyServer marklogic-client-api-functionaltests:runFragileTests || true
 	'''
 
 	sh label: 'run fast functional tests with reverse proxy', script: '''#!/bin/bash
 			export JAVA_HOME=$JAVA_HOME_DIR
 			export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR
 			export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH
 			cd java-client-api
-			./gradlew -PtestUseReverseProxyServer=true test-app:runReverseProxyServer marklogic-client-api-functionaltests:runFastFunctionalTests || true
+			./gradlew -PtestUseReverseProxyServer=true runReverseProxyServer marklogic-client-api-functionaltests:runFastFunctionalTests || true
 	'''
 
 	sh label: 'run slow functional tests with reverse proxy', script: '''#!/bin/bash
 			export JAVA_HOME=$JAVA_HOME_DIR
 			export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR
 			export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH
 			cd java-client-api
-			./gradlew -PtestUseReverseProxyServer=true test-app:runReverseProxyServer marklogic-client-api-functionaltests:runSlowFunctionalTests || true
+			./gradlew -PtestUseReverseProxyServer=true runReverseProxyServer marklogic-client-api-functionaltests:runSlowFunctionalTests || true
 	'''
 
 	postProcessTestResults()
@@ -149,43 +162,45 @@ pipeline {
 
 	parameters {
 		booleanParam(name: 'regressions', defaultValue: false, description: 'indicator if build is for regressions')
-		string(name: 'Email', defaultValue: '', description: 'Who should I say send the email to?')
-		string(name: 'JAVA_VERSION', defaultValue: 'JAVA8', description: 'Who should I say send the email to?')
+		string(name: 'JAVA_VERSION', defaultValue: 'JAVA17', description: 'Either JAVA17 or JAVA21')
 	}
 
 	environment {
-		JAVA_HOME_DIR = getJava()
+		JAVA_HOME_DIR = getJavaHomePath()
 		GRADLE_DIR = ".gradle"
 		DMC_USER = credentials('MLBUILD_USER')
 		DMC_PASSWORD = credentials('MLBUILD_PASSWORD')
 	}
 
 	stages {
+
 		stage('pull-request-tests') {
 			when {
 				not {
 					expression { return params.regressions }
 				}
 			}
 			steps {
-				setupDockerMarkLogic("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi-rootless:11.3.2-ubi-rootless-2.2.2")
+				setupDockerMarkLogic("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-12")
 				sh label: 'run marklogic-client-api tests', script: '''#!/bin/bash
           export JAVA_HOME=$JAVA_HOME_DIR
           export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR
           export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH
           cd java-client-api
-          // Ensure all modules can be built first.
+
+          echo "Ensure all subprojects can be built first."
           ./gradlew clean build -x test
 
-          // Run a sufficient number of tests to verify the PR.
-          ./gradlew cleanTest marklogic-client-api:test || true
+          echo "Run a sufficient number of tests to verify the PR."
+					./gradlew marklogic-client-api:test --tests ReadDocumentPageTest || true
+
+					echo "Run a test with the reverse proxy server to ensure it's fine."
+					./gradlew -PtestUseReverseProxyServer=true runReverseProxyServer marklogic-client-api-functionaltests:test --tests SearchWithPageLengthTest || true
         '''
-				// Omitting this until MLE-24523 can be addressed
-				// ./gradlew -PtestUseReverseProxyServer=true test-app:runReverseProxyServer marklogic-client-api-functionaltests:runFastFunctionalTests || true
-				junit '**/build/**/TEST*.xml'
 			}
 			post {
 				always {
+					junit '**/build/**/TEST*.xml'
 					updateWorkspacePermissions()
 					tearDownDocker()
 				}
@@ -218,36 +233,35 @@ pipeline {
 				}
 			}
 			steps {
-				runTests("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi-rootless:11.3.2-ubi-rootless-2.2.2")
-				junit '**/build/**/TEST*.xml'
+				runTests("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-11")
 			}
 			post {
 				always {
+					junit '**/build/**/TEST*.xml'
 					updateWorkspacePermissions()
 					tearDownDocker()
 				}
 			}
 		}
 
-		// Omitting this until MLE-24523 can be addressed
-//		stage('regressions-11-reverseProxy') {
-//			when {
-//				allOf {
-//					branch 'develop'
-//					expression {return params.regressions}
-//				}
-//			}
-//			steps {
-//				runTestsWithReverseProxy("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-11")
-//				junit '**/build/**/TEST*.xml'
-//			}
-//			post {
-//				always {
-//					updateWorkspacePermissions()
-//					tearDownDocker()
-//				}
-//			}
-//		}
+		stage('regressions-12-reverseProxy') {
+			when {
+				allOf {
+					branch 'develop'
+					expression {return params.regressions}
+				}
+			}
+			steps {
+				runTestsWithReverseProxy("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-12")
+			}
+			post {
+				always {
+					junit '**/build/**/TEST*.xml'
+					updateWorkspacePermissions()
+					tearDownDocker()
+				}
+			}
+		}
 
 		stage('regressions-12') {
 			when {
@@ -257,18 +271,18 @@ pipeline {
 				}
 			}
 			steps {
-				runTests("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi-rootless:12.0.0-ubi-rootless-2.2.0")
-				junit '**/build/**/TEST*.xml'
+				runTests("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-12")
 			}
 			post {
 				always {
+					junit '**/build/**/TEST*.xml'
 					updateWorkspacePermissions()
 					tearDownDocker()
 				}
 			}
 		}
 
-		stage('regressions-10.0') {
+		stage('regressions-10') {
 			when {
 				allOf {
 					branch 'develop'
@@ -277,10 +291,10 @@ pipeline {
 			}
 			steps {
 				runTests("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-10")
-				junit '**/build/**/TEST*.xml'
 			}
 			post {
 				always {
+					junit '**/build/**/TEST*.xml'
 					updateWorkspacePermissions()
 					tearDownDocker()
 				}

docker-compose.yaml

@@ -18,10 +18,7 @@ services:
       - ${MARKLOGIC_LOGS_VOLUME}:/var/opt/MarkLogic/Logs
     ports:
       - "8000-8002:8000-8002"
-      - "8010-8014:8010-8014"
-      - "8022:8022"
-      - "8054-8059:8054-8059"
-      - "8093:8093"
+      - "8010-8015:8010-8015" # Range of ports used by app servers, at least one of which - 8015 - is created by a test.
 
 volumes:
   marklogicLogs:

marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/TestDatabaseClientConnection.java

@@ -124,11 +124,10 @@ void invalidPort() {
 		DatabaseClient client = newDatabaseClientBuilder().withPort(assumedInvalidPort).build();
 
 		MarkLogicIOException ex = Assertions.assertThrows(MarkLogicIOException.class, () -> client.checkConnection());
-		String expected = "Error occurred while calling http://localhost:60123/v1/ping; java.net.ConnectException: " +
-							  "Failed to connect to localhost/127.0.0.1:60123 ; possible reasons for the error include " +
+		String expected = "Failed to connect to localhost/127.0.0.1:60123 ; possible reasons for the error include " +
 							  "that a MarkLogic app server may not be listening on the port, or MarkLogic was stopped " +
 							  "or restarted during the request; check the MarkLogic server logs for more information.";
-		assertEquals(expected, ex.getMessage());
+		assertTrue(ex.getMessage().contains(expected), "Unexpected error: " + ex.getMessage());
 	}
 
   @Test

marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/BulkIOCallersFnTest.java

@@ -42,7 +42,7 @@ public class BulkIOCallersFnTest extends BasicJavaClientREST {
     private static String host = null;
 
     private static int modulesPort = 8000;
-    private static int restTestport = 8093;
+    private static int restTestport = 8015;
     private static String restServerName = "TestDynamicIngest";
 
     private static SecurityContext secContext = null;

test-app/README.md

@@ -34,3 +34,8 @@ You can also specify custom mappings via the Gradle task. For example, if you ha
 port 8123 and you want to associate a path of "/my/custom/server" to it, you can do:
 
     ./gradlew runBlock -PrpsCustomMappings=/my/custom/server,8123
+
+To run one or more tests with the reverse proxy server being started, the tests being run, and then the server being 
+stopped, do the following (you can see examples of this in the project `Jenkinsfile` as well):
+
+		./gradlew -PtestUseReverseProxyServer=true runReverseProxyServer marklogic-client-api:test --tests ReadDocumentPageTest