diff --git a/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/SolrExporter.java b/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/SolrExporter.java
index ebcff4e87a6..d227a8b3b54 100644
--- a/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/SolrExporter.java
+++ b/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/SolrExporter.java
@@ -102,7 +102,9 @@ void start() throws IOException {
   }
 
   void stop() {
-    httpServer.stop();
+    if (httpServer != null) {
+      httpServer.stop();
+    }
 
     metricsCollector.removeObserver(prometheusCollector);
 
diff --git a/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/exporter/SolrExporterTestBase.java b/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/exporter/SolrExporterTestBase.java
index 46b7c45e538..70061fa914f 100644
--- a/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/exporter/SolrExporterTestBase.java
+++ b/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/exporter/SolrExporterTestBase.java
@@ -20,6 +20,8 @@
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
+import java.lang.invoke.MethodHandles;
+import java.net.BindException;
 import java.net.ServerSocket;
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -36,11 +38,15 @@
 import org.apache.solr.prometheus.PrometheusExporterTestBase;
 import org.apache.solr.prometheus.utils.Helpers;
 import org.junit.After;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /** Test base class. */
 @ThreadLeakScope(ThreadLeakScope.Scope.NONE)
 public class SolrExporterTestBase extends PrometheusExporterTestBase {
 
+  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
   private SolrExporter solrExporter;
   private CloseableHttpClient httpClient;
   private int promtheusExporterPort;
@@ -57,20 +63,34 @@ public void tearDown() throws Exception {
 
   protected void startMetricsExporterWithConfiguration(String scrapeConfiguration)
       throws Exception {
-    try (ServerSocket socket = new ServerSocket(0)) {
-      promtheusExporterPort = socket.getLocalPort();
-    }
 
-    solrExporter =
-        new SolrExporter(
-            promtheusExporterPort,
-            25,
-            10,
-            SolrScrapeConfiguration.solrCloud(cluster.getZkServer().getZkAddress()),
-            Helpers.loadConfiguration(scrapeConfiguration),
-            "test");
+    final int maxAttempts = 3;
+    for (int attempt = 1; attempt <= maxAttempts; attempt++) {
+      try (ServerSocket socket = new ServerSocket(0)) {
+        promtheusExporterPort = socket.getLocalPort();
+      }
+
+      try {
+        solrExporter =
+            new SolrExporter(
+                promtheusExporterPort,
+                25,
+                10,
+                SolrScrapeConfiguration.solrCloud(cluster.getZkServer().getZkAddress()),
+                Helpers.loadConfiguration(scrapeConfiguration),
+                "test");
+
+        solrExporter.start();
+        break;
+      } catch (BindException e) {
+        solrExporter.stop();
+        if (attempt == maxAttempts) {
+          throw e;
+        }
+        log.warn("Failed to start exporter with port bind exception, retrying on a new port");
+      }
+    }
 
-    solrExporter.start();
     httpClient = HttpClients.createDefault();
 
     for (int i = 0; i < 50; ++i) {