Merge pull request #12813 from rabbitmq/bp/v4.0.x/pr-12775

Fixes for test flakes (backport #12775)

Author: dcorbacho

deps/oauth2_client/test/system_SUITE.erl

@@ -90,9 +90,11 @@ init_per_group(https, Config) ->
     application:ensure_all_started(cowboy),
     Config0 = rabbit_ct_helpers:run_setup_steps(Config),
     CertsDir = ?config(rmq_certsdir, Config0),
+    ct:log("certsdir: ~p", [CertsDir]),
     CaCertFile = filename:join([CertsDir, "testca", "cacert.pem"]),
     WrongCaCertFile = filename:join([CertsDir, "server", "server.pem"]),
     [{group, https},
+        {certsDir, CertsDir},
         {oauth_provider_id, <<"uaa">>},
         {oauth_provider, build_https_oauth_provider(<<"uaa">>, CaCertFile)},
         {oauth_provider_with_issuer, keep_only_issuer_and_ssl_options(
@@ -110,29 +112,29 @@ init_per_group(https_down, Config) ->
 
     [{issuer, build_issuer("https")},
         {oauth_provider_id, <<"uaa">>},
-        {oauth_provider, build_https_oauth_provider(<<"uaa">>, CaCertFile)} | Config];
+        {oauth_provider, build_https_oauth_provider(<<"uaa">>, CaCertFile)} | Config0];
 
 init_per_group(openid_configuration_with_path, Config) ->
     [{use_openid_configuration_with_path, true} | Config];
 
 init_per_group(with_all_oauth_provider_settings, Config) ->
     Config0 = rabbit_ct_helpers:run_setup_steps(Config),
-    CertsDir = ?config(rmq_certsdir, Config0),
+    CertsDir = ?config(certsDir, Config0),
     CaCertFile = filename:join([CertsDir, "testca", "cacert.pem"]),
-
+    ct:log("certsdir: ~p", [CertsDir]),
     [{with_all_oauth_provider_settings, true},
      {oauth_provider_id, <<"uaa">>},
-     {oauth_provider, build_https_oauth_provider(<<"uaa">>, CaCertFile)} | Config];
+     {oauth_provider, build_https_oauth_provider(<<"uaa">>, CaCertFile)} | Config0];
 
 init_per_group(without_all_oauth_providers_settings, Config) ->
     Config0 = rabbit_ct_helpers:run_setup_steps(Config),
-    CertsDir = ?config(rmq_certsdir, Config0),
+    CertsDir = ?config(certsDir, Config0),
     CaCertFile = filename:join([CertsDir, "testca", "cacert.pem"]),
-
+    ct:log("certsdir: ~p", [CertsDir]),
     [{with_all_oauth_provider_settings, false},
         {oauth_provider_id, <<"uaa">>},
         {oauth_provider, keep_only_issuer_and_ssl_options(
-            build_https_oauth_provider(<<"uaa">>, CaCertFile))} | Config];
+            build_https_oauth_provider(<<"uaa">>, CaCertFile))} | Config0];
 
 init_per_group(with_default_oauth_provider, Config) ->
     OAuthProvider = ?config(oauth_provider, Config),
@@ -244,9 +246,8 @@ init_per_testcase(TestCase, Config) ->
 
     case ?config(group, Config) of
         https ->
-            ct:log("Start https with expectations ~p", [ListOfExpectations]),
-            start_https_oauth_server(?AUTH_PORT, ?config(rmq_certsdir, Config),
-                ListOfExpectations);
+            start_https_oauth_server(?AUTH_PORT, ?config(certsDir, Config),
+                ListOfExpectations);        
         _ ->
             do_nothing
     end,
@@ -262,6 +263,8 @@ end_per_testcase(_, Config) ->
     case ?config(group, Config) of
         https ->
             stop_https_auth_server();
+        without_all_oauth_providers_settings ->
+            stop_https_auth_server();
         _ ->
             do_nothing
     end,
@@ -618,8 +621,8 @@ start_https_oauth_server(Port, CertsDir, Expectations) when is_list(Expectations
         {'_', [{Path, oauth_http_mock, Expected} || #{request := #{path := Path}}
             = Expected <- Expectations ]}
     ]),
-    ct:log("start_https_oauth_server with expectation list : ~p -> dispatch: ~p",
-        [Expectations, Dispatch]),
+    ct:log("start_https_oauth_server with expectation : ~p  -> dispatch: ~p . certsDir: ~p",
+        [Expectations, Dispatch, CertsDir]),
     {ok, _} = cowboy:start_tls(
         mock_http_auth_listener,
             [{port, Port},
@@ -630,8 +633,8 @@ start_https_oauth_server(Port, CertsDir, Expectations) when is_list(Expectations
 
 start_https_oauth_server(Port, CertsDir, #{request := #{path := Path}} = Expected) ->
     Dispatch = cowboy_router:compile([{'_', [{Path, oauth_http_mock, Expected}]}]),
-    ct:log("start_https_oauth_server with expectation : ~p  -> dispatch: ~p",
-        [Expected, Dispatch]),
+    ct:log("start_https_oauth_server with expectation : ~p  -> dispatch: ~p . certsDir: ~p",
+        [Expected, Dispatch, CertsDir]),
     {ok, _} = cowboy:start_tls(
         mock_http_auth_listener,
             [{port, Port},

deps/rabbit/test/amqp_client_SUITE.erl

@@ -341,6 +341,7 @@ init_per_testcase(Testcase, Config) ->
 
 end_per_testcase(Testcase, Config) ->
     %% Assert that every testcase cleaned up.
+    rabbit_ct_broker_helpers:rpc(Config, 0, ?MODULE, delete_queues, []),
     eventually(?_assertEqual([], rpc(Config, rabbit_amqqueue, list, []))),
     %% Wait for sessions to terminate before starting the next test case.
     eventually(?_assertEqual([], rpc(Config, rabbit_amqp_session, list_local, []))),
@@ -350,6 +351,10 @@ end_per_testcase(Testcase, Config) ->
                              get_global_counters(Config))),
     rabbit_ct_helpers:testcase_finished(Config, Testcase).
 
+delete_queues() ->
+    [rabbit_amqqueue:delete(Q, false, false, <<"dummy">>)
+     || Q <- rabbit_amqqueue:list()].
+
 reliable_send_receive_with_outcomes_classic_queue(Config) ->
     reliable_send_receive_with_outcomes(<<"classic">>, Config).
 

deps/rabbitmq_ct_helpers/src/rabbit_ct_helpers.erl

@@ -584,9 +584,14 @@ ensure_rabbitmq_queues_cmd(Config) ->
 
 ensure_ssl_certs(Config) ->
     SrcDir = ?config(rabbitmq_ct_helpers_srcdir, Config),
+    UniqueDir = io_lib:format(
+                  "~s2-~p",
+                  [node(), erlang:unique_integer([positive,monotonic])]),
     CertsMakeDir = filename:join([SrcDir, "tools", "tls-certs"]),
     PrivDir = ?config(priv_dir, Config),
-    CertsDir = filename:join(PrivDir, "certs"),
+    CertsDir = filename:join([PrivDir, UniqueDir, "certs"]),
+    _ = filelib:ensure_dir(CertsDir),
+    _ = file:make_dir(CertsDir),
     CertsPwd = proplists:get_value(rmq_certspwd, Config, ?SSL_CERT_PASSWORD),
     Cmd = [
       "PASSWORD=" ++ CertsPwd,