Make crowd test more stable (#344)

* Forcefully terminate Bitbucket pods * Update crowd_test.go * Update crowd_test.go * Try getting atl_token 5 times with a 5 second interval * Add more logging on atl_token retrieval * Try 2 times * Increase tests timeout * Increase tests timeout --------- Co-authored-by: Yevhen Ivantsov <[email protected]>
atlassian-labs · Nov 15, 2023 · ba3b177 · ba3b177
1 parent c410930
commit ba3b177
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 3 deletions.
diff --git a/.github/workflows/e2e-test-no-domain.yaml b/.github/workflows/e2e-test-no-domain.yaml
@@ -113,7 +113,7 @@ jobs:
           # boto3 ignores AWS creds env vars for some reason
           mkdir -p ~/.aws
           echo -e "[default]\naws_access_key_id = ${AWS_ACCESS_KEY_ID}\naws_secret_access_key = ${AWS_SECRET_ACCESS_KEY}\naws_session_token = ${AWS_SESSION_TOKEN}" > ~/.aws/credentials
-          go test ./e2etest -v -timeout 85m -run Installer | tee ./e2etest/artifacts/e2etest.log
+          go test ./e2etest -v -timeout 100m -run Installer | tee ./e2etest/artifacts/e2etest.log
 
       - name: Upload test log files
         if: always()

diff --git a/.github/workflows/e2e-test-with-domain.yaml b/.github/workflows/e2e-test-with-domain.yaml
@@ -113,7 +113,7 @@ jobs:
           # boto3 ignores AWS creds env vars for some reason
           mkdir -p ~/.aws
           echo -e "[default]\naws_access_key_id = ${AWS_ACCESS_KEY_ID}\naws_secret_access_key = ${AWS_SECRET_ACCESS_KEY}\naws_session_token = ${AWS_SESSION_TOKEN}" > ~/.aws/credentials
-          go test ./e2etest -v -timeout 85m -run Installer | tee ./e2etest/artifacts/e2etest.log
+          go test ./e2etest -v -timeout 100m -run Installer | tee ./e2etest/artifacts/e2etest.log
 
       - name: Upload test log files
         if: always()

diff --git a/test/e2etest/crowd_test.go b/test/e2etest/crowd_test.go
@@ -233,18 +233,28 @@ func crowdTests(t *testing.T, testConfig TestConfig, bitbucketURL string, crowdU
 	createNewCrowdUser(t, userName, crowdURL, testConfig.CrowdPassword)
 
 	// before making calls to Bitbucket make sure we land on the same node and avoid using sticky cookie in requests
-	// scale bitbucket to 1 replica instead of 3
+	// scale bitbucket to 1 replica instead of 3.
 	log.Print("Scaling Bitbucket to 1")
 	_, kubectlError := k8s.RunKubectlAndGetOutputE(t, getKubectlOptions(t, testConfig), "scale", "sts/bitbucket", "-n", "atlassian", "--replicas=1")
 	assert.Nil(t, kubectlError)
+	// we need to give Bitbucket some time to unregister Hazelcast nodes and update cluster setting
+	time.Sleep(15 * time.Second)
 
 	// get BITBUCKETSESSIONID to use in the header in subsequent calls
 	// even though basic auth works, atl_token is different each time
 	bitbucketSessionID := getBitbucketSessionID(bitbucketURL, "admin", testConfig.BitbucketPassword)
 	assert.NotEmptyf(t, bitbucketSessionID, "BITBUCKETSESSIONID cannot be empty")
 
 	// now we need to extract atl_token from the hidden input in HTML response
+	// we will try 5 times, as token is extracted from html output and the test proved
+	// to be quite flaky as the token was missing
 	atlToken := getAtlToken(t, bitbucketURL, bitbucketSessionID)
+	if atlToken == "" {
+		log.Printf("atl_token is empty. Retrying in 5 seconds")
+		time.Sleep(5 * time.Second)
+		atlToken = getAtlToken(t, bitbucketURL, bitbucketSessionID)
+	}
+
 	assert.NotEmptyf(t, atlToken, "atl_token cannot be empty")
 
 	// add a new user directory in Bitbucket