Skip to content

Commit 42dfb92

Browse files
aayush3011annie-mac
aayush3011
and
annie-mac
authored
Adding fixes and test cases for retry policy (Azure#37188)
* Adding fixes and test cases for retry policy * Adding fixes and test cases for retry policy * refactor * one more clean * update * Resolving review comments * Resolving review comments * Resolving review comments * Resolving review comments * Fixing test cases * Resolving review comments * Resolving review comments * Fixing test cases --------- Co-authored-by: annie-mac <[email protected]>
1 parent 383da2b commit 42dfb92

File tree

11 files changed

+479
-159
lines changed

11 files changed

+479
-159
lines changed

‎sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/FaultInjectionWithAvailabilityStrategyTests.java‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4355,8 +4355,7 @@ private void execute(
43554355
regionSwitchHint,
43564356
connectionMode,
43574357
customMinRetryTimeInLocalRegionForWrites,
4358-
nonIdempotentWriteRetriesEnabled,
4359-
connectionMode);
4358+
nonIdempotentWriteRetriesEnabled);
43604359
try {
43614360

43624361
if (clearContainerBeforeExecution) {
@@ -4499,8 +4498,7 @@ private static CosmosAsyncClient buildCosmosClient(
44994498
CosmosRegionSwitchHint regionSwitchHint,
45004499
ConnectionMode connectionMode,
45014500
Duration customMinRetryTimeInLocalRegionForWrites,
4502-
Boolean nonIdempotentWriteRetriesEnabled,
4503-
ConnectionMode connectionMode) {
4501+
Boolean nonIdempotentWriteRetriesEnabled) {
45044502

45054503
CosmosClientTelemetryConfig telemetryConfig = new CosmosClientTelemetryConfig()
45064504
.diagnosticsHandler(new CosmosDiagnosticsLogger());

‎sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/FaultInjectionMetadataRequestRuleTests.java‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -117,9 +117,9 @@ public void faultInjectionServerErrorRuleTests_AddressRefresh_ConnectionDelay()
117117
FaultInjectionResultBuilders
118118
.getResultBuilder(FaultInjectionServerErrorType.CONNECTION_DELAY)
119119
.delay(Duration.ofSeconds(50)) // to simulate http connection timeout
120-
// changed from 2 to 8
121-
// 6 more address refresh request retries have been introduced by WebExceptionRetryPolicy
122-
.times(8)
120+
// changed from 2 to 6
121+
// 4 more address refresh request retries have been introduced by WebExceptionRetryPolicy
122+
.times(6)
123123
.build()
124124
)
125125
.duration(Duration.ofMinutes(10))
@@ -155,7 +155,7 @@ public void faultInjectionServerErrorRuleTests_AddressRefresh_ConnectionDelay()
155155
.block()
156156
.getDiagnostics();
157157
assertThat(cosmosDiagnostics.getContactedRegionNames().size()).isEqualTo(2);
158-
validateFaultInjectionRuleAppliedForAddressResolution(cosmosDiagnostics, addressRefreshConnectionDelay, 8);
158+
validateFaultInjectionRuleAppliedForAddressResolution(cosmosDiagnostics, addressRefreshConnectionDelay, 6);
159159
} catch (CosmosException e) {
160160
fail("Request should be able to succeed by retrying in another region. " + e.getDiagnostics());
161161
}
@@ -186,7 +186,7 @@ public void faultInjectionServerErrorRuleTests_AddressRefresh_ConnectionDelay()
186186
public void faultInjectionServerErrorRuleTests_AddressRefresh_ResponseDelay() throws JsonProcessingException {
187187

188188
// Test to validate if there is http request timeout for address refresh,
189-
// SDK will retry 3 times before fail the request
189+
// SDK will retry 2 times before fail the request
190190

191191
// We need to create a new client because client may have marked region unavailable in other tests
192192
// which can impact the test result
@@ -212,8 +212,8 @@ public void faultInjectionServerErrorRuleTests_AddressRefresh_ResponseDelay() th
212212
.result(
213213
FaultInjectionResultBuilders
214214
.getResultBuilder(FaultInjectionServerErrorType.RESPONSE_DELAY)
215-
.delay(Duration.ofSeconds(6)) // to simulate http request timeout
216-
.times(4)
215+
.delay(Duration.ofSeconds(11)) // to simulate http request timeout
216+
.times(3)
217217
.build()
218218
)
219219
.duration(Duration.ofMinutes(5))
@@ -255,7 +255,7 @@ public void faultInjectionServerErrorRuleTests_AddressRefresh_ResponseDelay() th
255255
CosmosDiagnostics cosmosDiagnostics = e.getDiagnostics();
256256
assertThat(cosmosDiagnostics.getContactedRegionNames().size()).isEqualTo(1);
257257
assertThat(cosmosDiagnostics.getContactedRegionNames().containsAll(Arrays.asList(this.readPreferredLocations.get(0).toLowerCase()))).isTrue();
258-
validateFaultInjectionRuleAppliedForAddressResolution(cosmosDiagnostics, addressRefreshResponseDelay, 4);
258+
validateFaultInjectionRuleAppliedForAddressResolution(cosmosDiagnostics, addressRefreshResponseDelay, 3);
259259
}
260260
} finally {
261261
addressRefreshResponseDelayRule.disable();
@@ -316,8 +316,8 @@ public void faultInjectionServerErrorRuleTests_AddressRefresh_byPartition() {
316316
.result(
317317
FaultInjectionResultBuilders
318318
.getResultBuilder(FaultInjectionServerErrorType.RESPONSE_DELAY)
319-
.delay(Duration.ofSeconds(6)) // to simulate http request timeout
320-
.times(4)
319+
.delay(Duration.ofSeconds(11)) // to simulate http request timeout
320+
.times(3)
321321
.build()
322322
)
323323
.duration(Duration.ofMinutes(5))

‎sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/WebExceptionRetryPolicyTest.java‎

Lines changed: 16 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -41,9 +41,9 @@ public void shouldRetryOnTimeoutForReadOperations() throws Exception {
4141

4242
// 1st Attempt
4343
webExceptionRetryPolicy.onBeforeSendRequest(dsr);
44+
assertThat(dsr.getResponseTimeout()).isEqualTo(Duration.ofSeconds(60));
4445
shouldRetry = webExceptionRetryPolicy.shouldRetry(cosmosException);
4546

46-
assertThat(dsr.getResponseTimeout()).isEqualTo(Duration.ofSeconds(60));
4747
validateSuccess(shouldRetry, ShouldRetryValidator.builder().
4848
nullException().
4949
shouldRetry(true).
@@ -52,10 +52,9 @@ public void shouldRetryOnTimeoutForReadOperations() throws Exception {
5252

5353
// 2nd Attempt
5454
retryContext.addStatusAndSubStatusCode(408, 10002);
55-
webExceptionRetryPolicy.onBeforeSendRequest(dsr);
55+
assertThat(dsr.getResponseTimeout()).isEqualTo(Duration.ofSeconds(60));
5656
shouldRetry = webExceptionRetryPolicy.shouldRetry(cosmosException);
5757

58-
assertThat(dsr.getResponseTimeout()).isEqualTo(Duration.ofSeconds(60));
5958
validateSuccess(shouldRetry, ShouldRetryValidator.builder().
6059
nullException().
6160
shouldRetry(true).
@@ -64,19 +63,7 @@ public void shouldRetryOnTimeoutForReadOperations() throws Exception {
6463

6564
// 3rd Attempt
6665
retryContext.addStatusAndSubStatusCode(408, 10002);
67-
webExceptionRetryPolicy.onBeforeSendRequest(dsr);
68-
shouldRetry = webExceptionRetryPolicy.shouldRetry(cosmosException);
69-
7066
assertThat(dsr.getResponseTimeout()).isEqualTo(Duration.ofSeconds(60));
71-
validateSuccess(shouldRetry, ShouldRetryValidator.builder().
72-
nullException().
73-
shouldRetry(true).
74-
backOffTime(Duration.ofSeconds(0)).
75-
build());
76-
77-
// 4th Attempt - retry is set to false, as we only make 3 retry attempts for now.
78-
retryContext.addStatusAndSubStatusCode(408, 10002);
79-
webExceptionRetryPolicy.onBeforeSendRequest(dsr);
8067
shouldRetry = webExceptionRetryPolicy.shouldRetry(cosmosException);
8168

8269
validateSuccess(shouldRetry, ShouldRetryValidator.builder().
@@ -102,13 +89,13 @@ public void shouldRetryOnTimeoutForMetaDataReadOperations() throws Exception {
10289

10390
//Default HttpTimeout Policy
10491
dsr = RxDocumentServiceRequest.createFromName(mockDiagnosticsClientContext(),
105-
OperationType.Read, "/dbs/db/colls/col/docs/doc", ResourceType.DatabaseAccount);
92+
OperationType.Read, "/dbs/db", ResourceType.DatabaseAccount);
10693

10794
// 1st Attempt
10895
webExceptionRetryPolicy.onBeforeSendRequest(dsr);
96+
assertThat(dsr.getResponseTimeout()).isEqualTo(Duration.ofSeconds(5));
10997
shouldRetry = webExceptionRetryPolicy.shouldRetry(cosmosException);
11098

111-
assertThat(dsr.getResponseTimeout()).isEqualTo(Duration.ofSeconds(5));
11299
validateSuccess(shouldRetry, ShouldRetryValidator.builder().
113100
nullException().
114101
shouldRetry(true).
@@ -117,10 +104,10 @@ public void shouldRetryOnTimeoutForMetaDataReadOperations() throws Exception {
117104

118105
// 2nd Attempt
119106
retryContext.addStatusAndSubStatusCode(408, 10002);
120-
webExceptionRetryPolicy.onBeforeSendRequest(dsr);
107+
assertThat(dsr.getResponseTimeout()).isEqualTo(Duration.ofSeconds(10));
121108
shouldRetry = webExceptionRetryPolicy.shouldRetry(cosmosException);
122109

123-
assertThat(dsr.getResponseTimeout()).isEqualTo(Duration.ofSeconds(10));
110+
124111
validateSuccess(shouldRetry, ShouldRetryValidator.builder().
125112
nullException().
126113
shouldRetry(true).
@@ -129,21 +116,10 @@ public void shouldRetryOnTimeoutForMetaDataReadOperations() throws Exception {
129116

130117
//3rd Attempt
131118
retryContext.addStatusAndSubStatusCode(408, 10002);
132-
webExceptionRetryPolicy.onBeforeSendRequest(dsr);
133-
shouldRetry = webExceptionRetryPolicy.shouldRetry(cosmosException);
134-
135119
assertThat(dsr.getResponseTimeout()).isEqualTo(Duration.ofSeconds(20));
136-
validateSuccess(shouldRetry, ShouldRetryValidator.builder().
137-
nullException().
138-
shouldRetry(true).
139-
backOffTime(Duration.ofSeconds(0)).
140-
build());
141-
142-
// 4th Attempt - retry is set to false, as we only make 3 retry attempts for now.
143-
retryContext.addStatusAndSubStatusCode(408, 10002);
144-
webExceptionRetryPolicy.onBeforeSendRequest(dsr);
145120
shouldRetry = webExceptionRetryPolicy.shouldRetry(cosmosException);
146121

122+
147123
validateSuccess(shouldRetry, ShouldRetryValidator.builder().
148124
nullException().
149125
shouldRetry(false).
@@ -171,9 +147,9 @@ public void shouldRetryOnTimeoutForQueryPlanOperations() throws Exception {
171147

172148
// 1st Attempt
173149
webExceptionRetryPolicy.onBeforeSendRequest(dsr);
150+
assertThat(dsr.getResponseTimeout()).isEqualTo(Duration.ofMillis(500));
174151
shouldRetry = webExceptionRetryPolicy.shouldRetry(cosmosException);
175152

176-
assertThat(dsr.getResponseTimeout()).isEqualTo(Duration.ofMillis(500));
177153
validateSuccess(shouldRetry, ShouldRetryValidator.builder().
178154
nullException().
179155
shouldRetry(true).
@@ -182,31 +158,20 @@ public void shouldRetryOnTimeoutForQueryPlanOperations() throws Exception {
182158

183159
// 2nd Attempt
184160
retryContext.addStatusAndSubStatusCode(408, 10002);
185-
webExceptionRetryPolicy.onBeforeSendRequest(dsr);
161+
// webExceptionRetryPolicy.onBeforeSendRequest(dsr);
162+
assertThat(dsr.getResponseTimeout()).isEqualTo(Duration.ofSeconds(5));
186163
shouldRetry = webExceptionRetryPolicy.shouldRetry(cosmosException);
187164

188-
assertThat(dsr.getResponseTimeout()).isEqualTo(Duration.ofSeconds(5));
189165
validateSuccess(shouldRetry, ShouldRetryValidator.builder().
190166
nullException().
191167
shouldRetry(true).
192168
backOffTime(Duration.ofSeconds(1)).
193169
build());
194170

195-
//3rd Attempt
171+
//3rd Attempt - retry is set to false, as we only make 2 retry attempts for now.
196172
retryContext.addStatusAndSubStatusCode(408, 10002);
197-
webExceptionRetryPolicy.onBeforeSendRequest(dsr);
198-
shouldRetry = webExceptionRetryPolicy.shouldRetry(cosmosException);
199-
173+
// webExceptionRetryPolicy.onBeforeSendRequest(dsr);
200174
assertThat(dsr.getResponseTimeout()).isEqualTo(Duration.ofSeconds(10));
201-
validateSuccess(shouldRetry, ShouldRetryValidator.builder().
202-
nullException().
203-
shouldRetry(true).
204-
backOffTime(Duration.ofSeconds(0)).
205-
build());
206-
207-
// 4th Attempt - retry is set to false, as we only make 3 retry attempts for now.
208-
retryContext.addStatusAndSubStatusCode(408, 10002);
209-
webExceptionRetryPolicy.onBeforeSendRequest(dsr);
210175
shouldRetry = webExceptionRetryPolicy.shouldRetry(cosmosException);
211176

212177
validateSuccess(shouldRetry, ShouldRetryValidator.builder().
@@ -244,7 +209,7 @@ public void shouldNotRetryOnTimeoutForWriteOperations() throws Exception {
244209

245210
//Metadata Write - Should not Retry
246211
dsr = RxDocumentServiceRequest.createFromName(mockDiagnosticsClientContext(),
247-
OperationType.Create, "/dbs/db/colls/col/docs/docId", ResourceType.Database);
212+
OperationType.Create, "/dbs/db", ResourceType.DatabaseAccount);
248213

249214
webExceptionRetryPolicy = new WebExceptionRetryPolicy(new RetryContext());
250215
webExceptionRetryPolicy.onBeforeSendRequest(dsr);
@@ -276,20 +241,19 @@ public void httpNetworkFailureOnAddressRefresh() throws Exception {
276241

277242
// 1st Attempt
278243
webExceptionRetryPolicy.onBeforeSendRequest(dsr);
244+
assertThat(dsr.getResponseTimeout()).isEqualTo(Duration.ofMillis(500));
279245
Mono<ShouldRetryResult> shouldRetry = webExceptionRetryPolicy.shouldRetry(cosmosException);
280246

281-
assertThat(dsr.getResponseTimeout()).isEqualTo(Duration.ofMillis(500));
282247
validateSuccess(shouldRetry, ShouldRetryValidator.builder()
283248
.nullException()
284249
.shouldRetry(true)
285250
.backOffTime(Duration.ofSeconds(0))
286251
.build());
287252

288253
// 2nd Attempt
289-
webExceptionRetryPolicy.onBeforeSendRequest(dsr);
254+
assertThat(dsr.getResponseTimeout()).isEqualTo(Duration.ofSeconds(5));
290255
shouldRetry = webExceptionRetryPolicy.shouldRetry(cosmosException);
291256

292-
assertThat(dsr.getResponseTimeout()).isEqualTo(Duration.ofSeconds(5));
293257
validateSuccess(shouldRetry, ShouldRetryValidator.builder()
294258
.nullException()
295259
.shouldRetry(true)
@@ -298,19 +262,9 @@ public void httpNetworkFailureOnAddressRefresh() throws Exception {
298262

299263

300264
// 3rd Attempt
301-
webExceptionRetryPolicy.onBeforeSendRequest(dsr);
302-
shouldRetry = webExceptionRetryPolicy.shouldRetry(cosmosException);
303-
304265
assertThat(dsr.getResponseTimeout()).isEqualTo(Duration.ofSeconds(10));
305-
validateSuccess(shouldRetry, ShouldRetryValidator.builder()
306-
.nullException()
307-
.shouldRetry(true)
308-
.backOffTime(Duration.ofSeconds(0))
309-
.build());
310-
311-
// 4th Attempt
312-
webExceptionRetryPolicy.onBeforeSendRequest(dsr);
313266
shouldRetry = webExceptionRetryPolicy.shouldRetry(cosmosException);
267+
314268
validateSuccess(shouldRetry, ShouldRetryValidator.builder()
315269
.nullException()
316270
.shouldRetry(false)

0 commit comments

Comments
 (0)