|
7 | 7 | from pyathena import DataError |
8 | 8 | from pyathena.util import ( |
9 | 9 | RetryConfig, |
| 10 | + _is_throttling_error, |
| 11 | + _retry_api_call, |
| 12 | + _without_retries, |
10 | 13 | is_retryable_error, |
11 | 14 | parse_output_location, |
12 | 15 | retry_api_call, |
@@ -257,3 +260,85 @@ def test_is_retryable_error(code, message, expected): |
257 | 260 | error = ClientError({"Error": {"Code": code, "Message": message}}, "GetTableMetadata") |
258 | 261 | assert is_retryable_error(error, RetryConfig()) is expected |
259 | 262 | assert is_retryable_error(ValueError("no response"), RetryConfig()) is False |
| 263 | + |
| 264 | + |
| 265 | +@pytest.mark.parametrize( |
| 266 | + ("code", "message", "expected"), |
| 267 | + [ |
| 268 | + ("ThrottlingException", "Rate exceeded", True), |
| 269 | + ("TooManyRequestsException", "Too many requests", True), |
| 270 | + # Glue's own throttling, reported by Athena inside a MetadataException. |
| 271 | + ( |
| 272 | + "MetadataException", |
| 273 | + "Rate exceeded (Service: AmazonDataCatalog; Status Code: 400; " |
| 274 | + "Error Code: ThrottlingException; Request ID: example; Proxy: null)", |
| 275 | + True, |
| 276 | + ), |
| 277 | + ( |
| 278 | + "MetadataException", |
| 279 | + "Not authorized (Service: AmazonDataCatalog; Status Code: 400; " |
| 280 | + "Error Code: AccessDeniedException; Request ID: example; Proxy: null)", |
| 281 | + False, |
| 282 | + ), |
| 283 | + ("MetadataException", "Table ThrottlingException not found", False), |
| 284 | + ("InternalServerException", "Internal error", False), |
| 285 | + ], |
| 286 | +) |
| 287 | +def test_is_throttling_error(code, message, expected): |
| 288 | + error = ClientError({"Error": {"Code": code, "Message": message}}, "GetTableMetadata") |
| 289 | + assert _is_throttling_error(error) is expected |
| 290 | + assert _is_throttling_error(ValueError("no response")) is False |
| 291 | + |
| 292 | + |
| 293 | +def test_without_retries(): |
| 294 | + config = RetryConfig( |
| 295 | + exceptions=("ThrottlingException", "MetadataException", "InternalServerException"), |
| 296 | + attempt=4, |
| 297 | + multiplier=2, |
| 298 | + max_delay=30, |
| 299 | + exponential_base=3, |
| 300 | + ) |
| 301 | + |
| 302 | + derived = _without_retries(config, ["ThrottlingException", "MetadataException"]) |
| 303 | + |
| 304 | + assert derived.exceptions == ("InternalServerException",) |
| 305 | + assert (derived.attempt, derived.multiplier, derived.max_delay) == (4, 2, 30) |
| 306 | + assert derived.exponential_base == 3 |
| 307 | + # The original policy is left as it was. |
| 308 | + assert config.exceptions == ( |
| 309 | + "ThrottlingException", |
| 310 | + "MetadataException", |
| 311 | + "InternalServerException", |
| 312 | + ) |
| 313 | + |
| 314 | + |
| 315 | +@pytest.mark.parametrize( |
| 316 | + ("code", "expected_calls"), |
| 317 | + [ |
| 318 | + # Stopped at once, although the policy retries it. |
| 319 | + ("ThrottlingException", 1), |
| 320 | + # Other retryable codes keep the policy's attempts. |
| 321 | + ("InternalServerException", 3), |
| 322 | + ], |
| 323 | +) |
| 324 | +def test_retry_api_call_stops_on_predicate(code, expected_calls): |
| 325 | + error = ClientError({"Error": {"Code": code, "Message": ""}}, "GetTableMetadata") |
| 326 | + calls = 0 |
| 327 | + |
| 328 | + def call(): |
| 329 | + nonlocal calls |
| 330 | + calls += 1 |
| 331 | + raise error |
| 332 | + |
| 333 | + config = RetryConfig( |
| 334 | + exceptions=("ThrottlingException", "InternalServerException"), |
| 335 | + attempt=3, |
| 336 | + multiplier=0, |
| 337 | + max_delay=0, |
| 338 | + ) |
| 339 | + |
| 340 | + with pytest.raises(ClientError) as caught: |
| 341 | + _retry_api_call(call, config, None, _is_throttling_error) |
| 342 | + |
| 343 | + assert caught.value is error |
| 344 | + assert calls == expected_calls |
0 commit comments