From 9f05e09e62a9b7749e0d4c54104189c7b0a6028f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20De=20Ara=C3=BAjo?= Date: Mon, 20 Oct 2025 14:14:56 +0000 Subject: [PATCH 01/10] chore(ci_visibility): retry Test Optimization API calls on server errors --- ddtrace/internal/ci_visibility/_api_client.py | 22 ++++++++++++++++++- .../ci_visibility/telemetry/api_request.py | 4 ++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/ddtrace/internal/ci_visibility/_api_client.py b/ddtrace/internal/ci_visibility/_api_client.py index e2a010dc676..e34a4b9491f 100644 --- a/ddtrace/internal/ci_visibility/_api_client.py +++ b/ddtrace/internal/ci_visibility/_api_client.py @@ -51,6 +51,7 @@ from ddtrace.internal.utils.http import Response from ddtrace.internal.utils.http import get_connection from ddtrace.internal.utils.http import verify_url +from ddtrace.internal.utils.retry import fibonacci_backoff_with_jitter from ddtrace.internal.utils.time import StopWatch @@ -81,6 +82,22 @@ class TestVisibilitySkippableItemsError(Exception): pass +class CIVisibilityAPIError(Exception): + def __init__(self, status: int) -> None: + self.status = status + + +class CIVisibilityAPIClientError(CIVisibilityAPIError): + pass + + +class CIVisibilityAPIServerError(CIVisibilityAPIError): + pass + + +_RETRIABLE_ERRORS = (*_NETWORK_ERRORS, CIVisibilityAPIServerError) + + @dataclasses.dataclass(frozen=True) class EarlyFlakeDetectionSettings: enabled: bool = False @@ -300,6 +317,7 @@ def _do_request(self, method: str, endpoint: str, payload: str, timeout: t.Optio if conn is not None: conn.close() + @fibonacci_backoff_with_jitter(attempts=5, until=lambda e: not isinstance(e, _RETRIABLE_ERRORS)) def _do_request_with_telemetry( self, method: str, @@ -342,7 +360,9 @@ def _do_request_with_telemetry( error_type = ERROR_TYPES.CODE_4XX if response.status < 500 else ERROR_TYPES.CODE_5XX if response.status == 403: raise CIVisibilityAuthenticationException() - raise ValueError("API response status code: %d", response.status) + if response.status >= 500: + raise CIVisibilityAPIServerError(response.status) + raise CIVisibilityAPIClientError(response.status) try: sw.stop() # Stop the timer before parsing the response response_body = response.body diff --git a/ddtrace/internal/ci_visibility/telemetry/api_request.py b/ddtrace/internal/ci_visibility/telemetry/api_request.py index 77f3ea5f626..cb1caab20d2 100644 --- a/ddtrace/internal/ci_visibility/telemetry/api_request.py +++ b/ddtrace/internal/ci_visibility/telemetry/api_request.py @@ -25,7 +25,7 @@ def record_api_request( error: Optional[ERROR_TYPES] = None, ): log.debug( - "Recording early flake detection telemetry for %s: %s, %s, %s", + "Recording Test Optimization telemetry for %s: %s, %s, %s", metric_names.count, duration, response_bytes, @@ -47,5 +47,5 @@ def record_api_request( def record_api_request_error(error_metric_name: str, error: ERROR_TYPES): - log.debug("Recording early flake detection request error telemetry: %s", error) + log.debug("Recording Test Optimization request error telemetry: %s", error) telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE.CIVISIBILITY, error_metric_name, 1, (("error_type", error),)) From 4c6c872093634195a4a594eadc4b1d19dd95ec29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20De=20Ara=C3=BAjo?= Date: Mon, 20 Oct 2025 14:47:55 +0000 Subject: [PATCH 02/10] add tests --- ddtrace/internal/ci_visibility/_api_client.py | 3 + .../test_ci_visibility_api_client.py | 63 +++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/ddtrace/internal/ci_visibility/_api_client.py b/ddtrace/internal/ci_visibility/_api_client.py index e34a4b9491f..a27fecfa77f 100644 --- a/ddtrace/internal/ci_visibility/_api_client.py +++ b/ddtrace/internal/ci_visibility/_api_client.py @@ -86,6 +86,9 @@ class CIVisibilityAPIError(Exception): def __init__(self, status: int) -> None: self.status = status + def __str__(self) -> str: + return f"Error calling Test Optimization API (status: {self.status})" + class CIVisibilityAPIClientError(CIVisibilityAPIError): pass diff --git a/tests/ci_visibility/api_client/test_ci_visibility_api_client.py b/tests/ci_visibility/api_client/test_ci_visibility_api_client.py index ca085a85ed4..5ce4ba973e3 100644 --- a/tests/ci_visibility/api_client/test_ci_visibility_api_client.py +++ b/tests/ci_visibility/api_client/test_ci_visibility_api_client.py @@ -8,6 +8,7 @@ from ddtrace.ext.test_visibility import _get_default_test_visibility_contrib_config from ddtrace.internal.ci_visibility import CIVisibility from ddtrace.internal.ci_visibility._api_client import AgentlessTestVisibilityAPIClient +from ddtrace.internal.ci_visibility._api_client import CIVisibilityAPIServerError from ddtrace.internal.ci_visibility._api_client import EVPProxyTestVisibilityAPIClient from ddtrace.internal.ci_visibility._api_client import ITRData from ddtrace.internal.ci_visibility._api_client import TestVisibilityAPISettings @@ -222,6 +223,68 @@ def test_civisibility_api_client_settings_do_request_call_optionals( itr_skipping_level, git_data=git_data, dd_service=dd_service, dd_env=dd_env ) + def test_civisibility_api_client_settings_retry_on_errors(self): + """Tests that the API call to the settings endpoint is retried in case of server errors.""" + client = self._get_test_client( + itr_skipping_level=ITR_SKIPPING_LEVEL.TEST, + api_key="my_api_key", + dd_service=None, + dd_env=None, + git_data=self.git_data_parameters[0], + ) + with mock.patch.object( + client, + "_do_request", + side_effect=[ + _get_setting_api_response(status_code=500), + _get_setting_api_response(status_code=500), + _get_setting_api_response(), + ], + ) as mock_do_request: + with mock.patch("ddtrace.internal.utils.retry.sleep"): + settings = client.fetch_settings(read_from_cache=False) + + assert settings == TestVisibilityAPISettings() + + assert mock_do_request.call_count == 3 + for call_args, _ in mock_do_request.call_args_list: + assert call_args[0] == "POST" + assert json.loads(call_args[2]) == self._get_expected_do_request_setting_payload( + ITR_SKIPPING_LEVEL.TEST, git_data=self.git_data_parameters[0], dd_service=None, dd_env=None + ) + + def test_civisibility_api_client_settings_fail_after_5_retries(self): + """Tests that the API call to the settings endpoint is retried in case of server errors.""" + client = self._get_test_client( + itr_skipping_level=ITR_SKIPPING_LEVEL.TEST, + api_key="my_api_key", + dd_service=None, + dd_env=None, + git_data=self.git_data_parameters[0], + ) + with mock.patch.object( + client, + "_do_request", + side_effect=[ + _get_setting_api_response(status_code=500), + _get_setting_api_response(status_code=500), + _get_setting_api_response(status_code=500), + _get_setting_api_response(status_code=500), + _get_setting_api_response(status_code=500), + _get_setting_api_response(), + ], + ) as mock_do_request: + with mock.patch("ddtrace.internal.utils.retry.sleep"): + with pytest.raises(CIVisibilityAPIServerError): + _ = client.fetch_settings(read_from_cache=False) + + assert mock_do_request.call_count == 5 + for call_args, _ in mock_do_request.call_args_list: + assert call_args[0] == "POST" + assert json.loads(call_args[2]) == self._get_expected_do_request_setting_payload( + ITR_SKIPPING_LEVEL.TEST, git_data=self.git_data_parameters[0], dd_service=None, dd_env=None + ) + @pytest.mark.parametrize("client_timeout", [None, 5]) @pytest.mark.parametrize("request_timeout", [None, 10]) @pytest.mark.parametrize( From fbf4fae096f54d839d47e3f70018e2c1e8999a62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20De=20Ara=C3=BAjo?= Date: Wed, 22 Oct 2025 10:04:28 +0000 Subject: [PATCH 03/10] test with more error types --- .../api_client/test_ci_visibility_api_client.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/ci_visibility/api_client/test_ci_visibility_api_client.py b/tests/ci_visibility/api_client/test_ci_visibility_api_client.py index 5ce4ba973e3..7ac44254cfa 100644 --- a/tests/ci_visibility/api_client/test_ci_visibility_api_client.py +++ b/tests/ci_visibility/api_client/test_ci_visibility_api_client.py @@ -1,5 +1,7 @@ from contextlib import contextmanager +from http.client import RemoteDisconnected import json +import socket from unittest import mock import pytest @@ -237,7 +239,7 @@ def test_civisibility_api_client_settings_retry_on_errors(self): "_do_request", side_effect=[ _get_setting_api_response(status_code=500), - _get_setting_api_response(status_code=500), + TimeoutError(), _get_setting_api_response(), ], ) as mock_do_request: @@ -267,10 +269,10 @@ def test_civisibility_api_client_settings_fail_after_5_retries(self): "_do_request", side_effect=[ _get_setting_api_response(status_code=500), - _get_setting_api_response(status_code=500), - _get_setting_api_response(status_code=500), - _get_setting_api_response(status_code=500), - _get_setting_api_response(status_code=500), + _get_setting_api_response(status_code=504), + TimeoutError(), + RemoteDisconnected(), + socket.timeout(), _get_setting_api_response(), ], ) as mock_do_request: From 2c18cccb63ac5078274007122cf8271b8f26395f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20De=20Ara=C3=BAjo?= Date: Wed, 22 Oct 2025 12:55:28 +0000 Subject: [PATCH 04/10] fix some tests --- .../test_ci_visibility_api_client_setting_responses.py | 4 ++-- .../test_ci_visibility_api_client_skippable_responses.py | 4 +++- ...test_ci_visibility_api_client_test_management_responses.py | 4 +++- .../test_ci_visibility_api_client_unique_tests_responses.py | 4 +++- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/ci_visibility/api_client/test_ci_visibility_api_client_setting_responses.py b/tests/ci_visibility/api_client/test_ci_visibility_api_client_setting_responses.py index d42e143bf20..e8e0cf06711 100644 --- a/tests/ci_visibility/api_client/test_ci_visibility_api_client_setting_responses.py +++ b/tests/ci_visibility/api_client/test_ci_visibility_api_client_setting_responses.py @@ -138,8 +138,8 @@ def test_civisibility_api_client_setting_parsed(self, setting_response, expected def test_civisibility_api_client_setting_errors(self, do_request_side_effect, expected_exception): """Tests that the client reports errors correctly based on the API response""" client = self._get_test_client() - with mock.patch.object(client, "_do_request", side_effect=[do_request_side_effect]), pytest.raises( + with mock.patch.object(client, "_do_request", side_effect=do_request_side_effect), pytest.raises( expected_exception - ): + ), mock.patch("ddtrace.internal.utils.retry.sleep"): settings = client.fetch_settings(read_from_cache=False) assert settings is None diff --git a/tests/ci_visibility/api_client/test_ci_visibility_api_client_skippable_responses.py b/tests/ci_visibility/api_client/test_ci_visibility_api_client_skippable_responses.py index 5ebe8667319..6b78d986c24 100644 --- a/tests/ci_visibility/api_client/test_ci_visibility_api_client_skippable_responses.py +++ b/tests/ci_visibility/api_client/test_ci_visibility_api_client_skippable_responses.py @@ -577,6 +577,8 @@ def test_civisibility_api_client_skippable_parsed_covered_files(self, skippable_ def test_civisibility_api_client_skippable_errors(self, do_request_side_effect): """Tests that the client reports errors correctly gives a None item without crashing""" client = self._get_test_client() - with mock.patch.object(client, "_do_request", side_effect=[do_request_side_effect]): + with mock.patch.object(client, "_do_request", side_effect=do_request_side_effect), mock.patch( + "ddtrace.internal.utils.retry.sleep" + ): skippable_items = client.fetch_skippable_items(read_from_cache=False) assert skippable_items is None diff --git a/tests/ci_visibility/api_client/test_ci_visibility_api_client_test_management_responses.py b/tests/ci_visibility/api_client/test_ci_visibility_api_client_test_management_responses.py index 3d95f0cafe4..3b81192e09e 100644 --- a/tests/ci_visibility/api_client/test_ci_visibility_api_client_test_management_responses.py +++ b/tests/ci_visibility/api_client/test_ci_visibility_api_client_test_management_responses.py @@ -119,6 +119,8 @@ def test_api_client_test_management_tests_parsed(self): def test_api_client_test_management_tests_errors(self, do_request_side_effect): """Tests that the client correctly handles errors in the Test Management test API response""" client = self._get_test_client() - with mock.patch.object(client, "_do_request", side_effect=[do_request_side_effect]): + with mock.patch.object(client, "_do_request", side_effect=do_request_side_effect), mock.patch( + "ddtrace.internal.utils.retry.sleep" + ): settings = client.fetch_test_management_tests(read_from_cache=False) assert settings is None diff --git a/tests/ci_visibility/api_client/test_ci_visibility_api_client_unique_tests_responses.py b/tests/ci_visibility/api_client/test_ci_visibility_api_client_unique_tests_responses.py index 04b260947ff..48ce759775b 100644 --- a/tests/ci_visibility/api_client/test_ci_visibility_api_client_unique_tests_responses.py +++ b/tests/ci_visibility/api_client/test_ci_visibility_api_client_unique_tests_responses.py @@ -102,6 +102,8 @@ def test_civisibility_api_client_known_tests_parsed(self, known_test_response, e def test_civisibility_api_client_known_tests_errors(self, do_request_side_effect): """Tests that the client correctly handles errors in the known test API response""" client = self._get_test_client() - with mock.patch.object(client, "_do_request", side_effect=[do_request_side_effect]): + with mock.patch.object(client, "_do_request", side_effect=do_request_side_effect), mock.patch( + "ddtrace.internal.utils.retry.sleep" + ): settings = client.fetch_known_tests(read_from_cache=False) assert settings is None From 50d55924e993fd52d7471d79fc6b4579fc1dc403 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20De=20Ara=C3=BAjo?= Date: Wed, 22 Oct 2025 12:55:59 +0000 Subject: [PATCH 05/10] itr:noskip From 0e467e65ce5b55517bc40852de4c0c4bc8096399 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20De=20Ara=C3=BAjo?= Date: Wed, 22 Oct 2025 12:56:25 +0000 Subject: [PATCH 06/10] itr:noskip From bef9f69dc267743725d21298bd8b71cfaeba35bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20De=20Ara=C3=BAjo?= Date: Tue, 28 Oct 2025 14:50:06 +0000 Subject: [PATCH 07/10] Add own decorator for retrying exceptions --- ddtrace/internal/ci_visibility/_api_client.py | 32 +++---------------- ddtrace/internal/ci_visibility/errors.py | 16 ++++++++++ ddtrace/internal/utils/retry.py | 32 +++++++++++++++++++ .../test_ci_visibility_api_client.py | 3 +- ...visibility_api_client_setting_responses.py | 5 +-- ...sibility_api_client_skippable_responses.py | 2 +- ...ty_api_client_test_management_responses.py | 2 +- ...ility_api_client_unique_tests_responses.py | 2 +- 8 files changed, 59 insertions(+), 35 deletions(-) diff --git a/ddtrace/internal/ci_visibility/_api_client.py b/ddtrace/internal/ci_visibility/_api_client.py index a27fecfa77f..5c326271080 100644 --- a/ddtrace/internal/ci_visibility/_api_client.py +++ b/ddtrace/internal/ci_visibility/_api_client.py @@ -29,6 +29,8 @@ from ddtrace.internal.ci_visibility.constants import SUITE from ddtrace.internal.ci_visibility.constants import TEST from ddtrace.internal.ci_visibility.constants import TEST_MANAGEMENT_TESTS_ENDPOINT +from ddtrace.internal.ci_visibility.errors import CIVisibilityAPIClientError +from ddtrace.internal.ci_visibility.errors import CIVisibilityAPIServerError from ddtrace.internal.ci_visibility.errors import CIVisibilityAuthenticationException from ddtrace.internal.ci_visibility.git_data import GitData from ddtrace.internal.ci_visibility.telemetry.api_request import APIRequestMetricNames @@ -51,7 +53,7 @@ from ddtrace.internal.utils.http import Response from ddtrace.internal.utils.http import get_connection from ddtrace.internal.utils.http import verify_url -from ddtrace.internal.utils.retry import fibonacci_backoff_with_jitter +from ddtrace.internal.utils.retry import fibonacci_backoff_with_jitter_on_exceptions from ddtrace.internal.utils.time import StopWatch @@ -72,32 +74,6 @@ _NETWORK_ERRORS = (TimeoutError, socket.timeout, RemoteDisconnected) -class TestVisibilitySettingsError(Exception): - __test__ = False - pass - - -class TestVisibilitySkippableItemsError(Exception): - __test__ = False - pass - - -class CIVisibilityAPIError(Exception): - def __init__(self, status: int) -> None: - self.status = status - - def __str__(self) -> str: - return f"Error calling Test Optimization API (status: {self.status})" - - -class CIVisibilityAPIClientError(CIVisibilityAPIError): - pass - - -class CIVisibilityAPIServerError(CIVisibilityAPIError): - pass - - _RETRIABLE_ERRORS = (*_NETWORK_ERRORS, CIVisibilityAPIServerError) @@ -320,7 +296,7 @@ def _do_request(self, method: str, endpoint: str, payload: str, timeout: t.Optio if conn is not None: conn.close() - @fibonacci_backoff_with_jitter(attempts=5, until=lambda e: not isinstance(e, _RETRIABLE_ERRORS)) + @fibonacci_backoff_with_jitter_on_exceptions(attempts=5, exceptions=_RETRIABLE_ERRORS) def _do_request_with_telemetry( self, method: str, diff --git a/ddtrace/internal/ci_visibility/errors.py b/ddtrace/internal/ci_visibility/errors.py index d3b5ed71a58..7cda174d41a 100644 --- a/ddtrace/internal/ci_visibility/errors.py +++ b/ddtrace/internal/ci_visibility/errors.py @@ -25,3 +25,19 @@ class CIVisibilityProcessError(CIVisibilityError): class CIVisibilityAuthenticationException(Exception): pass + + +class CIVisibilityAPIError(Exception): + def __init__(self, status: int) -> None: + self.status = status + + def __str__(self) -> str: + return f"Error calling Test Optimization API (status: {self.status})" + + +class CIVisibilityAPIClientError(CIVisibilityAPIError): + pass + + +class CIVisibilityAPIServerError(CIVisibilityAPIError): + pass diff --git a/ddtrace/internal/utils/retry.py b/ddtrace/internal/utils/retry.py index 00db74ac884..b2675f604de 100644 --- a/ddtrace/internal/utils/retry.py +++ b/ddtrace/internal/utils/retry.py @@ -55,9 +55,41 @@ def retry_wrapped(*args, **kwargs): return retry_decorator +def retry_on_exceptions( + after: t.Iterable[float], + exceptions: t.Tuple[t.Type[BaseException], ...], +) -> t.Callable: + def retry_decorator(f): + @wraps(f) + def retry_wrapped(*args, **kwargs): + for delay in after: + try: + return f(*args, **kwargs) + except Exception as e: + if not isinstance(e, exceptions): + raise # Not a retriable exception, don't keep retrying + sleep(delay) + + # Last chance to succeed. If it fails, we don't catch the exception. + return f(*args, **kwargs) + + return retry_wrapped + + return retry_decorator + + def fibonacci_backoff_with_jitter(attempts, initial_wait=1.0, until=lambda result: result is None): # type: (int, float, t.Callable[[t.Any], bool]) -> t.Callable return retry( after=[random.uniform(0, initial_wait * (1.618**i)) for i in range(attempts - 1)], # nosec until=until, ) + + +def fibonacci_backoff_with_jitter_on_exceptions( + attempts: int, exceptions: t.Tuple[t.Type[BaseException], ...] +) -> t.Callable: + return retry_on_exceptions( + after=[random.uniform(0, 1.618**i) for i in range(attempts - 1)], + exceptions=exceptions, + ) diff --git a/tests/ci_visibility/api_client/test_ci_visibility_api_client.py b/tests/ci_visibility/api_client/test_ci_visibility_api_client.py index 7ac44254cfa..e054e2c68e7 100644 --- a/tests/ci_visibility/api_client/test_ci_visibility_api_client.py +++ b/tests/ci_visibility/api_client/test_ci_visibility_api_client.py @@ -10,7 +10,6 @@ from ddtrace.ext.test_visibility import _get_default_test_visibility_contrib_config from ddtrace.internal.ci_visibility import CIVisibility from ddtrace.internal.ci_visibility._api_client import AgentlessTestVisibilityAPIClient -from ddtrace.internal.ci_visibility._api_client import CIVisibilityAPIServerError from ddtrace.internal.ci_visibility._api_client import EVPProxyTestVisibilityAPIClient from ddtrace.internal.ci_visibility._api_client import ITRData from ddtrace.internal.ci_visibility._api_client import TestVisibilityAPISettings @@ -277,7 +276,7 @@ def test_civisibility_api_client_settings_fail_after_5_retries(self): ], ) as mock_do_request: with mock.patch("ddtrace.internal.utils.retry.sleep"): - with pytest.raises(CIVisibilityAPIServerError): + with pytest.raises(socket.timeout): # raises the last exception _ = client.fetch_settings(read_from_cache=False) assert mock_do_request.call_count == 5 diff --git a/tests/ci_visibility/api_client/test_ci_visibility_api_client_setting_responses.py b/tests/ci_visibility/api_client/test_ci_visibility_api_client_setting_responses.py index e8e0cf06711..544c9ff3a1a 100644 --- a/tests/ci_visibility/api_client/test_ci_visibility_api_client_setting_responses.py +++ b/tests/ci_visibility/api_client/test_ci_visibility_api_client_setting_responses.py @@ -9,6 +9,7 @@ from ddtrace.internal.ci_visibility._api_client import EarlyFlakeDetectionSettings from ddtrace.internal.ci_visibility._api_client import TestVisibilityAPISettings +from ddtrace.internal.ci_visibility.errors import CIVisibilityAPIServerError from ddtrace.internal.ci_visibility.errors import CIVisibilityAuthenticationException from ddtrace.internal.utils.http import Response from tests.ci_visibility.api_client._util import TestTestVisibilityAPIClientBase @@ -129,7 +130,7 @@ def test_civisibility_api_client_setting_parsed(self, setting_response, expected [socket.timeout, socket.timeout], [RemoteDisconnected, RemoteDisconnected], [Response(403), CIVisibilityAuthenticationException], - [Response(500), ValueError], + [Response(500), CIVisibilityAPIServerError], [Response(200, "this is not json"), JSONDecodeError], [Response(200, '{"valid_json": "invalid_structure"}'), KeyError], [Response(200, '{"errors": "there was an error"}'), ValueError], @@ -138,7 +139,7 @@ def test_civisibility_api_client_setting_parsed(self, setting_response, expected def test_civisibility_api_client_setting_errors(self, do_request_side_effect, expected_exception): """Tests that the client reports errors correctly based on the API response""" client = self._get_test_client() - with mock.patch.object(client, "_do_request", side_effect=do_request_side_effect), pytest.raises( + with mock.patch.object(client, "_do_request", side_effect=[do_request_side_effect] * 5), pytest.raises( expected_exception ), mock.patch("ddtrace.internal.utils.retry.sleep"): settings = client.fetch_settings(read_from_cache=False) diff --git a/tests/ci_visibility/api_client/test_ci_visibility_api_client_skippable_responses.py b/tests/ci_visibility/api_client/test_ci_visibility_api_client_skippable_responses.py index 6b78d986c24..da3130f54d7 100644 --- a/tests/ci_visibility/api_client/test_ci_visibility_api_client_skippable_responses.py +++ b/tests/ci_visibility/api_client/test_ci_visibility_api_client_skippable_responses.py @@ -577,7 +577,7 @@ def test_civisibility_api_client_skippable_parsed_covered_files(self, skippable_ def test_civisibility_api_client_skippable_errors(self, do_request_side_effect): """Tests that the client reports errors correctly gives a None item without crashing""" client = self._get_test_client() - with mock.patch.object(client, "_do_request", side_effect=do_request_side_effect), mock.patch( + with mock.patch.object(client, "_do_request", side_effect=[do_request_side_effect] * 5), mock.patch( "ddtrace.internal.utils.retry.sleep" ): skippable_items = client.fetch_skippable_items(read_from_cache=False) diff --git a/tests/ci_visibility/api_client/test_ci_visibility_api_client_test_management_responses.py b/tests/ci_visibility/api_client/test_ci_visibility_api_client_test_management_responses.py index 3b81192e09e..35911a5d951 100644 --- a/tests/ci_visibility/api_client/test_ci_visibility_api_client_test_management_responses.py +++ b/tests/ci_visibility/api_client/test_ci_visibility_api_client_test_management_responses.py @@ -119,7 +119,7 @@ def test_api_client_test_management_tests_parsed(self): def test_api_client_test_management_tests_errors(self, do_request_side_effect): """Tests that the client correctly handles errors in the Test Management test API response""" client = self._get_test_client() - with mock.patch.object(client, "_do_request", side_effect=do_request_side_effect), mock.patch( + with mock.patch.object(client, "_do_request", side_effect=[do_request_side_effect] * 5), mock.patch( "ddtrace.internal.utils.retry.sleep" ): settings = client.fetch_test_management_tests(read_from_cache=False) diff --git a/tests/ci_visibility/api_client/test_ci_visibility_api_client_unique_tests_responses.py b/tests/ci_visibility/api_client/test_ci_visibility_api_client_unique_tests_responses.py index 48ce759775b..bba65c12d4c 100644 --- a/tests/ci_visibility/api_client/test_ci_visibility_api_client_unique_tests_responses.py +++ b/tests/ci_visibility/api_client/test_ci_visibility_api_client_unique_tests_responses.py @@ -102,7 +102,7 @@ def test_civisibility_api_client_known_tests_parsed(self, known_test_response, e def test_civisibility_api_client_known_tests_errors(self, do_request_side_effect): """Tests that the client correctly handles errors in the known test API response""" client = self._get_test_client() - with mock.patch.object(client, "_do_request", side_effect=do_request_side_effect), mock.patch( + with mock.patch.object(client, "_do_request", side_effect=[do_request_side_effect] * 5), mock.patch( "ddtrace.internal.utils.retry.sleep" ): settings = client.fetch_known_tests(read_from_cache=False) From bdea43f7700b67e04a107f3b5bac8d2bc7352a94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20De=20Ara=C3=BAjo?= Date: Tue, 28 Oct 2025 14:55:38 +0000 Subject: [PATCH 08/10] . --- ddtrace/internal/utils/retry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/internal/utils/retry.py b/ddtrace/internal/utils/retry.py index b2675f604de..2081ab5f3de 100644 --- a/ddtrace/internal/utils/retry.py +++ b/ddtrace/internal/utils/retry.py @@ -67,7 +67,7 @@ def retry_wrapped(*args, **kwargs): return f(*args, **kwargs) except Exception as e: if not isinstance(e, exceptions): - raise # Not a retriable exception, don't keep retrying + raise # Not a retriable exception, don't keep retrying. sleep(delay) # Last chance to succeed. If it fails, we don't catch the exception. From 3b4efa925acbbfb9ee682b559ea483013f8ce093 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20De=20Ara=C3=BAjo?= Date: Wed, 29 Oct 2025 09:04:17 +0000 Subject: [PATCH 09/10] randomness --- ddtrace/internal/utils/retry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/internal/utils/retry.py b/ddtrace/internal/utils/retry.py index 2081ab5f3de..82a64ee8ad2 100644 --- a/ddtrace/internal/utils/retry.py +++ b/ddtrace/internal/utils/retry.py @@ -90,6 +90,6 @@ def fibonacci_backoff_with_jitter_on_exceptions( attempts: int, exceptions: t.Tuple[t.Type[BaseException], ...] ) -> t.Callable: return retry_on_exceptions( - after=[random.uniform(0, 1.618**i) for i in range(attempts - 1)], + after=[random.uniform(0, 1.618**i) for i in range(attempts - 1)], # nosec exceptions=exceptions, ) From f0eb381c8634b117072f7663151dcb4c3278e58a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20De=20Ara=C3=BAjo?= Date: Wed, 29 Oct 2025 09:49:15 +0000 Subject: [PATCH 10/10] move function to ci_visibility namespace --- ddtrace/internal/ci_visibility/_api_client.py | 2 +- ddtrace/internal/ci_visibility/utils.py | 48 +++++++++++++++++-- ddtrace/internal/utils/retry.py | 32 ------------- .../test_ci_visibility_api_client.py | 4 +- ...visibility_api_client_setting_responses.py | 2 +- 5 files changed, 49 insertions(+), 39 deletions(-) diff --git a/ddtrace/internal/ci_visibility/_api_client.py b/ddtrace/internal/ci_visibility/_api_client.py index 5c326271080..2c5a9ae7738 100644 --- a/ddtrace/internal/ci_visibility/_api_client.py +++ b/ddtrace/internal/ci_visibility/_api_client.py @@ -46,6 +46,7 @@ from ddtrace.internal.ci_visibility.telemetry.test_management import TEST_MANAGEMENT_TELEMETRY from ddtrace.internal.ci_visibility.telemetry.test_management import record_test_management_tests_count from ddtrace.internal.ci_visibility.utils import combine_url_path +from ddtrace.internal.ci_visibility.utils import fibonacci_backoff_with_jitter_on_exceptions from ddtrace.internal.logger import get_logger from ddtrace.internal.test_visibility.coverage_lines import CoverageLines from ddtrace.internal.utils.formats import asbool @@ -53,7 +54,6 @@ from ddtrace.internal.utils.http import Response from ddtrace.internal.utils.http import get_connection from ddtrace.internal.utils.http import verify_url -from ddtrace.internal.utils.retry import fibonacci_backoff_with_jitter_on_exceptions from ddtrace.internal.utils.time import StopWatch diff --git a/ddtrace/internal/ci_visibility/utils.py b/ddtrace/internal/ci_visibility/utils.py index 22b748693a8..68aae145a34 100644 --- a/ddtrace/internal/ci_visibility/utils.py +++ b/ddtrace/internal/ci_visibility/utils.py @@ -1,8 +1,11 @@ +from functools import wraps import inspect import logging import os +import random import re -import typing +from time import sleep +import typing as t import ddtrace from ddtrace import config as ddconfig @@ -28,7 +31,7 @@ def get_relative_or_absolute_path_for_path(path: str, start_directory: str): return relative_path -def get_source_file_path_for_test_method(test_method_object, repo_directory: str) -> typing.Union[str, None]: +def get_source_file_path_for_test_method(test_method_object, repo_directory: str) -> t.Union[str, None]: try: file_object = inspect.getfile(test_method_object) except TypeError: @@ -39,7 +42,7 @@ def get_source_file_path_for_test_method(test_method_object, repo_directory: str def get_source_lines_for_test_method( test_method_object, -) -> typing.Union[typing.Tuple[int, int], typing.Tuple[None, None]]: +) -> t.Union[t.Tuple[int, int], t.Tuple[None, None]]: try: source_lines_tuple = inspect.getsourcelines(test_method_object) except (TypeError, OSError): @@ -156,3 +159,42 @@ def _get_test_framework_telemetry_name(test_framework: str) -> TEST_FRAMEWORKS: if framework.value == test_framework: return framework return TEST_FRAMEWORKS.MANUAL + + +def retry_on_exceptions( + after: t.Iterable[float], + exceptions: t.Tuple[t.Type[BaseException], ...], +) -> t.Callable: + """ + Decorator to automatically retry a function if it raises specified exceptions. + """ + + def retry_decorator(f): + @wraps(f) + def retry_wrapped(*args, **kwargs): + for delay in after: + try: + return f(*args, **kwargs) + except Exception as e: + if not isinstance(e, exceptions): + raise # Not a retriable exception, don't keep retrying. + sleep(delay) + + # Last chance to succeed. If it fails, we don't catch the exception. + return f(*args, **kwargs) + + return retry_wrapped + + return retry_decorator + + +def fibonacci_backoff_with_jitter_on_exceptions( + attempts: int, exceptions: t.Tuple[t.Type[BaseException], ...] +) -> t.Callable: + """ + Decorator to automatically retry a function if it raises specified exceptions, with exponential backoff delays. + """ + return retry_on_exceptions( + after=[random.uniform(0, 1.618**i) for i in range(attempts - 1)], # nosec + exceptions=exceptions, + ) diff --git a/ddtrace/internal/utils/retry.py b/ddtrace/internal/utils/retry.py index 82a64ee8ad2..00db74ac884 100644 --- a/ddtrace/internal/utils/retry.py +++ b/ddtrace/internal/utils/retry.py @@ -55,41 +55,9 @@ def retry_wrapped(*args, **kwargs): return retry_decorator -def retry_on_exceptions( - after: t.Iterable[float], - exceptions: t.Tuple[t.Type[BaseException], ...], -) -> t.Callable: - def retry_decorator(f): - @wraps(f) - def retry_wrapped(*args, **kwargs): - for delay in after: - try: - return f(*args, **kwargs) - except Exception as e: - if not isinstance(e, exceptions): - raise # Not a retriable exception, don't keep retrying. - sleep(delay) - - # Last chance to succeed. If it fails, we don't catch the exception. - return f(*args, **kwargs) - - return retry_wrapped - - return retry_decorator - - def fibonacci_backoff_with_jitter(attempts, initial_wait=1.0, until=lambda result: result is None): # type: (int, float, t.Callable[[t.Any], bool]) -> t.Callable return retry( after=[random.uniform(0, initial_wait * (1.618**i)) for i in range(attempts - 1)], # nosec until=until, ) - - -def fibonacci_backoff_with_jitter_on_exceptions( - attempts: int, exceptions: t.Tuple[t.Type[BaseException], ...] -) -> t.Callable: - return retry_on_exceptions( - after=[random.uniform(0, 1.618**i) for i in range(attempts - 1)], # nosec - exceptions=exceptions, - ) diff --git a/tests/ci_visibility/api_client/test_ci_visibility_api_client.py b/tests/ci_visibility/api_client/test_ci_visibility_api_client.py index e054e2c68e7..bfb9bbb6782 100644 --- a/tests/ci_visibility/api_client/test_ci_visibility_api_client.py +++ b/tests/ci_visibility/api_client/test_ci_visibility_api_client.py @@ -242,7 +242,7 @@ def test_civisibility_api_client_settings_retry_on_errors(self): _get_setting_api_response(), ], ) as mock_do_request: - with mock.patch("ddtrace.internal.utils.retry.sleep"): + with mock.patch("ddtrace.internal.ci_visibility.utils.sleep"): settings = client.fetch_settings(read_from_cache=False) assert settings == TestVisibilityAPISettings() @@ -275,7 +275,7 @@ def test_civisibility_api_client_settings_fail_after_5_retries(self): _get_setting_api_response(), ], ) as mock_do_request: - with mock.patch("ddtrace.internal.utils.retry.sleep"): + with mock.patch("ddtrace.internal.ci_visibility.utils.sleep"): with pytest.raises(socket.timeout): # raises the last exception _ = client.fetch_settings(read_from_cache=False) diff --git a/tests/ci_visibility/api_client/test_ci_visibility_api_client_setting_responses.py b/tests/ci_visibility/api_client/test_ci_visibility_api_client_setting_responses.py index 544c9ff3a1a..582680d8c8b 100644 --- a/tests/ci_visibility/api_client/test_ci_visibility_api_client_setting_responses.py +++ b/tests/ci_visibility/api_client/test_ci_visibility_api_client_setting_responses.py @@ -141,6 +141,6 @@ def test_civisibility_api_client_setting_errors(self, do_request_side_effect, ex client = self._get_test_client() with mock.patch.object(client, "_do_request", side_effect=[do_request_side_effect] * 5), pytest.raises( expected_exception - ), mock.patch("ddtrace.internal.utils.retry.sleep"): + ), mock.patch("ddtrace.internal.ci_visibility.utils.sleep"): settings = client.fetch_settings(read_from_cache=False) assert settings is None