From b532c53dc4d91dd256451e91a7a78e809960db9a Mon Sep 17 00:00:00 2001 From: N/Ame <173015200+grqz@users.noreply.github.com> Date: Sat, 28 Dec 2024 00:42:25 +1300 Subject: [PATCH] Add a basic retry on `fetch` (#44) * add retry: 3 times, interval 1.5s * var casing * - [server] increase retry interval to 5s - [plugin/http] increase client timeout to 20s * [plugin/script] add timeout: 20.0s * use repr for error causes --------- Co-authored-by: grqx_wsl <173253225+grqx@users.noreply.github.com> --- .../extractor/getpot_bgutil_http.py | 10 ++-- .../extractor/getpot_bgutil_script.py | 10 +++- server/src/session_manager.ts | 56 +++++++++++-------- 3 files changed, 44 insertions(+), 32 deletions(-) diff --git a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py index d0a4d35..13f2ffb 100644 --- a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py +++ b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py @@ -40,12 +40,12 @@ def _validate_get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data f'{base_url}/ping', extensions={'timeout': 5.0}, proxies={'all': None})) except Exception as e: raise UnsupportedRequest( - f'Error reaching GET /ping (caused by {e!s})') from e + f'Error reaching GET /ping (caused by {e!r})') from e try: response = json.load(response) except json.JSONDecodeError as e: raise UnsupportedRequest( - f'Error parsing response JSON (caused by {e!s})' + f'Error parsing response JSON (caused by {e!r})' f', response: {response.read()}') from e if response.get('version') != self.VERSION: self._logger.warning( @@ -72,16 +72,16 @@ def _get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data_sync_id= 'data_sync_id': data_sync_id, 'proxy': proxy, }).encode(), headers={'Content-Type': 'application/json'}, - extensions={'timeout': 12.5}, proxies={'all': None})) + extensions={'timeout': 20.0}, proxies={'all': None})) except Exception as e: raise RequestError( - f'Error reaching POST /get_pot (caused by {e!s})') from e + f'Error reaching POST /get_pot (caused by {e!r})') from e try: response_json = json.load(response) except Exception as e: raise RequestError( - f'Error parsing response JSON (caused by {e!s}). response = {response.read().decode()}') from e + f'Error parsing response JSON (caused by {e!r}). response = {response.read().decode()}') from e if error_msg := response_json.get('error'): raise RequestError(error_msg) diff --git a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py index 7e616a1..45bbe2b 100644 --- a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py +++ b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py @@ -76,10 +76,14 @@ def _get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data_sync_id= try: stdout, stderr, returncode = Popen.run( - command_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + command_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + timeout=20.0) + except subprocess.TimeoutExpired as e: + raise RequestError( + f'_get_pot_via_script failed: Timeout expired when trying to run script (caused by {e!r})') except Exception as e: raise RequestError( - f'_get_pot_via_script failed: Unable to run script (caused by {e!s})') from e + f'_get_pot_via_script failed: Unable to run script (caused by {e!r})') from e msg = f'stdout:\n{stdout.strip()}' if stderr.strip(): # Empty strings are falsy @@ -94,7 +98,7 @@ def _get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data_sync_id= script_data_resp = json.loads(stdout.splitlines()[-1]) except json.JSONDecodeError as e: raise RequestError( - f'Error parsing JSON response from _get_pot_via_script (caused by {e!s})') from e + f'Error parsing JSON response from _get_pot_via_script (caused by {e!r})') from e else: self._logger.debug( f'_get_pot_via_script response = {script_data_resp}') diff --git a/server/src/session_manager.ts b/server/src/session_manager.ts index f3111b6..cf0d88e 100644 --- a/server/src/session_manager.ts +++ b/server/src/session_manager.ts @@ -165,30 +165,38 @@ export class SessionManager { const bgConfig: BgConfig = { fetch: async (url: any, options: any): Promise => { - try { - const response = await axios.post(url, options.body, { - headers: { - ...options.headers, - "User-Agent": - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", - }, - httpsAgent: dispatcher, - }); - - return { - ok: true, - json: async () => { - return response.data; - }, - }; - } catch (e) { - return { - ok: false, - json: async () => { - return null; - }, - status: e.response?.status || e.code, - }; + const maxRetries = 3; + for (let attempts = 1; attempts <= maxRetries; attempts++) { + try { + const response = await axios.post(url, options.body, { + headers: { + ...options.headers, + "User-Agent": + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", + }, + httpsAgent: dispatcher, + }); + + return { + ok: true, + json: async () => { + return response.data; + }, + }; + } catch (e) { + if (attempts >= maxRetries) { + return { + ok: false, + json: async () => { + return null; + }, + status: e.response?.status || e.code, + }; + } + await new Promise((resolve) => + setTimeout(resolve, 5000), + ); + } } }, globalObj: globalThis,