Add a basic retry on fetch (#44)

* add retry: 3 times, interval 1.5s * var casing * - [server] increase retry interval to 5s - [plugin/http] increase client timeout to 20s * [plugin/script] add timeout: 20.0s * use repr for error causes --------- Co-authored-by: grqx_wsl <[email protected]>
Brainicism · Dec 27, 2024 · b532c53 · b532c53
1 parent f2ce320
commit b532c53
Show file tree

Hide file tree

Showing 3 changed files with 44 additions and 32 deletions.
diff --git a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py
@@ -40,12 +40,12 @@ def _validate_get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data
                 f'{base_url}/ping', extensions={'timeout': 5.0}, proxies={'all': None}))
         except Exception as e:
             raise UnsupportedRequest(
-                f'Error reaching GET /ping (caused by {e!s})') from e
+                f'Error reaching GET /ping (caused by {e!r})') from e
         try:
             response = json.load(response)
         except json.JSONDecodeError as e:
             raise UnsupportedRequest(
-                f'Error parsing response JSON (caused by {e!s})'
+                f'Error parsing response JSON (caused by {e!r})'
                 f', response: {response.read()}') from e
         if response.get('version') != self.VERSION:
             self._logger.warning(
@@ -72,16 +72,16 @@ def _get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data_sync_id=
                     'data_sync_id': data_sync_id,
                     'proxy': proxy,
                 }).encode(), headers={'Content-Type': 'application/json'},
-                extensions={'timeout': 12.5}, proxies={'all': None}))
+                extensions={'timeout': 20.0}, proxies={'all': None}))
         except Exception as e:
             raise RequestError(
-                f'Error reaching POST /get_pot (caused by {e!s})') from e
+                f'Error reaching POST /get_pot (caused by {e!r})') from e
 
         try:
             response_json = json.load(response)
         except Exception as e:
             raise RequestError(
-                f'Error parsing response JSON (caused by {e!s}). response = {response.read().decode()}') from e
+                f'Error parsing response JSON (caused by {e!r}). response = {response.read().decode()}') from e
 
         if error_msg := response_json.get('error'):
             raise RequestError(error_msg)

diff --git a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py
@@ -76,10 +76,14 @@ def _get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data_sync_id=
 
         try:
             stdout, stderr, returncode = Popen.run(
-                command_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+                command_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
+                timeout=20.0)
+        except subprocess.TimeoutExpired as e:
+            raise RequestError(
+                f'_get_pot_via_script failed: Timeout expired when trying to run script (caused by {e!r})')
         except Exception as e:
             raise RequestError(
-                f'_get_pot_via_script failed: Unable to run script (caused by {e!s})') from e
+                f'_get_pot_via_script failed: Unable to run script (caused by {e!r})') from e
 
         msg = f'stdout:\n{stdout.strip()}'
         if stderr.strip():  # Empty strings are falsy
@@ -94,7 +98,7 @@ def _get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data_sync_id=
             script_data_resp = json.loads(stdout.splitlines()[-1])
         except json.JSONDecodeError as e:
             raise RequestError(
-                f'Error parsing JSON response from _get_pot_via_script (caused by {e!s})') from e
+                f'Error parsing JSON response from _get_pot_via_script (caused by {e!r})') from e
         else:
             self._logger.debug(
                 f'_get_pot_via_script response = {script_data_resp}')

diff --git a/server/src/session_manager.ts b/server/src/session_manager.ts
@@ -165,30 +165,38 @@ export class SessionManager {
 
         const bgConfig: BgConfig = {
             fetch: async (url: any, options: any): Promise<any> => {
-                try {
-                    const response = await axios.post(url, options.body, {
-                        headers: {
-                            ...options.headers,
-                            "User-Agent":
-                                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
-                        },
-                        httpsAgent: dispatcher,
-                    });
-
-                    return {
-                        ok: true,
-                        json: async () => {
-                            return response.data;
-                        },
-                    };
-                } catch (e) {
-                    return {
-                        ok: false,
-                        json: async () => {
-                            return null;
-                        },
-                        status: e.response?.status || e.code,
-                    };
+                const maxRetries = 3;
+                for (let attempts = 1; attempts <= maxRetries; attempts++) {
+                    try {
+                        const response = await axios.post(url, options.body, {
+                            headers: {
+                                ...options.headers,
+                                "User-Agent":
+                                    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+                            },
+                            httpsAgent: dispatcher,
+                        });
+
+                        return {
+                            ok: true,
+                            json: async () => {
+                                return response.data;
+                            },
+                        };
+                    } catch (e) {
+                        if (attempts >= maxRetries) {
+                            return {
+                                ok: false,
+                                json: async () => {
+                                    return null;
+                                },
+                                status: e.response?.status || e.code,
+                            };
+                        }
+                        await new Promise((resolve) =>
+                            setTimeout(resolve, 5000),
+                        );
+                    }
                 }
             },
             globalObj: globalThis,