Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add a basic retry on fetch #44

Merged
merged 5 commits into from
Dec 27, 2024
Merged

Add a basic retry on fetch #44

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
3be502e
add retry: 3 times, interval 1.5s
grqx Dec 26, 2024
b6f96c0
var casing
grqx Dec 27, 2024
545c4c1
- [server] increase retry interval to 5s
grqx Dec 27, 2024
54b9255
[plugin/script] add timeout: 20.0s
grqx Dec 27, 2024
11e15ce
use repr for error causes
grqx Dec 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from __future__ import annotations

Check warning on line 1 in plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py

View workflow job for this annotation

GitHub Actions / Test plugin (server method)

yt-dlp failed when testing HTTP server 

yt-dlp returned 1 exit status

import json
import typing
Expand Down Expand Up @@ -40,12 +40,12 @@
f'{base_url}/ping', extensions={'timeout': 5.0}, proxies={'all': None}))
except Exception as e:
raise UnsupportedRequest(
f'Error reaching GET /ping (caused by {e!s})') from e
f'Error reaching GET /ping (caused by {e!r})') from e
try:
response = json.load(response)
except json.JSONDecodeError as e:
raise UnsupportedRequest(
f'Error parsing response JSON (caused by {e!s})'
f'Error parsing response JSON (caused by {e!r})'
f', response: {response.read()}') from e
if response.get('version') != self.VERSION:
self._logger.warning(
Expand All @@ -72,16 +72,16 @@
'data_sync_id': data_sync_id,
'proxy': proxy,
}).encode(), headers={'Content-Type': 'application/json'},
extensions={'timeout': 12.5}, proxies={'all': None}))
extensions={'timeout': 20.0}, proxies={'all': None}))
except Exception as e:
raise RequestError(
f'Error reaching POST /get_pot (caused by {e!s})') from e
f'Error reaching POST /get_pot (caused by {e!r})') from e

try:
response_json = json.load(response)
except Exception as e:
raise RequestError(
f'Error parsing response JSON (caused by {e!s}). response = {response.read().decode()}') from e
f'Error parsing response JSON (caused by {e!r}). response = {response.read().decode()}') from e

if error_msg := response_json.get('error'):
raise RequestError(error_msg)
Expand Down
10 changes: 7 additions & 3 deletions plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from __future__ import annotations

Check warning on line 1 in plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py

View workflow job for this annotation

GitHub Actions / Test plugin (script method)

yt-dlp failed when testing script 

yt-dlp returned 1 exit status

import json
import os.path
Expand Down Expand Up @@ -76,10 +76,14 @@

try:
stdout, stderr, returncode = Popen.run(
command_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
command_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
timeout=20.0)
except subprocess.TimeoutExpired as e:
raise RequestError(
f'_get_pot_via_script failed: Timeout expired when trying to run script (caused by {e!r})')
except Exception as e:
raise RequestError(
f'_get_pot_via_script failed: Unable to run script (caused by {e!s})') from e
f'_get_pot_via_script failed: Unable to run script (caused by {e!r})') from e

msg = f'stdout:\n{stdout.strip()}'
if stderr.strip(): # Empty strings are falsy
Expand All @@ -94,7 +98,7 @@
script_data_resp = json.loads(stdout.splitlines()[-1])
except json.JSONDecodeError as e:
raise RequestError(
f'Error parsing JSON response from _get_pot_via_script (caused by {e!s})') from e
f'Error parsing JSON response from _get_pot_via_script (caused by {e!r})') from e
else:
self._logger.debug(
f'_get_pot_via_script response = {script_data_resp}')
Expand Down
56 changes: 32 additions & 24 deletions server/src/session_manager.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,30 +165,38 @@ export class SessionManager {

const bgConfig: BgConfig = {
fetch: async (url: any, options: any): Promise<any> => {
try {
const response = await axios.post(url, options.body, {
headers: {
...options.headers,
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
},
httpsAgent: dispatcher,
});

return {
ok: true,
json: async () => {
return response.data;
},
};
} catch (e) {
return {
ok: false,
json: async () => {
return null;
},
status: e.response?.status || e.code,
};
const maxRetries = 3;
for (let attempts = 1; attempts <= maxRetries; attempts++) {
try {
const response = await axios.post(url, options.body, {
headers: {
...options.headers,
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
},
httpsAgent: dispatcher,
});

return {
ok: true,
json: async () => {
return response.data;
},
};
} catch (e) {
if (attempts >= maxRetries) {
return {
ok: false,
json: async () => {
return null;
},
status: e.response?.status || e.code,
};
}
await new Promise((resolve) =>
setTimeout(resolve, 5000),
);
}
}
},
globalObj: globalThis,
Expand Down
Loading