diff --git a/package-lock.json b/package-lock.json index 785e1d1..553ef9e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "streamyx", - "version": "4.0.0-beta.58", + "version": "4.0.0-beta.59", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "streamyx", - "version": "4.0.0-beta.58", + "version": "4.0.0-beta.59", "dependencies": { "@streamyx/core": "file:src/core", "@streamyx/crunchyroll": "file:packages/crunchyroll", diff --git a/package.json b/package.json index b8ab5fb..7078624 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "streamyx", - "version": "4.0.0-beta.58", + "version": "4.0.0-beta.59", "author": "Vitaly Gashkov ", "description": "Powerful media downloader", "main": "dist/src/app/main.js", diff --git a/packages/dasha b/packages/dasha index f30c489..3c572c8 160000 --- a/packages/dasha +++ b/packages/dasha @@ -1 +1 @@ -Subproject commit f30c4890349e680c89104b20419454fa341f4427 +Subproject commit 3c572c8fa2827de4f6e0b9f7770b1c5a6bbdb37c diff --git a/packages/wive b/packages/wive index 3206d4b..90e6c3c 160000 --- a/packages/wive +++ b/packages/wive @@ -1 +1 @@ -Subproject commit 3206d4b0251c902806550b6a412c6bb85f115729 +Subproject commit 90e6c3c5b4da5d8a24e7306e53a2f91f2bb95864 diff --git a/src/app b/src/app index 63db267..dcafbff 160000 --- a/src/app +++ b/src/app @@ -1 +1 @@ -Subproject commit 63db26717854ff26bdd775b6fc15a1e1ad710377 +Subproject commit dcafbff4a4d4e10beca6d186d9558295ed79b9ec diff --git a/src/core/lib/http.ts b/src/core/lib/http.ts index 61f327c..2a8ce22 100644 --- a/src/core/lib/http.ts +++ b/src/core/lib/http.ts @@ -22,17 +22,53 @@ const HTTP_METHOD = { const USER_AGENTS = { chromeWindows: - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36', chromeMacOS: - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36', chromeLinux: - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36', + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36', smartTv: 'Mozilla/5.0 (SMART-TV; LINUX; Tizen 6.0) AppleWebKit/537.36 (KHTML, like Gecko) 76.0.3809.146/6.0 TV Safari/537.36', tizen: 'Mozilla/5.0 (Linux; U; Tizen 2.0; en-us) AppleWebKit/537.1 (KHTML, like Gecko) Mobile TizenBrowser/2.0', }; +const COMMON_HEADERS = { + Accept: + 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', + 'Accept-Encoding': 'gzip, deflate, br, zstd', + 'Accept-Language': 'ru-RU,ru;q=0.9,en-NL;q=0.8,en-US;q=0.7,en;q=0.6,vi;q=0.5', + 'Sec-Ch-Ua-Mobile': '?0', + 'Upgrade-Insecure-Requests': '1', + 'Sec-Fetch-Site': 'none', + 'Sec-Fetch-Mode': 'navigate', + 'Sec-Fetch-User': '?1', + 'Sec-Fetch-Dest': 'document', +}; + +const DEFAULT_HEADERS_MAP = { + darwin: { + ...COMMON_HEADERS, + 'User-Agent': USER_AGENTS.chromeMacOS, + 'Sec-Ch-Ua': '"Google Chrome";v="127", "Chromium";v="127", "Not.A/Brand";v="24"', + 'Sec-Ch-Ua-Platform': '"macOS"', + }, + linux: { + ...COMMON_HEADERS, + 'User-Agent': USER_AGENTS.chromeLinux, + 'Sec-Ch-Ua': '"Not/A)Brand";v="8", "Chromium";v="127", "Google Chrome";v="127"', + 'Sec-Ch-Ua-Platform': '"Linux"', + }, + win32: { + ...COMMON_HEADERS, + 'User-Agent': USER_AGENTS.chromeWindows, + 'Sec-Ch-Ua': '"Not/A)Brand";v="8", "Chromium";v="127", "Google Chrome";v="127"', + 'Sec-Ch-Ua-Platform': '"Windows"', + }, +}; + +const DEFAULT_HEADERS = DEFAULT_HEADERS_MAP[process.platform as 'darwin' | 'linux' | 'win32']; + const parseUrlFromResource = (resource: string | URL | Request) => resource instanceof Request ? new URL(resource.url) @@ -72,7 +108,7 @@ class Http implements IHttp { #proxy?: string | null; constructor({ proxy }: { proxy?: string | null } = {}) { - this.headers = { 'User-Agent': USER_AGENTS.tizen }; + this.headers = DEFAULT_HEADERS; this.cookies = []; this.#sessions = new Map(); this.#retryThreshold = 3; @@ -133,7 +169,7 @@ class Http implements IHttp { followRedirect: !redirect || redirect === 'follow', proxyUrl: this.#proxy || undefined, ...options, - headers: { ...this.headers, ...options.headers }, + headers: { ...this.headers, ...options.headers, ...DEFAULT_HEADERS }, useHeaderGenerator: true, headerGeneratorOptions: { browsers: ['chrome'] }, http2: true, @@ -156,7 +192,7 @@ class Http implements IHttp { for (const cookie of headers['set-cookie'] || []) newHeaders.append('set-cookie', cookie); this.appendCookies(response.headers['set-cookie'] || ''); delete response.headers['set-cookie']; - return new Response(response.body, { + return new Response(response.rawBody, { headers: newHeaders, status: status, }); @@ -201,14 +237,17 @@ class Http implements IHttp { } page .evaluate( - (resource, init) => { + (resource, init: globalThis.RequestInit) => { const fetchData = async () => { - const response = await globalThis.fetch( - resource as globalThis.RequestInfo, - init as globalThis.RequestInit - ); + const initBody = init.body as string | { type: 'Buffer'; data: number[] }; + const body = + typeof initBody === 'object' && initBody.type === 'Buffer' + ? Uint8Array.from(initBody.data) + : init.body; + init.body = body; + const response = await globalThis.fetch(resource as globalThis.RequestInfo, init); return { - body: await response.text(), + body: new Uint8Array(await response.arrayBuffer()), init: { headers: response.headers as unknown as Headers, status: response.status, @@ -221,7 +260,11 @@ class Http implements IHttp { resource, init ) - .then(({ body, init }) => (isWaitingForRedirect ? {} : resolve(new Response(body, init)))) + .then(({ body, init }) => { + return isWaitingForRedirect + ? {} + : resolve(new Response(Buffer.from(Object.values(body)), init)); + }) .catch((e) => { logger.debug(`Error while evaluate browser fetch: ${e?.message}`); return { body: null, init: undefined }; diff --git a/src/core/lib/service.ts b/src/core/lib/service.ts index 0d88a3d..07e5865 100644 --- a/src/core/lib/service.ts +++ b/src/core/lib/service.ts @@ -2,7 +2,7 @@ import { RunArgs } from './args'; import { IHttp } from './http'; import { IPrompt } from './prompt'; import { createStore } from './store'; -import { Http } from './http'; +import { Http, http } from './http'; import { default as fs } from './fs'; import { logger as log } from './logger'; import { prompt } from './prompt'; @@ -81,7 +81,7 @@ export type Plugin = (streamyx: StreamyxCore) => PluginInstance; export const create = (name: string): StreamyxCore => ({ log, - http: new Http(), + http, prompt, fs, store: createStore(name), @@ -120,6 +120,7 @@ export const registerService = >( core: StreamyxCore; }; const name = instance.name; + core.http = new Http(); core.store = createStore(name); core.log.debug(`Service registered: ${name}`); instance.core = core;