From ada6ff9a252b48dcaf9af5ebb5ab9d01739de5be Mon Sep 17 00:00:00 2001 From: Vitaly Gashkov Date: Mon, 25 Sep 2023 18:19:37 +0500 Subject: [PATCH 01/11] update funding --- .github/FUNDING.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index d3a85b6..27658b8 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1 +1,3 @@ -custom: ['boosty.to/vitalygashkov'] +patreon: vitalygashkov +ko_fi: vitalygashkov +custom: ['PayPal.me/vitalygashkov', 'boosty.to/vitalygashkov/donate'] From 9277dafe96d1bdb99b3507d70a2fd5fe77fd1362 Mon Sep 17 00:00:00 2001 From: Vitaly Gashkov Date: Thu, 12 Oct 2023 21:51:58 +0500 Subject: [PATCH 02/11] more types --- src/args.ts | 65 ++++++++++++++++++++++++++++++++-------------- src/downloader.ts | 3 ++- src/providers | 2 +- src/settings.ts | 51 +++++++++++++++++------------------- src/utils.ts | 2 +- streamyx.ts | 66 +++++++++++++++++++++++------------------------ 6 files changed, 105 insertions(+), 84 deletions(-) diff --git a/src/args.ts b/src/args.ts index 91708ce..48dd7ca 100644 --- a/src/args.ts +++ b/src/args.ts @@ -102,7 +102,7 @@ const config: ParseArgsConfigWithDescriptions = { }, }; -const getProcessedArgs = () => { +const getProcessedArgs = (): RunArgs => { const args = parseArgs(config); // TODO: Remove when parseArgs will be stable @@ -112,30 +112,30 @@ const getProcessedArgs = () => { } return { - urls: args.positionals, + urls: args.positionals as string[], videoHeight: parseInt(String(values['video-quality'] || '').replaceAll('p', '')), - audioQuality: values['audio-quality'], + audioQuality: values['audio-quality'] ? String(values['audio-quality']) : undefined, episodes: parseNumberRange(String(values['episodes'] || '')), seasons: parseNumberRange(String(values['seasons'] || '')), - movieTemplate: values['movie-template'], - episodeTemplate: values['episode-template'], + movieTemplate: String(values['movie-template']), + episodeTemplate: String(values['episode-template']), connections: parseInt(String(values['connections'])), - hdr: values['hdr'], - '3d': values['3d'], - hardsub: values['hardsub'], + hdr: Boolean(values['hdr']), + '3d': Boolean(values['3d']), + hardsub: Boolean(values['hardsub']), subtitleLanguages: parseArrayFromString(String(values['subs-lang'] || '')), audioLanguages: parseArrayFromString(String(values['audio-lang'] || '')), - skipSubtitles: values['skip-subs'], - skipAudio: values['skip-audio'], - skipVideo: values['skip-video'], - skipMux: values['skip-mux'], - trimBegin: values['trim-begin'], - trimEnd: values['trim-end'], - pssh: String(values['pssh'] || ''), - headers: parseHeadersFromString(String(values['headers'] || '')), - debug: values['debug'], - version: values['version'], - help: values['help'], + skipSubtitles: Boolean(values['skip-subs']), + skipAudio: Boolean(values['skip-audio']), + skipVideo: Boolean(values['skip-video']), + skipMux: Boolean(values['skip-mux']), + trimBegin: values['trim-begin'] ? String(values['trim-begin']) : undefined, + trimEnd: values['trim-end'] ? String(values['trim-end']) : undefined, + pssh: values['pssh'] ? String(values['pssh']) : undefined, + headers: values['headers'] ? parseHeadersFromString(String(values['headers'])) : undefined, + debug: Boolean(values['debug']), + version: Boolean(values['version']), + help: Boolean(values['help']), }; }; @@ -170,6 +170,33 @@ const printOptions = (options: Record = {}) => { } }; +export type RunArgs = { + urls: string[]; + videoHeight?: number; + audioQuality?: string; + episodes: number[]; + seasons: number[]; + movieTemplate: string; + episodeTemplate: string; + connections: number; + hdr: boolean; + '3d': boolean; + hardsub: boolean; + subtitleLanguages: string[]; + audioLanguages: string[]; + skipSubtitles: boolean; + skipAudio: boolean; + skipVideo: boolean; + skipMux: boolean; + trimBegin?: string; + trimEnd?: string; + pssh?: string; + headers?: Record; + debug: boolean; + version: boolean; + help: boolean; +}; + const printHelp = () => { printDescription(); printVersion(); diff --git a/src/downloader.ts b/src/downloader.ts index 4aff9cf..8cf7a67 100644 --- a/src/downloader.ts +++ b/src/downloader.ts @@ -6,6 +6,7 @@ import fs from './fs'; import { getDecryptersPool, getDecryptionKeys } from './drm'; import { decrypt } from './mp4decrypt'; import { mux } from './ffmpeg'; +import { RunArgs } from './args'; interface DownloadOptions { numberOfConnections: number; @@ -26,7 +27,7 @@ class Downloader { _config: any; _workDir = fs.join(fs.appDir, 'downloads'); - constructor(params: any) { + constructor(params: RunArgs) { this._params = params; this.http = new Http(); } diff --git a/src/providers b/src/providers index 4f56cb5..ac5e62b 160000 --- a/src/providers +++ b/src/providers @@ -1 +1 @@ -Subproject commit 4f56cb5a5ce458e5e2e686c552198369fecc0042 +Subproject commit ac5e62b5e718c7fce056d4275a28ed4df118b5a7 diff --git a/src/settings.ts b/src/settings.ts index a686bb2..b1e028b 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -1,38 +1,33 @@ import { join } from 'path'; import fs from './fs'; -export enum SettingsVideoQuality { - Lowest = 'lowest', - HD1080p = '1080p', - HD720p = '720p', - SD576p = '576p', - SD480p = '480p', - SD360p = '360p', - SD240p = '240p', - SD144p = '144p', - SD80p = '80p', - Highest = 'highest', -} +export type VideoQuality = (typeof VIDEO_QUALITY)[keyof typeof VIDEO_QUALITY]; +export const VIDEO_QUALITY = { + lowest: 'lowest', + hd1080p: '1080p', + hd720p: '720p', + sd576p: '576p', + sd480p: '480p', + sd360p: '360p', + sd240p: '240p', + sd144p: '144p', + sd80p: '80p', + highest: 'highest', +} as const; -export enum SettingsVideoDynamicRange { - High = 'HDR', - Standart = 'SDR', -} +export type VideoDynamicRange = (typeof VIDEO_DYNAMIC_RANGE)[keyof typeof VIDEO_DYNAMIC_RANGE]; +export const VIDEO_DYNAMIC_RANGE = { high: 'HDR', standart: 'SDR' } as const; -export enum SettingsAudioQuality { - Lowest = 'lowest', - Highest = 'highest', -} +export type AudioQuality = (typeof AUDIO_QUALITY)[keyof typeof AUDIO_QUALITY]; +export const AUDIO_QUALITY = { lowest: 'lowest', highest: 'highest' } as const; -export enum SubtitleStoreType { - Embedded = 'embedded', - External = 'external', -} +export type SubtitleStoreType = (typeof SUBTITLE_STORE_TYPE)[keyof typeof SUBTITLE_STORE_TYPE]; +export const SUBTITLE_STORE_TYPE = { embedded: 'embedded', external: 'external' } as const; export interface Settings { askForOptionsBeforeDownload: boolean; defaultDownloadPath: string; - preferredVideoQuality: SettingsVideoQuality; + preferredVideoQuality: VideoQuality; preferredAudioQuality: string; preferredAudioLanguages: string[]; preferredSubtitleLanguages: string[]; @@ -51,8 +46,8 @@ export interface Settings { const defaultSettings: Settings = { askForOptionsBeforeDownload: true, defaultDownloadPath: join(fs.homeDir, 'Downloads'), - preferredVideoQuality: SettingsVideoQuality.Highest, - preferredAudioQuality: SettingsAudioQuality.Highest, + preferredVideoQuality: VIDEO_QUALITY.highest, + preferredAudioQuality: AUDIO_QUALITY.highest, preferredAudioLanguages: [], preferredSubtitleLanguages: [], preferHdr: true, @@ -61,7 +56,7 @@ const defaultSettings: Settings = { downloadProxy: null, metadataProxy: null, numberOfConnections: 16, - storeSubtitlesAs: SubtitleStoreType.Embedded, + storeSubtitlesAs: SUBTITLE_STORE_TYPE.embedded, movieFilenameTemplate: '{title}.{audioType}.{quality}.{provider}.{format}.{codec}', seriesFilenameTemplate: '{show}.S{s}E{e}.{title}.{audioType}.{quality}.{provider}.{format}.{codec}', diff --git a/src/utils.ts b/src/utils.ts index 9801dce..be0ddca 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -22,7 +22,7 @@ const sleep = async (seconds: number) => const bold = (text: string) => `\x1b[1m${text}\x1b[0m`; const parseNumberRange = (rangeStr: string) => { - if (!rangeStr?.replace(/\D/g, '').trim()) return NaN; + if (!rangeStr?.replace(/\D/g, '').trim()) return []; const numbers: number[] = []; rangeStr .replaceAll(' ', '') diff --git a/streamyx.ts b/streamyx.ts index 07ee8fc..144b066 100644 --- a/streamyx.ts +++ b/streamyx.ts @@ -1,55 +1,53 @@ process.title = 'streamyx'; -import { getProcessedArgs, printHelp, printVersion } from './src/args'; +import { RunArgs, getProcessedArgs, printHelp, printVersion } from './src/args'; import { logger } from './src/logger'; -import { migrateFiles, parseMainDomain, validateUrl } from './src/utils'; +import { validateUrl } from './src/utils'; import { printDecryptionKeys } from './src/drm'; -import { createProvider } from './src/providers'; +import { getProviderByUrl } from './src/providers'; import { Downloader } from './src/downloader'; +import { loadSettings } from './src/settings'; +import { Provider } from './src/providers/provider'; -const streamyx: any = { - logger, - downloader: null, - providers: new Map(), -}; - -const startDownload = async (config: any) => { - if (!streamyx.downloader) return; - if (typeof config.drmConfig === 'function') config.drmConfig = await config.drmConfig(); - await streamyx.downloader.start(config); +const initializeProvider = async (url: string, args: RunArgs) => { + const provider = getProviderByUrl(url, args); + if (!provider) { + logger.error(`Suitable provider not found`); + process.exit(1); + } + await provider.init(); + return provider; }; -const loadProvider = async (name: string, url: string, args: any) => { - const provider = streamyx.providers.get(name) ?? createProvider(name, args); - if (provider) { - await provider.init(); - const hasProvider = streamyx.providers.has(provider.name); - if (!hasProvider) streamyx.providers.set(provider.name, provider); - const configs = await provider.getConfigList(url); - for (const config of configs) await startDownload(config); - } else { - streamyx.logger.error(`Provider <${name}> not found`); +const download = async (url: string, provider: Provider, args: RunArgs) => { + const downloader = new Downloader(args); + const configs = await provider.getConfigList(url); + for (const config of configs) { + if (typeof config.drmConfig === 'function') config.drmConfig = await config.drmConfig(); + await downloader.start(config); } }; -const loadProviders = async () => { - await migrateFiles().catch(() => null); +const initialize = async () => { + const settings = await loadSettings(); + const args = getProcessedArgs(); if (args.version) printVersion(); if (args.help) printHelp(); - streamyx.logger.setLogLevel(args.debug ? 'debug' : 'info'); - streamyx.downloader = new Downloader(args); - const urls: string[] = args.urls?.length ? args.urls : ['']; - for (const url of urls) { + logger.setLogLevel(args.debug ? 'debug' : 'info'); + + const urls = args.urls.length ? args.urls : ['']; + for (const rawUrl of urls) { if (args.pssh) { - await printDecryptionKeys(url, args.pssh, args.headers); + await printDecryptionKeys(rawUrl, args.pssh, args.headers); break; } - const validUrl = await validateUrl(url); - const domain = parseMainDomain(validUrl); - if (domain) await loadProvider(domain, validUrl, args); + + const url = await validateUrl(rawUrl); + const provider = await initializeProvider(url, args); + await download(url, provider, args); } process.exit(); }; -loadProviders(); +initialize(); From ef29304a5a1c457fa48359080cd917ecf569da9c Mon Sep 17 00:00:00 2001 From: Vitaly Gashkov Date: Thu, 12 Oct 2023 21:52:23 +0500 Subject: [PATCH 03/11] update package lock --- package-lock.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 758f816..8c4513d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "streamyx", - "version": "3.6.29", + "version": "3.6.30", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "streamyx", - "version": "3.6.29", + "version": "3.6.30", "dependencies": { "blowfish-node": "^1.1.3", "dasha": "^2.3.6", @@ -8630,6 +8630,7 @@ }, "packages/wive": { "version": "0.0.1", + "hasInstallScript": true, "license": "UNLICENSED", "dependencies": { "jsrsasign": "^10.8.6", From f2bb8c2a37c7106c9f14269d344875d20a4aa508 Mon Sep 17 00:00:00 2001 From: Vitaly Gashkov Date: Thu, 12 Oct 2023 22:23:13 +0500 Subject: [PATCH 04/11] refactor: move load args to args module; add get settings method --- src/args.ts | 9 ++++++++- src/settings.ts | 9 +++++++-- streamyx.ts | 10 +++------- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/args.ts b/src/args.ts index 48dd7ca..36d17e1 100644 --- a/src/args.ts +++ b/src/args.ts @@ -205,4 +205,11 @@ const printHelp = () => { printOptions(config.options); }; -export { getProcessedArgs, printVersion, printHelp }; +const loadArgs = () => { + const args = getProcessedArgs(); + if (args.version) printVersion(); + if (args.help) printHelp(); + return args; +}; + +export { getProcessedArgs, printVersion, printHelp, loadArgs }; diff --git a/src/settings.ts b/src/settings.ts index b1e028b..9706026 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -63,7 +63,7 @@ const defaultSettings: Settings = { chromePath: null, }; -export const getSettingsPath = async () => { +const getSettingsPath = async () => { const configDir = join(process.cwd(), 'config'); await fs.createDir(configDir); const settingsPath = join(configDir, 'settings.json'); @@ -71,9 +71,14 @@ export const getSettingsPath = async () => { return settingsPath; }; +let settings = defaultSettings; + +export const getSettings = (): Readonly => settings; + export const loadSettings = async (): Promise => { const settingsPath = await getSettingsPath(); - return fs.readJson(settingsPath); + settings = await fs.readJson(settingsPath).catch(() => defaultSettings); + return settings; }; export const saveSettings = async (settings: Partial) => { diff --git a/streamyx.ts b/streamyx.ts index 144b066..ed1a3f3 100644 --- a/streamyx.ts +++ b/streamyx.ts @@ -1,6 +1,6 @@ process.title = 'streamyx'; -import { RunArgs, getProcessedArgs, printHelp, printVersion } from './src/args'; +import { RunArgs, loadArgs } from './src/args'; import { logger } from './src/logger'; import { validateUrl } from './src/utils'; import { printDecryptionKeys } from './src/drm'; @@ -29,13 +29,9 @@ const download = async (url: string, provider: Provider, args: RunArgs) => { }; const initialize = async () => { - const settings = await loadSettings(); - - const args = getProcessedArgs(); - if (args.version) printVersion(); - if (args.help) printHelp(); + await loadSettings(); + const args = loadArgs(); logger.setLogLevel(args.debug ? 'debug' : 'info'); - const urls = args.urls.length ? args.urls : ['']; for (const rawUrl of urls) { if (args.pssh) { From 51763ee4252b21b70cb182200e06c73e524c5c80 Mon Sep 17 00:00:00 2001 From: Vitaly Gashkov Date: Thu, 12 Oct 2023 22:57:38 +0500 Subject: [PATCH 05/11] remove file migration --- src/utils.ts | 53 ---------------------------------------------------- 1 file changed, 53 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index be0ddca..a03f964 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -2,8 +2,6 @@ import { createInterface } from 'node:readline/promises'; import { stdin, stdout } from 'node:process'; import { delimiter, join } from 'node:path'; import { stat } from 'node:fs/promises'; -import fs from './fs'; -import { LOG_DIR } from './logger'; const prompt = async (message: string, type = 'input') => { const readline = createInterface({ input: stdin, output: stdout }); @@ -107,57 +105,6 @@ const validateUrl = async (url: string) => { const parseMainDomain = (url: string) => new URL(url).host.split('.').at(-2) || null; -const joinFiles = (...paths: string[]) => fs.join(fs.appDir, 'files', ...paths); - -export const migrateFiles = async () => { - const hasBinFolder = fs.exists(joinFiles('bin')); - const hasCdmFolder = fs.exists(joinFiles('cdm')); - const hasLogsFolder = fs.exists(joinFiles('logs')); - const shouldMigrate = hasBinFolder || hasCdmFolder || hasLogsFolder; - if (!shouldMigrate) return; - const binaryFiles = await fs.readDir(joinFiles('bin'), 'file'); - const binariesMoveQueue = binaryFiles.map((bin) => { - const oldPath = joinFiles('bin', bin); - const newPath = joinFiles(bin); - return fs.rename(oldPath, newPath); - }); - const cdmFolders = await fs.readDir(joinFiles('cdm'), 'dir'); - const cdmFolder = cdmFolders[0]; - const cdmFiles = await fs.readDir(joinFiles('cdm', cdmFolder), 'file'); - const cdmMoveQueue = cdmFiles.map((cdm) => { - const oldPath = joinFiles('cdm', cdmFolder, cdm); - const newPath = joinFiles(cdm); - return fs.rename(oldPath, newPath); - }); - await Promise.all([...binariesMoveQueue, ...cdmMoveQueue]); - const deleteQueue = [ - fs.delete(joinFiles('bin')).catch(() => null), - fs.delete(joinFiles('cdm')).catch(() => null), - ]; - try { - await Promise.all(deleteQueue); - } finally { - if (hasLogsFolder) { - const oldLogsPath = fs.join(fs.appDir, 'files', 'logs'); - const newLogsPath = LOG_DIR; - await fs.rename(oldLogsPath, newLogsPath); - } - } - const oldConfigPath = fs.join(fs.appDir, 'files', 'providers'); - if (fs.exists(oldConfigPath)) { - const newConfigPath = fs.join(fs.appDir, 'config'); - await fs.rename(oldConfigPath, newConfigPath); - const configFolders = await fs.readDir(fs.join(fs.appDir, 'config'), 'dir'); - const configFilesQueue = configFolders.map((configFolder) => { - return fs.rename( - fs.join(fs.appDir, 'config', configFolder, 'auth.json'), - fs.join(fs.appDir, 'config', configFolder, 'config.json') - ); - }); - await Promise.all(configFilesQueue); - } -}; - export { prompt, sleep, From 4b222a0d422b50e6379a56e18e78adc4f8372a95 Mon Sep 17 00:00:00 2001 From: Vitaly Gashkov Date: Thu, 12 Oct 2023 22:57:53 +0500 Subject: [PATCH 06/11] fix(browser): use correct options type --- src/browser.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/browser.ts b/src/browser.ts index f9f5fd0..f2db74e 100644 --- a/src/browser.ts +++ b/src/browser.ts @@ -1,18 +1,18 @@ -import { Browser, LaunchOptions, Page, Protocol } from 'puppeteer-core'; +import { Browser, BrowserLaunchArgumentOptions, Page, Protocol } from 'puppeteer-core'; import puppeteer from 'puppeteer-extra'; import StealthPlugin from 'puppeteer-extra-plugin-stealth'; import { logger } from './logger'; import { prompt } from './utils'; -import { loadSettings, saveSettings } from './settings'; +import { getSettings, saveSettings } from './settings'; puppeteer.use(StealthPlugin()); -export const launchBrowser = async (options: LaunchOptions = {}) => { - const { chromePath } = await loadSettings(); +export const launchBrowser = async (options: BrowserLaunchArgumentOptions = {}) => { + const { chromePath } = getSettings(); let executablePath: string | null = chromePath; let browser: Browser | null = null; let page: Page | null = null; - const mainOptions = { + const mainOptions: BrowserLaunchArgumentOptions = { headless: true, args: ['--no-sandbox'], userDataDir: './config/chrome', @@ -33,7 +33,7 @@ export const launchBrowser = async (options: LaunchOptions = {}) => { if (executablePath !== chromePath) saveSettings({ chromePath: executablePath }); const aboutBlankPage = (await browser.pages())[0]; if (aboutBlankPage) await aboutBlankPage.close(); - return { browser, page, executablePath }; + return { browser, page }; }; export const browserCookiesToList = (cookies: Protocol.Network.Cookie[]) => { From 1fc707d5a0e4c744f6b57f7306efa9cef5d8069b Mon Sep 17 00:00:00 2001 From: Vitaly Gashkov Date: Thu, 12 Oct 2023 23:07:51 +0500 Subject: [PATCH 07/11] refactor: merge ffmpeg & mp4decrypt into one module --- src/downloader.ts | 7 ++--- src/mp4decrypt.ts | 42 -------------------------- src/{ffmpeg.ts => process.ts} | 55 +++++++++++++++++++++++++---------- 3 files changed, 42 insertions(+), 62 deletions(-) delete mode 100644 src/mp4decrypt.ts rename src/{ffmpeg.ts => process.ts} (64%) diff --git a/src/downloader.ts b/src/downloader.ts index 8cf7a67..1dc350b 100644 --- a/src/downloader.ts +++ b/src/downloader.ts @@ -4,8 +4,7 @@ import { logger } from './logger'; import { Http } from './http'; import fs from './fs'; import { getDecryptersPool, getDecryptionKeys } from './drm'; -import { decrypt } from './mp4decrypt'; -import { mux } from './ffmpeg'; +import { ffmpeg, mp4decrypt } from './process'; import { RunArgs } from './args'; interface DownloadOptions { @@ -70,7 +69,7 @@ class Downloader { const kid = contentKeys[0].kid; const input = this.getFilepath(this.getTrackFilename(track.type, track.id, 'enc')); const output = this.getFilepath(this.getTrackFilename(track.type, track.id, 'dec')); - decryptQueue.push(decrypt(key, kid, input, output, true)); + decryptQueue.push(mp4decrypt(key, kid, input, output, true)); } await Promise.all(decryptQueue); logger.info(`Decrypted successfully`); @@ -103,7 +102,7 @@ class Downloader { } const output = this.getFilepath(this.getTrackFilename('', '', '', 'mkv')); const { trimBegin, trimEnd } = this._params; - await mux({ inputs, output, trimBegin, trimEnd, cleanup: true }); + await ffmpeg({ inputs, output, trimBegin, trimEnd, cleanup: true }); logger.info(`Muxed successfully`); } diff --git a/src/mp4decrypt.ts b/src/mp4decrypt.ts deleted file mode 100644 index ae394ce..0000000 --- a/src/mp4decrypt.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { spawn } from 'node:child_process'; -import { platform } from 'node:process'; -import { logger } from './logger'; -import fs from './fs'; -import { findExecutable } from './utils'; - -const findPath = async (exeName: string) => { - const globalPath = await findExecutable(exeName); - const localPath = fs.join(fs.appDir, 'files', exeName + (platform === 'win32' ? '.exe' : '')); - const path = globalPath || localPath; - return fs.exists(path) ? path : null; -}; - -const decrypt = async ( - key: string, - kid: string, - input: string, - output: string, - cleanup?: boolean -) => { - const exeName = 'mp4decrypt'; - const exePath = await findPath(exeName); - if (!exePath) { - logger.error(`Decryption failed. Required package is missing: ${exeName}`); - return; - } - const args = ['--show-progress', '--key', `${kid}:${key}`, input, output]; - const mp4decrypt = spawn(exePath, args); - mp4decrypt.stdout.setEncoding('utf8'); - mp4decrypt.stdout.on('data', (data) => logger.debug(data)); - mp4decrypt.stderr.setEncoding('utf8'); - mp4decrypt.stderr.on('error', (data) => logger.debug(String(data))); - await new Promise((resolve) => - mp4decrypt.on('close', () => { - mp4decrypt.kill('SIGINT'); - resolve(); - }) - ); - if (cleanup) await fs.delete(input); -}; - -export { decrypt }; diff --git a/src/ffmpeg.ts b/src/process.ts similarity index 64% rename from src/ffmpeg.ts rename to src/process.ts index 3ad4fa2..4e64831 100644 --- a/src/ffmpeg.ts +++ b/src/process.ts @@ -1,4 +1,4 @@ -import { spawn } from 'node:child_process'; +import { ChildProcessWithoutNullStreams, spawn } from 'node:child_process'; import { platform } from 'node:process'; import { logger } from './logger'; import fs from './fs'; @@ -11,7 +11,39 @@ const findPath = async (exeName: string) => { return fs.exists(path) ? path : null; }; -interface MuxOptions { +const withOutput = (process: ChildProcessWithoutNullStreams) => { + process.stdout.setEncoding('utf8'); + process.stdout.on('data', (data) => logger.debug(data)); + process.stderr.setEncoding('utf8'); + process.stderr.on('error', (data) => logger.debug(String(data))); +}; + +export const mp4decrypt = async ( + key: string, + kid: string, + input: string, + output: string, + cleanup?: boolean +) => { + const exeName = 'mp4decrypt'; + const exePath = await findPath(exeName); + if (!exePath) { + logger.error(`Decryption failed. Required package is missing: ${exeName}`); + return; + } + const args = ['--show-progress', '--key', `${kid}:${key}`, input, output]; + const process = spawn(exePath, args); + withOutput(process); + await new Promise((resolve) => + process.on('close', () => { + process.kill('SIGINT'); + resolve(); + }) + ); + if (cleanup) await fs.delete(input); +}; + +export interface MuxOptions { inputs: { id: number; language?: string; @@ -26,7 +58,7 @@ interface MuxOptions { cleanup?: boolean; } -const mux = async ({ inputs, output, trimBegin, trimEnd, cleanup }: MuxOptions) => { +export const ffmpeg = async ({ inputs, output, trimBegin, trimEnd, cleanup }: MuxOptions) => { const exeName = 'ffmpeg'; const exePath = await findPath(exeName); if (!exePath) { @@ -64,18 +96,9 @@ const mux = async ({ inputs, output, trimBegin, trimEnd, cleanup }: MuxOptions) args.push('-v', 'verbose'); args.push(output); - const ffmpeg = spawn(exePath, args); - - let error = ''; - ffmpeg.stderr.setEncoding('utf8'); - ffmpeg.stderr.on('data', (d) => (error += d)).on('end', () => error && logger.debug(error)); - let data = ''; - ffmpeg.stdout.setEncoding('utf8'); - ffmpeg.stderr.on('data', (d) => (data += d)).on('end', () => data && logger.debug(data)); - - await new Promise((resolve) => ffmpeg.on('close', resolve)); - ffmpeg.kill('SIGINT'); + const process = spawn(exePath, args); + withOutput(process); + await new Promise((resolve) => process.on('close', resolve)); + process.kill('SIGINT'); if (cleanup) for (const input of inputs) await fs.delete(input.path); }; - -export { mux }; From 7b16c95ade965e69d1704909c08ed714e643ed04 Mon Sep 17 00:00:00 2001 From: Vitaly Gashkov Date: Thu, 12 Oct 2023 23:17:07 +0500 Subject: [PATCH 08/11] refactor(browser): more types --- src/http.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/http.ts b/src/http.ts index f56f751..bee7638 100644 --- a/src/http.ts +++ b/src/http.ts @@ -7,7 +7,7 @@ import { IncomingHttpHeaders } from 'node:http'; import { URL } from 'node:url'; import { request, fetch, Request, RequestInit, Response } from 'undici'; import BodyReadable from 'undici/types/readable'; -import { Browser } from 'puppeteer-core'; +import { Browser, Page } from 'puppeteer-core'; import { logger } from './logger'; import { sleep } from './utils'; import { launchBrowser } from './browser'; @@ -51,7 +51,7 @@ class Http { #session?: ClientHttp2Session; #lastOrigin?: string; browser: Browser | null; - browserPage: any; + browserPage: Page | null; constructor() { this.headers = { 'User-Agent': USER_AGENTS.tizen }; @@ -62,6 +62,7 @@ class Http { this.#retryThreshold = 3; this.#retryDelayMs = 1500; this.browser = null; + this.browserPage = null; } get hasSessions() { @@ -87,9 +88,10 @@ class Http { } async fetchViaBrowser(resource: string | URL | Request, options?: RequestInit) { - await this.browserPage.goto(resource); + if (!this.browserPage) throw new Error('Launch browser before using fetch via browser'); + await this.browserPage.goto(resource.toString()); const { body, init } = await this.browserPage.evaluate( - (resource: any, options: any) => { + (resource, options) => { const fetchData = async () => { const response = await globalThis.fetch( resource as globalThis.RequestInfo, From 7e74ee541451855643927812c064fd696f99e5ce Mon Sep 17 00:00:00 2001 From: Vitaly Gashkov Date: Sun, 12 Nov 2023 16:49:36 +0500 Subject: [PATCH 09/11] fix: improve types, use russian language for browser --- src/browser.ts | 17 ++++++++++++++++- src/http.ts | 6 +++--- src/providers | 2 +- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/browser.ts b/src/browser.ts index f2db74e..ead50cc 100644 --- a/src/browser.ts +++ b/src/browser.ts @@ -14,7 +14,7 @@ export const launchBrowser = async (options: BrowserLaunchArgumentOptions = {}) let page: Page | null = null; const mainOptions: BrowserLaunchArgumentOptions = { headless: true, - args: ['--no-sandbox'], + args: ['--no-sandbox', '--start-maximized', '--lang=ru'], userDataDir: './config/chrome', ...options, }; @@ -33,6 +33,21 @@ export const launchBrowser = async (options: BrowserLaunchArgumentOptions = {}) if (executablePath !== chromePath) saveSettings({ chromePath: executablePath }); const aboutBlankPage = (await browser.pages())[0]; if (aboutBlankPage) await aboutBlankPage.close(); + + await page.evaluateOnNewDocument(() => { + Object.defineProperty(navigator, 'language', { + get: function () { + return 'ru'; + }, + }); + Object.defineProperty(navigator, 'languages', { + get: function () { + return ['ru']; + }, + }); + }); + await page.setExtraHTTPHeaders({ 'Accept-Language': 'ru' }); + return { browser, page }; }; diff --git a/src/http.ts b/src/http.ts index bee7638..ad9ce9a 100644 --- a/src/http.ts +++ b/src/http.ts @@ -5,7 +5,7 @@ import http2, { } from 'node:http2'; import { IncomingHttpHeaders } from 'node:http'; import { URL } from 'node:url'; -import { request, fetch, Request, RequestInit, Response } from 'undici'; +import { request, fetch, Request, RequestInit, Response, Headers, HeadersInit } from 'undici'; import BodyReadable from 'undici/types/readable'; import { Browser, Page } from 'puppeteer-core'; import { logger } from './logger'; @@ -100,10 +100,10 @@ class Http { return { body: await response.text(), init: { - headers: response.headers, + headers: response.headers as unknown as Headers, status: response.status, statusText: response.statusText, - }, + } as RequestInit, }; }; return fetchData(); diff --git a/src/providers b/src/providers index ac5e62b..bae27d7 160000 --- a/src/providers +++ b/src/providers @@ -1 +1 @@ -Subproject commit ac5e62b5e718c7fce056d4275a28ed4df118b5a7 +Subproject commit bae27d76d132beaf022d2b90dcd6e698c231021c From 4707b172471595b07ee5584ebbe4ed2ded0c63d9 Mon Sep 17 00:00:00 2001 From: Vitaly Gashkov Date: Sat, 25 Nov 2023 18:10:03 +0500 Subject: [PATCH 10/11] fix: handle if contentKeys empty --- src/downloader.ts | 2 +- src/providers | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/downloader.ts b/src/downloader.ts index 1dc350b..db91be1 100644 --- a/src/downloader.ts +++ b/src/downloader.ts @@ -47,7 +47,7 @@ class Downloader { if (drmConfig) { if (pssh) { contentKeys = await getDecryptionKeys(pssh, drmConfig); - if (!contentKeys.length) { + if (!contentKeys?.length) { logger.debug(`Decryption keys could not be obtained`); logger.debug(`Trying to decrypt through a CDM adapter (slower process)`); decryptersPool = await getDecryptersPool(pssh, drmConfig, this._params.connections); diff --git a/src/providers b/src/providers index bae27d7..04a8598 160000 --- a/src/providers +++ b/src/providers @@ -1 +1 @@ -Subproject commit bae27d76d132beaf022d2b90dcd6e698c231021c +Subproject commit 04a8598a4339cd4addc81584121b1c3d247ab8b0 From 04fddae8ab8770370e85a2426d28e2ba0d79f74b Mon Sep 17 00:00:00 2001 From: Vitaly Gashkov Date: Sat, 25 Nov 2023 18:10:38 +0500 Subject: [PATCH 11/11] chore: bump version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index cd65e90..800ab87 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "streamyx", - "version": "3.6.30", + "version": "3.6.31", "author": "Vitaly Gashkov ", "description": "Command-line video downloader", "main": "dist/streamyx.js",