Merge pull request #350 from Lezek123/proxy-setup

Improved proxy setup

Author: Leszek Wiesner

Dockerfile

@@ -28,6 +28,9 @@ COPY package.json package-lock.json ./
 # Install dependencies
 RUN npm install
 
+# Force-update yt-dlp
+RUN node ./node_modules/youtube-dl-exec/scripts/postinstall.js
+
 # Copy the rest of your application
 COPY . .
 

config.yml

@@ -46,7 +46,13 @@ youtube:
   # maxAllowedQuotaUsageInPercentage: 95
 # proxy:
 #   urls:
-#     ["socks5://proxy1:1080", "socks5://proxy2:1080"]
+#     - socks5://proxy1:1080
+#     - socks5://proxy2:1080
+#   exclusionDuration: 14400
+#   waitInterval: 60
+#   chain:
+#      url: socks5://proxy.chain:1080
+#      configDir: /etc
 aws:
   endpoint: http://localhost:4566
   region: us-east-1

docker-compose.yml

@@ -61,7 +61,6 @@ services:
       # - /var/run/docker.sock:/var/run/docker.sock
       - ./local/logs:/youtube-synch/local/logs
       - ./local/data:/youtube-synch/local/data
-      - ./proxychains4.conf:/etc/proxychains4.conf
       # mount Google Cloud's Application Default Credentials file. A default bind mount is created
       # as workaround for scenario when `YT_SYNCH__YOUTUBE__ADC_KEY_FILE_PATH` will be undefined,
       # since docker-compose does not support creating bind-mount volume with empty path.

package-lock.json

@@ -140,6 +140,7 @@
     "reflect-metadata": "^0.1.13",
     "rxjs": "^7.5.2",
     "sleep-promise": "^9.1.0",
+    "socks-proxy-agent": "^8.0.5",
     "swagger-ui-express": "^4.3.0",
     "uuid": "^9.0.0",
     "winston": "^3.8.2",

package.json

@@ -1,3 +1,4 @@
+import path from 'path'
 import fs from 'fs'
 import _ from 'lodash'
 import nodeCleanup from 'node-cleanup'
@@ -12,6 +13,8 @@ import { ContentProcessingClient, ContentProcessingService } from '../services/s
 import { YoutubePollingService } from '../services/syncProcessing/YoutubePollingService'
 import { IYoutubeApi, YoutubeApi } from '../services/youtube/api'
 import { Config, DisplaySafeConfig } from '../types'
+import { Socks5ProxyService } from '../services/proxy/Socks5ProxyService'
+import { THUMBNAILS_SUBDIR } from '../services/syncProcessing/ContentDownloadService'
 
 export class Service {
   private config: Config
@@ -25,15 +28,17 @@ export class Service {
   private youtubePollingService: YoutubePollingService
   private contentProcessingService: ContentProcessingService
   private contentProcessingClient: ContentProcessingClient
+  private socks5ProxyService?: Socks5ProxyService
   private isStopping = false
 
   constructor(config: Config) {
     this.config = config
     this.logging = LoggingService.withAppConfig(config.logs)
     this.logger = this.logging.createLogger('Server')
+    this.socks5ProxyService = config.proxy ? new Socks5ProxyService(config.proxy, this.logging) : undefined
     this.queryNodeApi = new QueryNodeApi(config.endpoints.queryNode, this.logging)
     this.dynamodbService = new DynamodbService(this.config.aws)
-    this.youtubeApi = YoutubeApi.create(this.config, this.dynamodbService.repo.stats, this.logging)
+    this.youtubeApi = YoutubeApi.create(this.config, this.dynamodbService.repo.stats, this.logging, this.socks5ProxyService)
     this.runtimeApi = new RuntimeApi(config.endpoints.joystreamNodeWs, this.logging)
     this.joystreamClient = new JoystreamClient(config, this.runtimeApi, this.queryNodeApi, this.logging)
     this.contentProcessingClient = new ContentProcessingClient({ ...config.sync, ...config.endpoints })
@@ -69,10 +74,15 @@ export class Service {
   private checkConfigDirectories(): void {
     if (this.config.sync.enable) {
       this.checkConfigDir('sync.downloadsDir', this.config.sync.downloadsDir)
+      const thumbsDir = path.join(this.config.sync.downloadsDir, THUMBNAILS_SUBDIR)
+      this.checkConfigDir('thumbnails', thumbsDir)
     }
     if (this.config.logs?.file) {
       this.checkConfigDir('logs.file.path', this.config.logs.file.path)
     }
+    if (this.config.proxy?.chain) {
+      this.checkConfigDir('proxy.chain.configDir', this.config.proxy.chain.configDir)
+    }
   }
 
   private async startSync(): Promise<void> {
@@ -84,7 +94,8 @@ export class Service {
         this.youtubeApi,
         this.runtimeApi,
         this.joystreamClient,
-        this.queryNodeApi
+        this.queryNodeApi,
+        this.socks5ProxyService
       )
 
       const {

proxychains4.conf

@@ -0,0 +1,63 @@
+import { flags } from '@oclif/command'
+import fs from 'fs'
+import path from 'path'
+import DefaultCommandBase from '../base/default'
+import { LoggingService } from '../../services/logging'
+import { YoutubeClient } from '../../services/youtube/api'
+import { Socks5ProxyService } from '../../services/proxy/Socks5ProxyService'
+import assert from 'assert'
+
+export default class DownloadVideo extends DefaultCommandBase {
+  static description = 'Perform a test video download.'
+
+  static flags = {
+    url: flags.string({
+        multiple: true,
+        description: 'YouTube video url(s) of the video(s) to be downloaded'
+    }),
+    skipProxy: flags.boolean({
+        description: 'Skip proxy even if configured.',
+        default: false,
+    }),
+    ...DefaultCommandBase.flags,
+  }
+
+  async run(): Promise<void> {
+    const { url: videoUrls, skipProxy } = this.parse(DownloadVideo).flags
+
+    const { downloadsDir } = this.appConfig.sync 
+    assert(downloadsDir, "Download dir needs to be specified!")
+
+    const logging = LoggingService.withCLIConfig()
+    let proxyConfig = this.appConfig.proxy
+    if (skipProxy) {
+        proxyConfig = undefined
+        this.log('Skipping proxy setup...')
+    } else {
+        this.log('Using proxy configuration:', proxyConfig)
+    }
+    const proxyService = proxyConfig ? new Socks5ProxyService(proxyConfig, logging) : undefined
+    const youtubeClient = new YoutubeClient(this.appConfig, logging, proxyService)
+    for (const videoUrl of videoUrls) {
+        const proxy = await proxyService?.getProxy(videoUrl)
+        if (proxy) {
+            this.log(`Selected proxy: ${proxy}`)
+        }
+        const checkResp = await youtubeClient.checkVideo(videoUrl, proxy)
+        this.log(`Downloading ${videoUrl}...`)
+        this.log(`Selected format: ${checkResp.format}`)
+        this.log(`Expected size: ${checkResp.filesize_approx}`)
+        const downloadResp = await youtubeClient.downloadVideo(videoUrl, downloadsDir, proxy)
+        assert(
+            fs.existsSync(path.join(downloadsDir, `${downloadResp.id}.${downloadResp.ext}`)),
+            'Download error: File not found'
+        )
+        proxyService?.unbindProxy(downloadResp.id)
+    }
+    this.log("All videos successfully downloaded!")
+  }
+
+  async finally(): Promise<void> {
+    /* Do nothing */
+  }
+}

src/app/index.ts

@@ -228,17 +228,41 @@ export const configSchema: JSONSchema7 = objectSchema({
       title: 'Socks5 proxy client configuration used by yt-dlp to bypass IP blockage by Youtube',
       description: 'Socks5 proxy client configuration used by yt-dlp to bypass IP blockage by Youtube',
       properties: {
+        chain: objectSchema({
+          title: 'Socks5 proxy chaining configuration',
+          properties: {
+            url: {
+              description: 'Url of the socks5 proxy that all other proxy requests will be chained through.',
+              type: 'string',
+              pattern: '^socks5://'
+            },
+            configDir: {
+              description: 'Absolute path to a location where proxychains4.conf file will be stored',
+              type: 'string'
+            },
+          },
+          required: ['url', 'configDir']
+        }),
         urls: {
           description: 'List of available socks5 proxy URLs',
           type: 'array',
           items: {
-            description: 'Socks5 proxy url, e.g. ["socks://localhost:1080"]',
+            description: 'Socks5 proxy url, e.g. ["socks5://localhost:1080"]',
             type: 'string',
+            pattern: '^socks5://'
           },
           minItems: 1,
+        },
+        waitInterval: {
+          description: 'How long should the application wait in case no proxies are available (in seconds)',
+          type: 'integer',
+        },
+        exclusionDuration: {
+          description: `How long should the proxy remain excluded in case it's blocked (in seconds)`,
+          type: 'integer',
         }
       },
-      required: [],
+      required: ['urls', 'exclusionDuration', 'waitInterval'],
     }),
 
     creatorOnboardingRequirements: objectSchema({

src/cli/commands/downloadVideo.ts

@@ -0,0 +1,68 @@
+import axios, { AddressFamily, AxiosRequestConfig } from 'axios'
+import assert from 'assert'
+import { argv } from 'process'
+import { SocksProxyAgent } from 'socks-proxy-agent'
+import { Readable, promises as streamPromise } from 'stream'
+import path from 'path'
+import { createWriteStream } from 'fs'
+import dns from 'dns'
+
+const isTS =  path.extname(__filename) === '.ts'
+export const SCRIPT_PATH = __filename.split(path.sep)
+  .map((part) => part === 'src' ? 'lib' : part)
+  .join(path.sep)
+  .replace('.ts', '.js')
+
+function validateArgs() {
+  try {
+    const assetUrl = new URL(argv[2])
+    assert(assetUrl.protocol === 'http:' || assetUrl.protocol === 'https:', `Invalid assetUrl protocol: ${assetUrl.protocol}`)
+
+    const destPath = argv[3]
+    if (!path.isAbsolute(destPath) || !path.extname(destPath)) {
+      throw new Error('destPath must be an absolute path to a file!')
+    }
+
+    let proxyUrl: URL | undefined = undefined
+    if (argv[4]) {
+      proxyUrl = new URL(argv[4])
+      assert(proxyUrl.protocol === 'socks5:', `Invalid proxyUrl protocol: ${proxyUrl.protocol}`)
+    }
+
+    return { assetUrl: assetUrl.toString(), destPath, proxyUrl: proxyUrl?.toString() }
+  } catch (e: any) {
+    throw new Error(`Invalid arguments: ${e.toString()}`)
+  }
+}
+
+// Define a custom lookup function that forces IPv4 resolution
+const ipv4Lookup: AxiosRequestConfig<any>['lookup'] = (hostname, options, callback) => {
+  return dns.lookup(hostname, { ...options, family: 4 }, (err, address, family) => callback(err, address, family as AddressFamily));
+}
+
+async function main() {
+  const { assetUrl, destPath, proxyUrl } = validateArgs()
+  const reqConfig: AxiosRequestConfig<any> = {
+    responseType: 'stream',
+    family: 4,
+    lookup: ipv4Lookup
+  }
+  if (proxyUrl) {
+    const proxyAgent = new SocksProxyAgent(proxyUrl)
+    reqConfig.httpAgent = proxyAgent
+    reqConfig.httpsAgent = proxyAgent
+  }
+  const response = await axios.get<Readable>(assetUrl, reqConfig)
+  const writeStream = createWriteStream(destPath)
+  await streamPromise.pipeline([response.data, writeStream])
+}
+
+// Execute script only if the file was executed directly (not imported as module)
+if (require.main === module && !isTS) {
+  main()
+    .then(() => process.exit(0))
+    .catch(e => {
+      console.error(e)
+      process.exit(1)
+    })
+}