diff --git a/README.md b/README.md index 00a189b..a576d99 100644 --- a/README.md +++ b/README.md @@ -29,11 +29,19 @@ Clone this git repository and build a Gem. The program should work without addit ``` Usage: yums3sync [options] - -s, --source SOURCE HTTP source URL - -b, --bucket BUCKET Target bucket name + +Specific options: + -s, --source [SOURCE] HTTP source URL + -b, --bucket [BUCKET] Target bucket name -p, --prefix PREFIX Target bucket prefix -k, --keep Never overwrite exitant files -n, --dry-run Don't make any changes + +Authentication options: + -a, --authentication Allow authentication method to http(s) + --username [USERNAME] Username to use in the http(s) + --password [PASSWORD] Password to use in the https + -d, --disable Disable SSL to check http(s) ``` Example usage: diff --git a/bin/yums3sync b/bin/yums3sync index 7702136..a36a819 100755 --- a/bin/yums3sync +++ b/bin/yums3sync @@ -3,35 +3,75 @@ require 'optparse' require 'yum_s3_sync' + options = {} opt_parser = OptionParser.new do |opts| opts.banner = 'Usage: yums3sync [options]' + opts.separator "" + opts.separator "Specific options:" - opts.on('-s', '--source SOURCE', 'HTTP source URL') do |s| + opts.on('-s', '--source [SOURCE]', 'HTTP source URL') do |s| options[:source_base] = s end - opts.on('-b', '--bucket BUCKET', 'Target bucket name') do |b| + + opts.on('-b', '--bucket [BUCKET]', 'Target bucket name') do |b| options[:target_bucket] = b end + opts.on('-p', '--prefix PREFIX', 'Target bucket prefix') do |p| options[:target_base] = p end + opts.on('-k', '--keep', 'Never overwrite exitant files') do |k| options[:keep] = true end + opts.on('-n', '--dry-run', 'Don\'t make any changes') do |n| options[:dry_run] = true end + + opts.separator "" + opts.separator "Authentication options:" + options[:authentication] = false + opts.on('-a', '--authentication', 'Allow authentication method to http(s)') do |a| + options[:authentication] = true + end + options[:username] = "" + opts.on('--username [USERNAME]', 'Username to use in the http(s)') do |u| + options[:username] = u + end + options[:password] = "" + opts.on('--password [PASSWORD]', 'Password to use in the https') do |pa| + options[:password] = pa + end + opts.on('-d', '--disable', 'Disable SSL to check http(s)') do |a| + options[:ssl] = true + end end -opt_parser.parse! +begin opt_parser.parse! +rescue OptionParser::InvalidOption => e + puts e + puts opt_parser + exit 1 +end if !options[:source_base] || !options[:target_bucket] || !options[:target_base] puts opt_parser exit 1 end -repo_syncer = YumS3Sync::RepoSyncer.new(options[:source_base], options[:target_bucket], options[:target_base], options[:keep], options[:dry_run]) +if options[:authentication] + if (options[:username].empty? or options[:password].empty?) + puts opt_parser + exit 1 + end +else + options.delete(:username) + options.delete(:password) +end + +repo_syncer = YumS3Sync::RepoSyncer.new(options[:source_base], options[:target_bucket], options[:target_base], options) begin repo_syncer.sync diff --git a/lib/yum_s3_sync/http_downloader.rb b/lib/yum_s3_sync/http_downloader.rb index ca8b3f2..458b349 100644 --- a/lib/yum_s3_sync/http_downloader.rb +++ b/lib/yum_s3_sync/http_downloader.rb @@ -1,10 +1,15 @@ require 'open-uri' require 'socket' +require 'openssl' + module YumS3Sync class HTTPDownloader - def initialize(baseurl) + def initialize(baseurl, options) @baseurl = baseurl + @keep = options[:keep] + @dry_run = options[:dry_run] + @options = options end def download(relative_url) @@ -12,9 +17,9 @@ def download(relative_url) url = "#{@baseurl}/#{relative_url}" puts "Downloading #{url}" - + begin - open("#{url}") + open("#{url}", @options) rescue OpenURI::HTTPError => e if e.io.status[0] == '404' raise "File #{url} does not exist 404" @@ -34,4 +39,4 @@ def download(relative_url) end end end -end +end \ No newline at end of file diff --git a/lib/yum_s3_sync/repo_syncer.rb b/lib/yum_s3_sync/repo_syncer.rb index 50045bc..b461bad 100644 --- a/lib/yum_s3_sync/repo_syncer.rb +++ b/lib/yum_s3_sync/repo_syncer.rb @@ -3,18 +3,39 @@ require 'yum_s3_sync' require 'parallel' + module YumS3Sync class RepoSyncer - def initialize(source_base, target_bucket, target_base, keep = false, dry_run = false) + def initialize(source_base, target_bucket, target_base, options) @source_base = source_base @target_bucket = target_bucket @target_base = target_base - @keep = keep - @dry_run = dry_run + @keep = options[:keep] + @dry_run = options[:dry_run] + if options[:ssl] == true + options[:ssl_verify_mode] = OpenSSL::SSL::VERIFY_NONE + options.delete(:ssl) + end + + if options[:authentication] == true + options[:http_basic_authentication] = [options[:username], options[:password]] + options.delete(:username) + options.delete(:password) + options.delete(:authentication) + end + if options[:authentication] == false + options.delete(:authentication) + end + options.delete(:keep) + options.delete(:dry_run) + options.delete(:source_base) + options.delete(:target_bucket) + options.delete(:target_base) + @options = options end def sync - http_downloader = YumS3Sync::HTTPDownloader.new(@source_base) + http_downloader = YumS3Sync::HTTPDownloader.new(@source_base, @options) source_repository = YumS3Sync::YumRepository.new(http_downloader) s3_downloader = YumS3Sync::S3Downloader.new(@target_bucket, @target_base) diff --git a/lib/yum_s3_sync/s3_deleter.rb b/lib/yum_s3_sync/s3_deleter.rb index 6cce219..3ae791a 100644 --- a/lib/yum_s3_sync/s3_deleter.rb +++ b/lib/yum_s3_sync/s3_deleter.rb @@ -1,4 +1,4 @@ -require 'aws-sdk' +require 'aws-sdk-v1' module YumS3Sync class S3Deleter diff --git a/lib/yum_s3_sync/s3_downloader.rb b/lib/yum_s3_sync/s3_downloader.rb index e07d137..a02d6cf 100644 --- a/lib/yum_s3_sync/s3_downloader.rb +++ b/lib/yum_s3_sync/s3_downloader.rb @@ -1,4 +1,4 @@ -require 'aws-sdk' +require 'aws-sdk-v1' module YumS3Sync class S3Downloader diff --git a/lib/yum_s3_sync/s3_file_lister.rb b/lib/yum_s3_sync/s3_file_lister.rb index cd751d9..75009a5 100644 --- a/lib/yum_s3_sync/s3_file_lister.rb +++ b/lib/yum_s3_sync/s3_file_lister.rb @@ -1,4 +1,4 @@ -require 'aws-sdk' +require 'aws-sdk-v1' require 'parallel' module YumS3Sync diff --git a/lib/yum_s3_sync/s3_uploader.rb b/lib/yum_s3_sync/s3_uploader.rb index bf3909c..33646d3 100644 --- a/lib/yum_s3_sync/s3_uploader.rb +++ b/lib/yum_s3_sync/s3_uploader.rb @@ -1,4 +1,4 @@ -require 'aws-sdk' +require 'aws-sdk-v1' module YumS3Sync class S3Uploader diff --git a/yum_s3_sync-0.0.13.gem b/yum_s3_sync-0.0.13.gem new file mode 100644 index 0000000..ae1a916 Binary files /dev/null and b/yum_s3_sync-0.0.13.gem differ diff --git a/yum_s3_sync.gemspec b/yum_s3_sync.gemspec index 81fcd38..d8748d7 100644 --- a/yum_s3_sync.gemspec +++ b/yum_s3_sync.gemspec @@ -13,14 +13,13 @@ Gem::Specification.new do |spec| spec.license = 'MIT' spec.files = `git ls-files -z`.split("\x0") - spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) } - spec.test_files = spec.files.grep(%r{^(test|spec|features)/}) + spec.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) } spec.require_paths = ['lib'] spec.add_development_dependency 'bundler', '~> 1.7' spec.add_development_dependency 'rake', '~> 10.0' - spec.add_dependency 'nokogiri', '>= 1.4.3' - spec.add_dependency 'parallel' - spec.add_dependency 'aws-sdk' + spec.add_runtime_dependency(%q, [">= 1.4.3"]) + spec.add_runtime_dependency(%q, [">= 1.6.1"]) + spec.add_runtime_dependency(%q, [">= 0"]) end