adds a url filter

Author: matstc

lib/arachnid.rb

@@ -23,6 +23,7 @@ def initialize(urls, options = {})
   def crawl(options = {})
     threads = options[:threads] || 1
     max_urls = options[:max_urls]
+    filter = options[:filter]
 
     @hydra = Typhoeus::Hydra.new(:max_concurrency => threads)
     @global_visited = BloomFilter::Native.new(:size => 1000000, :hashes => 5, :seed => 1, :bucket => 8, :raise => false)
@@ -40,6 +41,10 @@ def crawl(options = {})
           break
         end
 
+        if filter
+          next unless filter.call(q)
+        end
+
         @global_visited.insert(q)
         puts "Processing link: #{q}" if @debug