test.rb

#!/usr/bin/env ruby
# frozen_string_literal: true

$LOAD_PATH.unshift File.join(__dir__, "lib")
require "spidra"

API_KEY = ENV["SPIDRA_API_KEY"] or abort "Set SPIDRA_API_KEY"
client  = Spidra.new(API_KEY)

# ── helpers ───────────────────────────────────────────────────────────────────

results = []
$completed_crawl_job_id = nil
$batch_id               = nil
$first_log_uuid         = nil

def assert(condition, msg)
  raise msg unless condition
end

def test(name)
  print "  #{name} ... "
  $stdout.flush
  begin
    yield
    puts "\e[32mPASS\e[0m"
    :pass
  rescue => e
    puts "\e[31mFAIL\e[0m — #{e.message}"
    :fail
  end
end

# Poll a job manually until terminal or timeout
def poll_until_done(client_resource, id_method, id_val, timeout: 180, interval: 3)
  deadline = Time.now + timeout
  loop do
    status = client_resource.get(id_val)
    return status if %w[completed failed cancelled].include?(status["status"])
    raise "Timed out after #{timeout}s" if Time.now > deadline
    sleep interval
  end
end

# ── SCRAPE ────────────────────────────────────────────────────────────────────
puts "\n── Scrape ──────────────────────────────────────────────────────────────"

results << test("scrape.run — basic markdown") do
  job = client.scrape.run(
    urls:   [{ url: "https://books.toscrape.com" }],
    prompt: "What is the title of this website?"
  )
  assert job["status"] == "completed", "expected completed, got #{job["status"]}"
  assert job["jobId"].is_a?(String) && !job["jobId"].empty?, "missing jobId"
  content = job["result"]["content"]
  assert !content.nil? && content != "", "content is nil or empty (got #{content.inspect})"
end

results << test("scrape.run — JSON output") do
  job = client.scrape.run(
    urls:   [{ url: "https://books.toscrape.com" }],
    prompt: "Return the page title as JSON",
    output: "json"
  )
  assert job["status"] == "completed", "expected completed, got #{job["status"]}"
  assert !job["result"]["content"].nil?, "content is nil"
end

results << test("scrape.run — with JSON schema") do
  job = client.scrape.run(
    urls:   [{ url: "https://books.toscrape.com" }],
    prompt: "Extract the page title and the total number of books shown",
    output: "json",
    schema: {
      type: "object",
      required: ["page_title", "book_count"],
      properties: {
        page_title: { type: "string" },
        book_count: { type: ["integer", "null"] }
      }
    }
  )
  assert job["status"] == "completed", "expected completed, got #{job["status"]}"
  content = job["result"]["content"]
  assert content.is_a?(Hash), "expected Hash, got #{content.class}: #{content.inspect}"
  assert content.key?("page_title"), "missing page_title key in #{content.inspect}"
end

results << test("scrape.run — extractContentOnly") do
  job = client.scrape.run(
    urls:               [{ url: "https://books.toscrape.com" }],
    prompt:             "List 3 book titles",
    extractContentOnly: true
  )
  assert job["status"] == "completed", "expected completed, got #{job["status"]}"
  assert !job["result"]["content"].nil?, "content is nil"
end

results << test("scrape.run — useProxy + proxyCountry") do
  job = client.scrape.run(
    urls:         [{ url: "https://books.toscrape.com" }],
    prompt:       "What is the page title?",
    useProxy:     true,
    proxyCountry: "gb"
  )
  assert job["status"] == "completed", "expected completed, got #{job["status"]}"
  assert !job["result"]["content"].nil?, "content is nil"
end

results << test("scrape.run — multiple URLs (3 in parallel)") do
  job = client.scrape.run(
    urls: [
      { url: "https://books.toscrape.com/catalogue/page-1.html" },
      { url: "https://books.toscrape.com/catalogue/page-2.html" },
      { url: "https://books.toscrape.com/catalogue/page-3.html" }
    ],
    prompt: "Return the page number from the URL"
  )
  assert job["status"] == "completed", "expected completed, got #{job["status"]}"
  content = job["result"]["content"]
  assert content.is_a?(Array) && content.length == 3,
    "expected 3-element array, got #{content.class}: #{content.inspect}"
end

results << test("scrape.run — browser actions (scroll + wait)") do
  job = client.scrape.run(
    urls: [
      {
        url:     "https://books.toscrape.com",
        actions: [
          { type: "scroll", to: "50%" },
          { type: "wait",   duration: 500 }
        ]
      }
    ],
    prompt: "List 3 book titles visible on the page"
  )
  assert job["status"] == "completed", "expected completed, got #{job["status"]}"
  assert !job["result"]["content"].nil?, "content is nil"
end

results << test("scrape.run — browser action type (fill input)") do
  job = client.scrape.run(
    urls: [
      {
        url:     "https://en.wikipedia.org/wiki/Main_Page",
        actions: [
          { type: "type", selector: "input[name='search']", value: "Ruby programming" }
        ]
      }
    ],
    prompt: "What value is in the search input?"
  )
  assert job["status"] == "completed", "expected completed, got #{job["status"]}"
  assert !job["result"]["content"].nil?, "content is nil"
end

results << test("scrape.run — poll_interval and timeout options") do
  job = client.scrape.run(
    { urls: [{ url: "https://books.toscrape.com" }], prompt: "Page title only" },
    poll_interval: 4,
    timeout: 90
  )
  assert job["status"] == "completed", "expected completed, got #{job["status"]}"
  assert !job["result"]["content"].nil?, "content is nil"
end

results << test("scrape.submit + scrape.get — async flow") do
  submit = client.scrape.submit(
    urls:   [{ url: "https://books.toscrape.com" }],
    prompt: "Count the number of books shown on the page"
  )
  assert submit["jobId"].is_a?(String) && !submit["jobId"].empty?, "missing jobId from submit"

  job_id   = submit["jobId"]
  deadline = Time.now + 120
  result   = nil

  loop do
    result = client.scrape.get(job_id)
    break if %w[completed failed].include?(result["status"])
    raise "Timed out polling scrape job" if Time.now > deadline
    sleep 3
  end

  assert result["status"] == "completed", "expected completed, got #{result["status"]}"
  assert !result["result"]["content"].nil?, "content is nil"
end

# ── BATCH ─────────────────────────────────────────────────────────────────────
puts "\n── Batch ───────────────────────────────────────────────────────────────"

results << test("batch.run — 3 URLs") do
  batch = client.batch.run(
    urls: [
      "https://books.toscrape.com/catalogue/page-1.html",
      "https://books.toscrape.com/catalogue/page-2.html",
      "https://books.toscrape.com/catalogue/page-3.html"
    ],
    prompt: "How many books are listed on this page?",
    output: "json"
  )
  assert batch["status"] == "completed", "expected completed, got #{batch["status"]}"
  assert batch["batchId"].is_a?(String), "missing batchId"
  assert batch["totalUrls"] == 3, "expected 3 total URLs, got #{batch["totalUrls"]}"
  $batch_id = batch["batchId"]
end

results << test("batch.submit + batch.get — async flow") do
  submit = client.batch.submit(
    urls:   ["https://books.toscrape.com/catalogue/page-4.html"],
    prompt: "Extract the page number"
  )
  assert submit["batchId"].is_a?(String), "missing batchId from submit"

  batch_id = submit["batchId"]
  deadline = Time.now + 120
  result   = nil

  loop do
    result = client.batch.get(batch_id)
    break if %w[completed failed cancelled].include?(result["status"])
    raise "Timed out polling batch" if Time.now > deadline
    sleep 3
  end

  assert result["status"] == "completed", "expected completed, got #{result["status"]}"
end

results << test("batch.list") do
  page = client.batch.list(1, 10)
  assert page.is_a?(Hash), "expected Hash, got #{page.class}"
  assert page["jobs"].is_a?(Array), "expected jobs array, got #{page.keys.inspect}"
end

results << test("batch.retry — no-op when no failures (expected API error handled)") do
  assert !$batch_id.nil?, "no batch_id from earlier test"
  begin
    result = client.batch.retry($batch_id)
    # If it succeeds (e.g. some items failed), that's also fine
    assert result.is_a?(Hash), "expected Hash response"
  rescue Spidra::Error => e
    # API returns an error when there are no failed items — this is expected
    assert e.message.include?("retry") || e.message.downcase.include?("failed") || e.message.downcase.include?("no"),
      "unexpected error: #{e.message}"
  end
end

results << test("batch.cancel — submit then cancel") do
  submit = client.batch.submit(
    urls:   ["https://books.toscrape.com/catalogue/page-5.html",
             "https://books.toscrape.com/catalogue/page-6.html"],
    prompt: "Extract book titles"
  )
  batch_id = submit["batchId"]
  result   = client.batch.cancel(batch_id)
  assert result.is_a?(Hash), "expected Hash response from cancel"
end

# ── CRAWL ─────────────────────────────────────────────────────────────────────
puts "\n── Crawl ───────────────────────────────────────────────────────────────"

results << test("crawl.stats") do
  stats = client.crawl.stats
  assert stats.is_a?(Hash), "expected Hash, got #{stats.class}"
  assert stats.key?("total"), "missing total key, got #{stats.keys.inspect}"
end

results << test("crawl.history") do
  history = client.crawl.history(1, 5)
  assert history.is_a?(Hash), "expected Hash, got #{history.class}"
  assert history["jobs"].is_a?(Array), "expected jobs array, got #{history.keys.inspect}"
end

results << test("crawl.run — small site (5 pages)") do
  job = client.crawl.run(
    {
      baseUrl:              "https://books.toscrape.com",
      crawlInstruction:     "Follow links to individual book pages only, skip pagination and category pages",
      transformInstruction: "Extract the book title and price",
      maxPages:             5
    },
    timeout: 360
  )
  assert job["status"] == "completed", "expected completed, got #{job["status"]}"
  assert job["jobId"].is_a?(String), "missing jobId"
  result = job["result"]
  assert result.is_a?(Array) && result.length > 0, "expected non-empty result array, got #{result.inspect}"
  $completed_crawl_job_id = job["jobId"]
end

results << test("crawl.submit + crawl.get — async flow") do
  submit = client.crawl.submit(
    baseUrl:              "https://books.toscrape.com",
    crawlInstruction:     "Follow links to individual book pages only",
    transformInstruction: "Extract the book title",
    maxPages:             3
  )
  assert submit["jobId"].is_a?(String), "missing jobId from submit"

  job_id   = submit["jobId"]
  deadline = Time.now + 360
  result   = nil

  loop do
    result = client.crawl.get(job_id)
    break if %w[completed failed].include?(result["status"])
    raise "Timed out polling crawl" if Time.now > deadline
    sleep 5
  end

  assert result["status"] == "completed", "expected completed, got #{result["status"]}"
end

results << test("crawl.pages — signed download URLs") do
  assert !$completed_crawl_job_id.nil?, "no completed crawl job from earlier"
  result = client.crawl.pages($completed_crawl_job_id)
  assert result.is_a?(Hash), "expected Hash, got #{result.class}"
  assert result["pages"].is_a?(Array), "expected pages array, got #{result.keys.inspect}"
  first = result["pages"].first
  assert !first.nil?, "pages array is empty"
  assert first.key?("url"), "page entry missing url key: #{first.keys.inspect}"
end

results << test("crawl.extract — re-extract with new prompt") do
  assert !$completed_crawl_job_id.nil?, "no completed crawl job from earlier"
  result = client.crawl.extract(
    $completed_crawl_job_id,
    "Extract only the book title as a plain string"
  )
  assert result.is_a?(Hash), "expected Hash, got #{result.class}"
  assert result["jobId"].is_a?(String), "missing jobId in extract response: #{result.inspect}"
end

# ── LOGS ──────────────────────────────────────────────────────────────────────
puts "\n── Logs ────────────────────────────────────────────────────────────────"

results << test("logs.list — no filters") do
  result = client.logs.list
  assert result.is_a?(Hash), "expected Hash, got #{result.class}"
  logs = result.dig("data", "logs")
  assert logs.is_a?(Array), "expected data.logs array, got #{result.inspect}"
  $first_log_uuid = logs.first&.dig("uuid")
end

results << test("logs.list — filter by status=success") do
  result = client.logs.list(status: "success", limit: 5)
  logs   = result.dig("data", "logs")
  assert logs.is_a?(Array), "expected data.logs array"
end

results << test("logs.list — filter by channel=api") do
  result = client.logs.list(channel: "api", limit: 5)
  logs   = result.dig("data", "logs")
  assert logs.is_a?(Array), "expected data.logs array"
end

results << test("logs.list — pagination") do
  result = client.logs.list(page: 1, limit: 3)
  logs   = result.dig("data", "logs")
  assert logs.is_a?(Array), "expected data.logs array"
  assert logs.length <= 3, "expected at most 3 results, got #{logs.length}"
end

results << test("logs.get — single log detail") do
  assert !$first_log_uuid.nil?, "no log uuid from list"
  log = client.logs.get($first_log_uuid)
  assert log.is_a?(Hash), "expected Hash, got #{log.class}"
end

# ── USAGE ─────────────────────────────────────────────────────────────────────
puts "\n── Usage ───────────────────────────────────────────────────────────────"

results << test("usage.get — 7d") do
  result = client.usage.get("7d")
  rows   = result["data"]
  assert rows.is_a?(Array), "expected data array, got #{result.class}: #{result.inspect}"
  assert rows.length > 0, "expected at least one row"
  assert rows.first.key?("date"), "row missing date key: #{rows.first.keys.inspect}"
end

results << test("usage.get — 30d") do
  result = client.usage.get("30d")
  rows   = result["data"]
  assert rows.is_a?(Array), "expected data array"
end

results << test("usage.get — weekly") do
  result = client.usage.get("weekly")
  rows   = result["data"]
  assert rows.is_a?(Array), "expected data array"
end

# ── ERRORS ────────────────────────────────────────────────────────────────────
puts "\n── Error handling ──────────────────────────────────────────────────────"

results << test("bad API key raises a Spidra::Error") do
  bad_client = Spidra.new("bad-key-xyz")
  raised = false
  begin
    bad_client.scrape.run(
      urls:   [{ url: "https://example.com" }],
      prompt: "Extract the headline"
    )
  rescue Spidra::AuthenticationError, Spidra::InsufficientCreditsError, Spidra::Error
    raised = true
  end
  assert raised, "expected a Spidra::Error to be raised"
end

results << test("error classes inherit from Spidra::Error") do
  assert Spidra::AuthenticationError    < Spidra::Error, "AuthenticationError not a subclass"
  assert Spidra::InsufficientCreditsError < Spidra::Error, "InsufficientCreditsError not a subclass"
  assert Spidra::RateLimitError         < Spidra::Error, "RateLimitError not a subclass"
  assert Spidra::ServerError            < Spidra::Error, "ServerError not a subclass"
end

results << test("Spidra::Error exposes .status and .message") do
  begin
    Spidra.new("bad").scrape.run(urls: [{ url: "https://example.com" }], prompt: "x")
  rescue Spidra::Error => e
    assert e.respond_to?(:status),  "missing .status"
    assert e.respond_to?(:message), "missing .message"
  end
end

# ── SUMMARY ───────────────────────────────────────────────────────────────────
passed = results.count(:pass)
failed = results.count(:fail)
total  = results.length

puts "\n#{"─" * 72}"
puts "  #{total} tests  |  \e[32m#{passed} passed\e[0m  |  \e[31m#{failed} failed\e[0m"
puts "─" * 72

exit(failed > 0 ? 1 : 0)