Use local httpbin to run tests

build.sc

@@ -40,7 +40,8 @@ trait RequestsModule extends CrossScalaModule with PublishModule with Mima {
   object test extends ScalaTests with TestModule.Utest {
     def ivyDeps = Agg(
       ivy"com.lihaoyi::utest::0.7.10",
-      ivy"com.lihaoyi::ujson::1.3.13"
+      ivy"com.lihaoyi::ujson::1.3.13",
+      ivy"com.dimafeng::testcontainers-scala-core:0.41.3"
     )
   }
 }

requests/test/src-2/requests/Scala2RequestTests.scala

@@ -3,15 +3,15 @@ package requests
 import utest._
 import ujson._
 
-object Scala2RequestTests extends TestSuite{
+object Scala2RequestTests extends HttpbinTestSuite {
   val tests = Tests{
 
     test("params"){
 
       test("post"){
         for(chunkedUpload <- Seq(true, false)) {
           val res1 = requests.post(
-            "https://httpbin.org/post",
+            s"http://$localHttpbin/post",
             data = Map("hello" -> "world", "foo" -> "baz"),
             chunkedUpload = chunkedUpload
           ).text()
@@ -22,7 +22,7 @@ object Scala2RequestTests extends TestSuite{
       test("put") {
         for (chunkedUpload <- Seq(true, false)) {
           val res1 = requests.put(
-            "https://httpbin.org/put",
+            s"http://$localHttpbin/put",
             data = Map("hello" -> "world", "foo" -> "baz"),
             chunkedUpload = chunkedUpload
           ).text()
@@ -31,10 +31,10 @@ object Scala2RequestTests extends TestSuite{
       }
 
       test("send"){
-        requests.send("get")("https://httpbin.org/get?hello=world&foo=baz")
+        requests.send("get")(s"http://$localHttpbin/get?hello=world&foo=baz")
 
         val res1 = requests.send("put")(
-          "https://httpbin.org/put",
+          s"http://$localHttpbin/put",
           data = Map("hello" -> "world", "foo" -> "baz"),
           chunkedUpload = true
         ).text

requests/test/src/requests/HttpbinTestSuite.scala

@@ -0,0 +1,21 @@
+package requests
+
+import com.dimafeng.testcontainers.GenericContainer
+import org.testcontainers.containers.wait.strategy.Wait
+import utest._
+
+abstract class HttpbinTestSuite extends TestSuite {
+
+  private val containerDef = GenericContainer.Def(
+    "kennethreitz/httpbin",
+    exposedPorts = Seq(80),
+    waitStrategy = Wait.forHttp("/")
+  )
+  private val container = containerDef.start()
+
+  val localHttpbin: String = s"${container.containerIpAddress}:${container.mappedPort(80)}"
+
+  override def utestAfterAll(): Unit = {
+    container.stop()
+  }
+}

requests/test/src/requests/RequestTests.scala

@@ -3,7 +3,8 @@ package requests
 import utest._
 import ujson._
 
-object RequestTests extends TestSuite{
+object RequestTests extends HttpbinTestSuite {
+
   val tests = Tests{
     test("matchingMethodWorks"){
       val requesters = Seq(
@@ -13,18 +14,18 @@ object RequestTests extends TestSuite{
         requests.put
       )
 
-      for(protocol <- Seq("http", "https")){
+      for(baseUrl <- Seq(s"http://$localHttpbin", "https://httpbin.org")){
         for(r <- requesters){
           for(r2 <- requesters){
-            val res = r(s"$protocol://httpbin.org/${r2.verb.toLowerCase()}", check = false)
+            val res = r(s"$baseUrl/${r2.verb.toLowerCase()}", check = false)
             if (r.verb == r2.verb) assert(res.statusCode == 200)
             else assert(res.statusCode == 405)
 
             if (r.verb == r2.verb){
-              val res = r(s"$protocol://httpbin.org/${r2.verb.toLowerCase()}")
+              val res = r(s"$baseUrl/${r2.verb.toLowerCase()}")
               assert(res.statusCode == 200)
             }else intercept[RequestFailedException]{
-              r(s"$protocol://httpbin.org/${r2.verb.toLowerCase()}")
+              r(s"$baseUrl/${r2.verb.toLowerCase()}")
             }
           }
         }
@@ -34,26 +35,26 @@ object RequestTests extends TestSuite{
     test("params"){
       test("get"){
         // All in URL
-        val res1 = requests.get("https://httpbin.org/get?hello=world&foo=baz").text()
+        val res1 = requests.get(s"http://$localHttpbin/get?hello=world&foo=baz").text()
         assert(read(res1).obj("args") == Obj("foo" -> "baz", "hello" -> "world"))
 
         // All in params
         val res2 = requests.get(
-          "https://httpbin.org/get",
+          s"http://$localHttpbin/get",
           params = Map("hello" -> "world", "foo" -> "baz")
         )
         assert(read(res2).obj("args") == Obj("foo" -> "baz", "hello" -> "world"))
 
         // Mixed URL and params
         val res3 = requests.get(
-          "https://httpbin.org/get?hello=world",
+          s"http://$localHttpbin/get?hello=world",
           params = Map("foo" -> "baz")
         ).text()
         assert(read(res3).obj("args") == Obj("foo" -> "baz", "hello" -> "world"))
 
         // Needs escaping
         val res4 = requests.get(
-          "https://httpbin.org/get?hello=world",
+          s"http://$localHttpbin/get?hello=world",
           params = Map("++-- lol" -> " !@#$%")
         )
         assert(read(res4).obj("args") == Obj("++-- lol" -> " !@#$%", "hello" -> "world"))
@@ -63,7 +64,7 @@ object RequestTests extends TestSuite{
     test("multipart"){
       for(chunkedUpload <- Seq(true, false)) {
         val response = requests.post(
-          "http://httpbin.org/post",
+          s"http://$localHttpbin/post",
           data = MultiPart(
             MultiItem("file1", "Hello!".getBytes, "foo.txt"),
             MultiItem("file2", "Goodbye!")
@@ -79,76 +80,77 @@ object RequestTests extends TestSuite{
     test("cookies"){
       test("session"){
         val s = requests.Session(cookieValues = Map("hello" -> "world"))
-        val res1 = s.get("https://httpbin.org/cookies").text().trim
+        val res1 = s.get(s"http://$localHttpbin/cookies").text().trim
         assert(read(res1) == Obj("cookies" -> Obj("hello" -> "world")))
-        s.get("https://httpbin.org/cookies/set?freeform=test")
-        val res2 = s.get("https://httpbin.org/cookies").text().trim
+        s.get(s"http://$localHttpbin/cookies/set?freeform=test")
+        val res2 = s.get(s"http://$localHttpbin/cookies").text().trim
         assert(read(res2) == Obj("cookies" -> Obj("freeform" -> "test", "hello" -> "world")))
       }
       test("raw"){
-        val res1 = requests.get("https://httpbin.org/cookies").text().trim
+        val res1 = requests.get(s"http://$localHttpbin/cookies").text().trim
         assert(read(res1) == Obj("cookies" -> Obj()))
-        requests.get("https://httpbin.org/cookies/set?freeform=test")
-        val res2 = requests.get("https://httpbin.org/cookies").text().trim
+        requests.get(s"http://$localHttpbin/cookies/set?freeform=test")
+        val res2 = requests.get(s"http://$localHttpbin/cookies").text().trim
         assert(read(res2) == Obj("cookies" -> Obj()))
       }
       test("space"){
         val s = requests.Session(cookieValues = Map("hello" -> "hello, world"))
-        val res1 = s.get("https://httpbin.org/cookies").text().trim
+        val res1 = s.get(s"http://$localHttpbin/cookies").text().trim
         assert(read(res1) == Obj("cookies" -> Obj("hello" -> "hello, world")))
-        s.get("https://httpbin.org/cookies/set?freeform=test+test")
-        val res2 = s.get("https://httpbin.org/cookies").text().trim
+        s.get(s"http://$localHttpbin/cookies/set?freeform=test+test")
+        val res2 = s.get(s"http://$localHttpbin/cookies").text().trim
         assert(read(res2) == Obj("cookies" -> Obj("freeform" -> "test test", "hello" -> "hello, world")))
       }
     }
 
     test("redirects"){
       test("max"){
-        val res1 = requests.get("https://httpbin.org/absolute-redirect/4")
+        val res1 = requests.get(s"http://$localHttpbin/absolute-redirect/4")
         assert(res1.statusCode == 200)
-        val res2 = requests.get("https://httpbin.org/absolute-redirect/5")
+        val res2 = requests.get(s"http://$localHttpbin/absolute-redirect/5")
         assert(res2.statusCode == 200)
-        val res3 = requests.get("https://httpbin.org/absolute-redirect/6", check = false)
+        val res3 = requests.get(s"http://$localHttpbin/absolute-redirect/6", check = false)
         assert(res3.statusCode == 302)
-        val res4 = requests.get("https://httpbin.org/absolute-redirect/6", maxRedirects = 10)
+        val res4 = requests.get(s"http://$localHttpbin/absolute-redirect/6", maxRedirects = 10)
         assert(res4.statusCode == 200)
       }
       test("maxRelative"){
-        val res1 = requests.get("https://httpbin.org/relative-redirect/4")
+        val res1 = requests.get(s"http://$localHttpbin/relative-redirect/4")
         assert(res1.statusCode == 200)
-        val res2 = requests.get("https://httpbin.org/relative-redirect/5")
+        val res2 = requests.get(s"http://$localHttpbin/relative-redirect/5")
         assert(res2.statusCode == 200)
-        val res3 = requests.get("https://httpbin.org/relative-redirect/6", check = false)
+        val res3 = requests.get(s"http://$localHttpbin/relative-redirect/6", check = false)
         assert(res3.statusCode == 302)
-        val res4 = requests.get("https://httpbin.org/relative-redirect/6", maxRedirects = 10)
+        val res4 = requests.get(s"http://$localHttpbin/relative-redirect/6", maxRedirects = 10)
         assert(res4.statusCode == 200)
       }
     }
 
     test("test_reproduction"){
-      requests.get("http://httpbin.org/status/304").text()
+      requests.get(s"http://$localHttpbin/status/304").text()
 
     }
     test("streaming"){
-      val res1 = requests.get("http://httpbin.org/stream/5").text()
+      val res1 = requests.get(s"http://$localHttpbin/stream/5").text()
       assert(res1.linesIterator.length == 5)
-      val res2 = requests.get("http://httpbin.org/stream/52").text()
+      val res2 = requests.get(s"http://$localHttpbin/stream/52").text()
       assert(res2.linesIterator.length == 52)
     }
 
     test("timeouts"){
       test("read"){
         intercept[TimeoutException] {
-          requests.get("https://httpbin.org/delay/1", readTimeout = 10)
+          requests.get(s"http://$localHttpbin/delay/1", readTimeout = 10)
         }
-        requests.get("https://httpbin.org/delay/1", readTimeout = 2000)
+        requests.get(s"http://$localHttpbin/delay/1", readTimeout = 2000)
         intercept[TimeoutException] {
-          requests.get("https://httpbin.org/delay/3", readTimeout = 2000)
+          requests.get(s"http://$localHttpbin/delay/3", readTimeout = 2000)
         }
       }
       test("connect"){
         intercept[TimeoutException] {
-          requests.get("https://httpbin.org/delay/1", connectTimeout = 1)
+          // use remote httpbin.org so it needs more time to connect
+          requests.get(s"https://httpbin.org/delay/1", connectTimeout = 1)
         }
       }
     }
@@ -167,38 +169,38 @@ object RequestTests extends TestSuite{
     }
 
     test("decompress"){
-      val res1 = requests.get("https://httpbin.org/gzip")
-      assert(read(res1.text()).obj("headers").obj("Host").str == "httpbin.org")
+      val res1 = requests.get(s"http://$localHttpbin/gzip")
+      assert(read(res1.text()).obj("headers").obj("Host").str == localHttpbin)
 
-      val res2 = requests.get("https://httpbin.org/deflate")
-      assert(read(res2).obj("headers").obj("Host").str == "httpbin.org")
+      val res2 = requests.get(s"http://$localHttpbin/deflate")
+      assert(read(res2).obj("headers").obj("Host").str == localHttpbin)
 
-      val res3 = requests.get("https://httpbin.org/gzip", autoDecompress = false)
+      val res3 = requests.get(s"http://$localHttpbin/gzip", autoDecompress = false)
       assert(res3.bytes.length < res1.bytes.length)
 
-      val res4 = requests.get("https://httpbin.org/deflate", autoDecompress = false)
+      val res4 = requests.get(s"http://$localHttpbin/deflate", autoDecompress = false)
       assert(res4.bytes.length < res2.bytes.length)
 
       (res1.bytes.length, res2.bytes.length, res3.bytes.length, res4.bytes.length)
     }
 
     test("compression"){
       val res1 = requests.post(
-        "https://httpbin.org/post",
+        s"http://$localHttpbin/post",
         compress = requests.Compress.None,
         data = new RequestBlob.ByteSourceRequestBlob("Hello World")
       )
       assert(res1.text().contains(""""Hello World""""))
 
       val res2 = requests.post(
-        "https://httpbin.org/post",
+        s"http://$localHttpbin/post",
         compress = requests.Compress.Gzip,
         data = new RequestBlob.ByteSourceRequestBlob("I am cow")
       )
       assert(read(new String(res2.bytes))("data").toString.contains("data:application/octet-stream;base64,H4sIAAAAAA"))
 
       val res3 = requests.post(
-        "https://httpbin.org/post",
+        s"http://$localHttpbin/post",
         compress = requests.Compress.Deflate,
         data = new RequestBlob.ByteSourceRequestBlob("Hear me moo")
       )
@@ -208,7 +210,7 @@ object RequestTests extends TestSuite{
 
     test("headers"){
       test("default"){
-        val res = requests.get("https://httpbin.org/headers").text()
+        val res = requests.get(s"http://$localHttpbin/headers").text()
         val hs = read(res)("headers").obj
         assert(hs("User-Agent").str == "requests-scala")
         assert(hs("Accept-Encoding").str == "gzip, deflate")
@@ -306,7 +308,7 @@ object RequestTests extends TestSuite{
     // to the server. This preserves the 0.8.x behavior, and can always be overriden
     // by passing a comma-separated list of headers instead
     test("duplicateHeaders"){
-      val res = requests.get("https://httpbin.org/get", headers = Seq("x-y" -> "a", "x-y" -> "b"))
+      val res = requests.get(s"http://$localHttpbin/get", headers = Seq("x-y" -> "a", "x-y" -> "b"))
       assert(ujson.read(res)("headers")("X-Y") == Str("b")) // make sure it's not "a,b"
     }
   }