From 6e301e9e57458ba511a202a77d270249596796c3 Mon Sep 17 00:00:00 2001 From: Matthew Davidson Date: Wed, 14 Sep 2022 17:55:40 +0800 Subject: [PATCH 01/13] Initial addition of clj-http tests for comparison --- project.clj | 16 +- test-resources/big_array_json.json | 102 ++ test-resources/keystore | Bin 0 -> 3463 bytes test/aleph/http/clj_http/client_test.clj | 1795 ++++++++++++++++++++++ test/aleph/http/clj_http/core_test.clj | 1003 ++++++++++++ test/aleph/http/clj_http/util.clj | 87 ++ 6 files changed, 2999 insertions(+), 4 deletions(-) create mode 100644 test-resources/big_array_json.json create mode 100644 test-resources/keystore create mode 100644 test/aleph/http/clj_http/client_test.clj create mode 100644 test/aleph/http/clj_http/core_test.clj create mode 100644 test/aleph/http/clj_http/util.clj diff --git a/project.clj b/project.clj index 31a1ae80..7e96f6b4 100644 --- a/project.clj +++ b/project.clj @@ -21,7 +21,7 @@ [io.netty/netty-handler-proxy ~netty-version] [io.netty/netty-resolver ~netty-version] [io.netty/netty-resolver-dns ~netty-version]] - :profiles {:dev {:dependencies [[org.clojure/clojure "1.10.3"] + :profiles {:dev {:dependencies [[org.clojure/clojure "1.11.1"] [criterium "0.4.6"] [cheshire "5.10.0"] [org.slf4j/slf4j-simple "1.7.30"] @@ -31,7 +31,15 @@ :lein-to-deps {:source-paths ["deps"]} ;; This is for self-generating certs for testing ONLY: :test {:dependencies [[org.bouncycastle/bcprov-jdk15on "1.69"] - [org.bouncycastle/bcpkix-jdk15on "1.69"]] + [org.bouncycastle/bcpkix-jdk15on "1.69"] + + ;; for testing clj-http parity + [clj-http "3.12.3"] + [ring/ring-jetty-adapter "1.9.3"] + [org.apache.logging.log4j/log4j-api "2.17.1"] + [org.apache.logging.log4j/log4j-core "2.17.1"] + [org.apache.logging.log4j/log4j-1.2-api "2.17.1"]] + :javac-options ^:replace ["--release" "12"] ; necessary for some tests :jvm-opts ["-Dorg.slf4j.simpleLogger.defaultLogLevel=off"]}} :codox {:src-dir-uri "https://github.com/ztellman/aleph/tree/master/" :src-linenum-anchor-prefix "L" @@ -42,16 +50,16 @@ aleph.flow] :output-dir "doc"} :plugins [[lein-codox "0.10.7"] - [lein-jammin "0.1.1"] [lein-marginalia "0.9.1"] [lein-pprint "1.3.2"] [ztellman/lein-cljfmt "0.1.10"]] :java-source-paths ["src/aleph/utils"] :cljfmt {:indents {#".*" [[:inner 0]]}} :test-selectors {:default #(not - (some #{:benchmark :stress} + (some #{:benchmark :stress :integration} (cons (:tag %) (keys %)))) :benchmark :benchmark + :integration :integration :stress :stress :all (constantly true)} :jvm-opts ^:replace ["-server" diff --git a/test-resources/big_array_json.json b/test-resources/big_array_json.json new file mode 100644 index 00000000..51ccef7d --- /dev/null +++ b/test-resources/big_array_json.json @@ -0,0 +1,102 @@ +[ + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]} +] diff --git a/test-resources/keystore b/test-resources/keystore new file mode 100644 index 0000000000000000000000000000000000000000..2944ca21c6d5bbbe54c991353ed18cedb237520b GIT binary patch literal 3463 zcmY+GcQhLg*T)k{#7xcD)LuoR6s1;;XpPpUR?JcqX@zPlqEv0QS8CLzN>O_?trb*l zQVFe9d)2Q-`}TRy`##Tk{;|Ca@^2jd~ue`WK(VM@#Re^X2hK$-$P_!NN$A0wn`8UA1XyPOZq z@kqRFxYZrD2&YFr&!3C`t*3QYmj(p*a0`qF_Z#Lo>&X=f*E5u426wW>rsbncj*m1e z)7`j-JK}rbiGjn!jR*Pxbo|X{#q&!rPyLYUt&|rbbT?s7E>#&h*0FXK)V_}Q<|q@x z7UZ}*nZdBW_9czpy<7NLby3?LGDS98P<7lHldqiC^nzOPw=*3b8iyBa-m%5RcbCj8 zdOc=w0;*q6VBt)Iz%;lod?IQ1YNY%aL27xOmc}3V%Me+C zGh(_Y!oU9!MhpUDA7b{vG7%Zk6ez5F0-GSP2kMpQCGl^bcLNKZ$zCwv-JRQYumTcY5AF3;Up~^9qfGKO%sK zS*)ci>nV%saSUG_Bo9>Qq;q*sg?KTTp)QfBf-=dl-XLe^VF=;4{49SPYw6@sq3am- z;OZKr&z^UwM*VgIa!_OEpS286!}fL+1)H+e?sbFH>b_Ua4cgZnKOO2}MVjcXd^l_< z(~|M^!p}E|+mlTR3Xc{=dYxF^OfblYQeiIN7iIdiUUn@Utc`Xb-!%CntTI6Q`9LvT z+G8sL+oW)V=3UEHb?iR5aKGA^{%|@&m!d;fX8L}PzTjA|DYx-6j1)~fdrYJf{$HYxl zS|jenvq8th13#zr81`GZe|{WNuzJ_;+tXPOaxV5S+TZf8qN_0chlWHj{#q+?+b^%K z-q)~qH`+^LXr%;e$@7?W>sv?pPi~ak;Xib~()S1jg=^9Cnl3fdKEOmSrxcB^H<&MK zDpd?E(3goDJELzBOjez?5`kJW`HxDcts3Iu%FJf;8{~zW6z8_?hL-TyHS>=X#p7+4 zgE`)Rlx?a_T4LPFJqiO`9HhuQ;cm7zatLH#7&Gzs&aG|8#l+V0bFKHa{DOHCpyXV7 z0>-{;g+y39(7|9y2RPZHKcfwWiO0$PTT0kMboY?)rvtQ(;!_s zdcSk*klF~of1{F~0|^sk0$>0hfFMBdzcNDjKPD%V1I%XW?dNe3sjQ}~tfGumQc_h! zBJj|Le>XvC3-HjAzfvI$5b(F;{Ko0}nBIR}BA` zusA%_#jDCid`Z}7+JsuiVGss>NNWCZ>D%e>G611sf~a=-^z3a7HLPKKcO6&C%Vpk3 zia8SUu3aOL3J8X!tsSIs`#Nt~ouR*4){`*N1Op9f<~#S0bh{ag{A3p8K~dDKZ0HAJ6r;_S#LE zU%YMn-ibb9!lYDGEgJILDE9WZKwrM6WLYd7Ntx@%Qq2EUTh}6iAsOf<+k=}O z4&&){To}0GqT7S4msdM}g(~;XE2Lb>5}YghogetKuk^xwmcSJL^EW!mqf((|oS%iS z2n46b!eW(fiOIXEbg+Gow9CqNpmPltH(i~Ix|TMif61@-sgrYyT7#tN*%9epxdhIY zWdwKwWd%u<%dba~lU2bJ8y}Y7zV0(o^$v^IRSbtRDQq&~j0f&g0Vf7`msqUn09+(@ zKrcXA8a`&+7CuORIU5pB>#%JYnM&yq5lUYTloby8dmRIlD*?W)hjHG&B!xW}-bf9v zVhOnw7@^g@ys$6Z@sC$1-|V}{k*z{6MZFJ0Z|O5>o9=naJ#zyhYh2B82JL#E(TK~`ek{37XukxalYg|_j!(ecz+>7UkNEsR09g$ zHpA$hM9vhmh1*o+{?iykB6cfK{rIvmiu_hyC6@$=8@A+}P1S6>XXPp?%ALe(r6A&g z>SIf>&5sj>Fw3sy=L3?HhM?lp@Yh#8OuK~D(N}qdDs!&5b6OGZ!$k+^ygm-1YULuA z-A#U{YPyH#l$VqqQ9|{?+SdJ&PMn2LMu+VxGt9#cA8i(@3t8ts@xi z^F^o4HNdjn_Gy@$1;pZp?!BT1tcp)5(fH`8UoA{RdTT4*BBO$^D3M z$~>zttqWy+b59h!KPM$!ariieHtutXL*!i?pMskspHtUqccuN3VWCcd@%kf>SF|hVjEM! zfcw*pQ4maO|K@|;6Wvo_ZZBTIkM$nf+0})WU&N$rJEhJbM-aTTWW_{mEretE{YGgh zdI4F54>^p;BDL_#Ip~+2n+5G@X|AnW3UU5LKg=pQK4dNE!F9jD78jG-U5G{m%-%&S z0^^p;y8s0zJ(^6<#Uw7+z30JefV#FmW>YmSl9OglYf+cCg`lJwO(v(^pacuGNw;gW{60v9sVd^~n(7hKyNsy7HTX za&N44Xgnh?&zPT3g@9wLj~7?*;1#U-p88(j8}B?=E`&Oh@Wnh^Uw47*tZC zQDRhLDS49e>*Fj?swG(Xvg)TUlf|+UgKywn5Bb1}@}IWV#psP?(C7s{^;icb^1lpX(( zZYQDKogj31i38e|)SX#34u3Dx#uA_!Owai>$|%OCaeekma7CSKWtOeyfQNmYNmbe6Oyy8hEti6_sVoh-CwD2IAS4_hYkD zgpPY7GBFDVvFvnwsfRjxu|FQ?q^hYxqoqN##9d>n;^olq_#vtD*$85^?@}5oI4*#C zpmB*``P|!7&GEA;#Q2ofBmKL%BMn)$r;KxBAnq;M?3Dj=kd~-=4TVRpp*{kW7QdcHeS2PPvWOLiq8w)WLpzE{8sQ(s gfjP3o?_3rkb37XPZF+}}HJFbog?dW^0!A1759c;}7XSbN literal 0 HcmV?d00001 diff --git a/test/aleph/http/clj_http/client_test.clj b/test/aleph/http/clj_http/client_test.clj new file mode 100644 index 00000000..ba5c33ff --- /dev/null +++ b/test/aleph/http/clj_http/client_test.clj @@ -0,0 +1,1795 @@ +(ns aleph.http.clj-http.client-test + (:require [aleph.http.clj-http.core-test :refer [run-server]] + [aleph.http.clj-http.util :refer [request]] + [cheshire.core :as json] + [clj-http.client :as client] + [clj-http.conn-mgr :as conn] + [clj-http.util :as util] + [clojure.java.io :refer [resource]] + [clojure.string :as str] + [clojure.test :refer :all] + [cognitect.transit :as transit] + [ring.middleware.nested-params :refer [parse-nested-keys]] + [ring.util.codec :refer [form-decode-str]] + [slingshot.slingshot :refer [try+]]) + (:import java.io.ByteArrayInputStream + java.io.PipedInputStream + java.io.PipedOutputStream + java.net.UnknownHostException + org.apache.http.HttpEntity + org.apache.logging.log4j.LogManager)) + +(set! *warn-on-reflection* false) + +(defonce logger (LogManager/getLogger "clj-http.test.client-test")) + +(def base-req + {:scheme :http + :server-name "localhost" + :server-port 18080}) + +#_ +(defn request + ([req] + (client/request (merge base-req req))) + ([req respond raise] + (client/request (merge base-req req) respond raise))) + +(defn parse-form-params [s] + (->> (str/split (form-decode-str s) #"&") + (map #(str/split % #"=")) + (map #(vector + (map keyword (parse-nested-keys (first %))) + (second %))) + (reduce (fn [m [ks v]] + (assoc-in m ks v)) {}))) + +(deftest ^:integration roundtrip + (run-server) + ;; roundtrip with scheme as a keyword + (let [resp (request {:uri "/get" :method :get})] + (is (= 200 (:status resp))) + (is (= "close" (get-in resp [:headers "connection"]))) + (is (= "get" (:body resp)))) + ;; roundtrip with scheme as a string + (let [resp (request {:uri "/get" :method :get + :scheme "http"})] + (is (= 200 (:status resp))) + (is (= "close" (get-in resp [:headers "connection"]))) + (is (= "get" (:body resp)))) + (let [params {:a "1" :b "2"}] + (doseq [[content-type read-fn] + [[nil (comp parse-form-params slurp)] + [:x-www-form-urlencoded (comp parse-form-params slurp)] + [:edn (comp read-string slurp)] + [:transit+json #(client/parse-transit % :json)] + [:transit+msgpack #(client/parse-transit % :msgpack)]]] + (let [resp (request {:uri "/post" + :as :stream + :method :post + :content-type content-type + :form-params params})] + (is (= 200 (:status resp))) + (is (= "close" (get-in resp [:headers "connection"]))) + (is (= params (read-fn (:body resp))) + (str "failed with content-type [" content-type "]")))))) + +(deftest ^:integration roundtrip-async + (run-server) + ;; roundtrip with scheme as a keyword + (let [resp (promise) + exception (promise) + _ (request {:uri "/get" :method :get + :async? true} resp exception)] + (is (= 200 (:status @resp))) + (is (= "close" (get-in @resp [:headers "connection"]))) + (is (= "get" (:body @resp))) + (is (not (realized? exception)))) + ;; roundtrip with scheme as a string + (let [resp (promise) + exception (promise) + _ (request {:uri "/get" :method :get + :scheme "http" + :async? true} resp exception)] + (is (= 200 (:status @resp))) + (is (= "close" (get-in @resp [:headers "connection"]))) + (is (= "get" (:body @resp))) + (is (not (realized? exception)))) + + (let [params {:a "1" :b "2"}] + (doseq [[content-type read-fn] + [[nil (comp parse-form-params slurp)] + [:x-www-form-urlencoded (comp parse-form-params slurp)] + [:edn (comp read-string slurp)] + [:transit+json #(client/parse-transit % :json)] + [:transit+msgpack #(client/parse-transit % :msgpack)]]] + (let [resp (promise) + exception (promise) + _ (request {:uri "/post" + :as :stream + :method :post + :content-type content-type + :flatten-nested-keys [] + :form-params params + :async? true} resp exception)] + (is (= 200 (:status @resp))) + (is (= "close" (get-in @resp [:headers "connection"]))) + (is (= params (read-fn (:body @resp)))) + (is (not (realized? exception))))))) + +(def ^:dynamic *test-dynamic-var* nil) + +(deftest ^:integration async-preserves-dynamic-variable-bindings + (run-server) + (let [expected-var "cat"] + (binding [*test-dynamic-var* expected-var] + (let [test-fn (fn [uri success-p fail-p] + (request {:uri uri + :method :get + :scheme "http" + :async? true} + (fn [_] + (deliver success-p *test-dynamic-var*) + (deliver fail-p :success)) + (fn [_] + (deliver success-p :fail) + (deliver fail-p *test-dynamic-var*))))] + (testing "dynamic variables on success responses" + (let [success-p (promise) + fail-p (promise)] + (test-fn "/get" success-p fail-p) + (is (= @success-p expected-var *test-dynamic-var*)) + (is (= @fail-p :success) + "Verify that we went through the success path, not the failure"))) + + (testing "dynamic variables on fail responses" + (let [success-p (promise) + fail-p (promise)] + (test-fn "/json-bad" success-p fail-p) + (is (= @success-p :fail) + "Verify that we went through the failure path, not the success") + (is (= @fail-p expected-var *test-dynamic-var*)))))))) + +(deftest ^:integration multipart-async + (run-server) + (let [resp (promise) + exception (promise) + _ (request {:uri "/post" :method :post + :async? true + :multipart [{:name "title" :content "some-file"} + {:name "Content/Type" :content "text/plain"} + {:name "file" + :content (clojure.java.io/file + "test-resources/m.txt")}]} + resp + exception + )] + (is (= 200 (:status @resp))) + (is (not (realized? exception))) + #_(when (realized? exception) (prn @exception))) + + ;; Regression Testing https://github.com/dakrone/clj-http/issues/560 + (testing "multipart uploads larger than 25kb" + (let [resp (promise) + exception (promise) + ;; assumption: file > 5kb + file (clojure.java.io/file "test-resources/big_array_json.json") + + _ (request {:uri "/post" :method :post + :async? true + :multipart [{:name "part-1" :content file} + {:name "part-2" :content file} + {:name "part-3" :content file} + {:name "part-4" :content file} + {:name "part-5" :content file}]} + resp + exception)] + (is (= 200 (:status (deref resp 500 :failed)))) + (is (not (realized? exception)))))) + +(deftest ^:integration nil-input + (is (thrown-with-msg? Exception #"Host URL cannot be nil" + (client/get nil))) + (is (thrown-with-msg? Exception #"Host URL cannot be nil" + (client/post nil))) + (is (thrown-with-msg? Exception #"Host URL cannot be nil" + (client/put nil))) + (is (thrown-with-msg? Exception #"Host URL cannot be nil" + (client/delete nil)))) + +(defn async-identity-client + "A async client which simply respond the request" + [request respond raise] + (respond request)) + +(defn is-passed [middleware req] + (let [client (middleware identity)] + (is (= req (client req))))) + +(defn is-passed-async [middleware req] + (let [client (middleware async-identity-client) + resp (promise) + exception (promise) + _ (client req resp exception)] + (is (= req @resp)) + (is (not (realized? exception))))) + +(defn is-applied [middleware req-in req-out] + (let [client (middleware identity)] + (is (= req-out (client req-in))))) + +(defn is-applied-async [middleware req-in req-out] + (let [client (middleware async-identity-client) + resp (promise) + exception (promise) + _ (client req-in resp exception)] + (is (= req-out @resp)) + (is (not (realized? exception))))) + +(deftest redirect-on-get + (let [client (fn [req] + (if (= "example.com" (:server-name req)) + {:status 302 + :headers {"location" "http://example.net/bat"}} + {:status 200 + :req req})) + r-client (-> client client/wrap-url client/wrap-redirects) + resp (r-client {:server-name "example.com" :url "http://example.com" + :request-method :get})] + (is (= 200 (:status resp))) + (is (= :get (:request-method (:req resp)))) + (is (= :http (:scheme (:req resp)))) + (is (= ["http://example.com" "http://example.net/bat"] + (:trace-redirects resp))) + (is (= "/bat" (:uri (:req resp)))))) + +(deftest redirect-on-get-async + (let [client (fn [req respond raise] + (respond (if (= "example.com" (:server-name req)) + {:status 302 + :headers {"location" "http://example.net/bat"}} + {:status 200 + :req req}))) + r-client (-> client client/wrap-url client/wrap-redirects) + resp (promise) + exception (promise) + _ (r-client {:server-name "example.com" :url "http://example.com" + :request-method :get} resp exception)] + (is (= 200 (:status @resp))) + (is (= :get (:request-method (:req @resp)))) + (is (= :http (:scheme (:req @resp)))) + (is (= ["http://example.com" "http://example.net/bat"] + (:trace-redirects @resp))) + (is (= "/bat" (:uri (:req @resp)))) + (is (not (realized? exception))))) + +(deftest relative-redirect-on-get + (let [client (fn [req] + (if (:redirects-count req) + {:status 200 + :req req} + {:status 302 + :headers {"location" "/bat"}})) + r-client (-> client client/wrap-url client/wrap-redirects) + resp (r-client {:server-name "example.com" :url "http://example.com" + :request-method :get})] + (is (= 200 (:status resp))) + (is (= :get (:request-method (:req resp)))) + (is (= :http (:scheme (:req resp)))) + (is (= ["http://example.com" "http://example.com/bat"] + (:trace-redirects resp))) + (is (= "/bat" (:uri (:req resp)))))) + +(deftest relative-redirect-on-get-async + (let [client (fn [req respond raise] + (respond (if (:redirects-count req) + {:status 200 + :req req} + {:status 302 + :headers {"location" "/bat"}}))) + r-client (-> client client/wrap-url client/wrap-redirects) + resp (promise) + exception (promise) + _ (r-client {:server-name "example.com" :url "http://example.com" + :request-method :get} resp exception)] + (is (= 200 (:status @resp))) + (is (= :get (:request-method (:req @resp)))) + (is (= :http (:scheme (:req @resp)))) + (is (= ["http://example.com" "http://example.com/bat"] + (:trace-redirects @resp))) + (is (= "/bat" (:uri (:req @resp)))) + (is (not (realized? exception))))) + +(deftest trace-redirects-using-uri + (let [client (fn [req] {:status 200 :req req}) + r-client (-> client client/wrap-redirects) + resp (r-client {:scheme :http :server-name "example.com" :uri "/" + :request-method :get})] + (is (= 200 (:status resp))) + (is (= :get (:request-method (:req resp)))) + (is (= :http (:scheme (:req resp)))) + (is (= [] (:trace-redirects resp))))) + +(deftest trace-redirects-using-uri-async + (let [client (fn [req respond raise] (respond {:status 200 :req req})) + r-client (-> client client/wrap-redirects) + resp (promise) + exception (promise) + _ (r-client {:scheme :http :server-name "example.com" :uri "/" + :request-method :get} resp exception)] + (is (= 200 (:status @resp))) + (is (= :get (:request-method (:req @resp)))) + (is (= :http (:scheme (:req @resp)))) + (is (= [] (:trace-redirects @resp))) + (is (not (realized? exception))))) + +(deftest redirect-without-location-header + (let [client (fn [req] + {:status 302 :body "no redirection here"}) + r-client (-> client client/wrap-url client/wrap-redirects) + resp (r-client {:server-name "example.com" :url "http://example.com" + :request-method :get})] + (is (= 302 (:status resp))) + (is (= ["http://example.com"] (:trace-redirects resp))) + (is (= "no redirection here" (:body resp))))) + +(deftest redirect-without-location-header-async + (let [client (fn [req respond raise] + (respond {:status 302 :body "no redirection here"})) + r-client (-> client client/wrap-url client/wrap-redirects) + resp (promise) + exception (promise) + _ (r-client {:server-name "example.com" :url "http://example.com" + :request-method :get} resp exception)] + (is (= 302 (:status @resp))) + (is (= ["http://example.com"] (:trace-redirects @resp))) + (is (= "no redirection here" (:body @resp))) + (is (not (realized? exception))))) + +(deftest redirect-with-query-string + (let [client (fn [req] + (if (= "example.com" (:server-name req)) + {:status 302 + :headers {"location" "http://example.net/bat?x=y"}} + {:status 200 + :req req})) + r-client (-> client client/wrap-url client/wrap-redirects) + resp (r-client {:server-name "example.com" :url "http://example.com" + :request-method :get :query-params {:x "z"}})] + (is (= 200 (:status resp))) + (is (= :get (:request-method (:req resp)))) + (is (= :http (:scheme (:req resp)))) + (is (= ["http://example.com" "http://example.net/bat?x=y"] + (:trace-redirects resp))) + (is (= "/bat" (:uri (:req resp)))) + (is (= "x=y" (:query-string (:req resp)))) + (is (nil? (:query-params (:req resp)))))) + +(deftest redirect-with-query-string-async + (let [client (fn [req respond raise] + (respond (if (= "example.com" (:server-name req)) + {:status 302 + :headers {"location" "http://example.net/bat?x=y"}} + {:status 200 + :req req}))) + r-client (-> client client/wrap-url client/wrap-redirects) + resp (promise) + exception (promise) + _ (r-client {:server-name "example.com" :url "http://example.com" + :request-method :get :query-params {:x "z"}} + resp exception)] + (is (= 200 (:status @resp))) + (is (= :get (:request-method (:req @resp)))) + (is (= :http (:scheme (:req @resp)))) + (is (= ["http://example.com" "http://example.net/bat?x=y"] + (:trace-redirects @resp))) + (is (= "/bat" (:uri (:req @resp)))) + (is (= "x=y" (:query-string (:req @resp)))) + (is (nil? (:query-params (:req @resp)))) + (is (not (realized? exception))))) + +(deftest max-redirects + (let [client (fn [req] + (if (= "example.com" (:server-name req)) + {:status 302 + :headers {"location" "http://example.net/bat"}} + {:status 200 + :req req})) + r-client (-> client client/wrap-url client/wrap-redirects) + resp (r-client {:server-name "example.com" :url "http://example.com" + :request-method :get :max-redirects 0})] + (is (= 302 (:status resp))) + (is (= ["http://example.com"] (:trace-redirects resp))) + (is (= "http://example.net/bat" (get (:headers resp) "location"))))) + +(deftest max-redirects-async + (let [client (fn [req respond raise] + (respond (if (= "example.com" (:server-name req)) + {:status 302 + :headers {"location" "http://example.net/bat"}} + {:status 200 + :req req}))) + r-client (-> client client/wrap-url client/wrap-redirects) + resp (promise) + exception (promise) + _ (r-client {:server-name "example.com" :url "http://example.com" + :request-method :get :max-redirects 0} + resp exception)] + (is (= 302 (:status @resp))) + (is (= ["http://example.com"] (:trace-redirects @resp))) + (is (= "http://example.net/bat" (get (:headers @resp) "location"))) + (is (not (realized? exception))))) + +(deftest redirect-303-to-get-on-any-method + (doseq [method [:get :head :post :delete :put :option]] + (let [client (fn [req] + (if (= "example.com" (:server-name req)) + {:status 303 + :headers {"location" "http://example.net/bat"}} + {:status 200 + :req req})) + r-client (-> client client/wrap-url client/wrap-redirects) + resp (r-client {:server-name "example.com" :url "http://example.com" + :request-method method})] + (is (= 200 (:status resp))) + (is (= :get (:request-method (:req resp)))) + (is (= :http (:scheme (:req resp)))) + (is (= ["http://example.com" "http://example.net/bat"] + (:trace-redirects resp))) + (is (= "/bat" (:uri (:req resp))))))) + +(deftest redirect-303-to-get-on-any-method-async + (doseq [method [:get :head :post :delete :put :option]] + (let [client (fn [req respond raise] + (respond (if (= "example.com" (:server-name req)) + {:status 303 + :headers {"location" "http://example.net/bat"}} + {:status 200 + :req req}))) + r-client (-> client client/wrap-url client/wrap-redirects) + resp (promise) + exception (promise) + _ (r-client {:server-name "example.com" :url "http://example.com" + :request-method method} + resp exception)] + (is (= 200 (:status @resp))) + (is (= :get (:request-method (:req @resp)))) + (is (= :http (:scheme (:req @resp)))) + (is (= ["http://example.com" "http://example.net/bat"] + (:trace-redirects @resp))) + (is (= "/bat" (:uri (:req @resp)))) + (is (not (realized? exception)))))) + +(deftest pass-on-non-redirect + (let [client (fn [req] {:status 200 :body (:body req)}) + r-client (client/wrap-redirects client) + resp (r-client {:body "ok" :url "http://example.com"})] + (is (= 200 (:status resp))) + (is (= ["http://example.com"] (:trace-redirects resp))) + (is (= "ok" (:body resp))))) + +(deftest pass-on-non-redirect-async + (let [client (fn [req respond raise] + (respond {:status 200 :body (:body req)})) + r-client (client/wrap-redirects client) + resp (promise) + exception (promise) + _ (r-client {:body "ok" :url "http://example.com"} resp exception)] + (is (= 200 (:status @resp))) + (is (= ["http://example.com"] (:trace-redirects @resp))) + (is (= "ok" (:body @resp))) + (is (not (realized? exception))))) + +(deftest pass-on-non-redirectable-methods + (doseq [method [:put :post :delete] + status [301 302 307 308]] + (let [client (fn [req] {:status status :body (:body req) + :headers {"location" "http://example.com/bat"}}) + r-client (client/wrap-redirects client) + resp (r-client {:body "ok" :url "http://example.com" + :request-method method})] + (is (= status (:status resp))) + (is (= ["http://example.com"] (:trace-redirects resp))) + (is (= {"location" "http://example.com/bat"} (:headers resp))) + (is (= "ok" (:body resp)))))) + +(deftest pass-on-non-redirectable-methods-async + (doseq [method [:put :post :delete] + status [301 302 307 308]] + (let [client (fn [req respond raise] + (respond {:status status :body (:body req) + :headers {"location" "http://example.com/bat"}})) + r-client (client/wrap-redirects client) + resp (promise) + exception (promise) + _ (r-client {:body "ok" :url "http://example.com" + :request-method method} resp exception)] + (is (= status (:status @resp))) + (is (= ["http://example.com"] (:trace-redirects @resp))) + (is (= {"location" "http://example.com/bat"} (:headers @resp))) + (is (= "ok" (:body @resp))) + (is (not (realized? exception)))))) + +(deftest force-redirects-on-non-redirectable-methods + (doseq [method [:put :post :delete] + [status expected-method] [[301 :get] [302 :get] [307 method]]] + (let [client (fn [{:keys [trace-redirects body] :as req}] + (if trace-redirects + {:status 200 :body body :trace-redirects trace-redirects + :req req} + {:status status :body body :req req + :headers {"location" "http://example.com/bat"}})) + r-client (client/wrap-redirects client) + resp (r-client {:body "ok" :url "http://example.com" + :request-method method + :force-redirects true})] + (is (= 200 (:status resp))) + (is (= ["http://example.com" "http://example.com/bat"] + (:trace-redirects resp))) + (is (= "ok" (:body resp))) + (is (= expected-method (:request-method (:req resp))))))) + +(deftest force-redirects-on-non-redirectable-methods-async + (doseq [method [:put :post :delete] + [status expected-method] [[301 :get] [302 :get] [307 method]]] + (let [client (fn [{:keys [trace-redirects body] :as req} respond raise] + (respond (if trace-redirects + {:status 200 :body body + :trace-redirects trace-redirects + :req req} + {:status status :body body :req req + :headers {"location" + "http://example.com/bat"}}))) + r-client (client/wrap-redirects client) + resp (promise) + exception (promise) + _ (r-client {:body "ok" :url "http://example.com" + :request-method method + :force-redirects true} resp exception)] + (is (= 200 (:status @resp))) + (is (= ["http://example.com" "http://example.com/bat"] + (:trace-redirects @resp))) + (is (= "ok" (:body @resp))) + (is (= expected-method (:request-method (:req @resp)))) + (is (not (realized? exception)))))) + +(deftest pass-on-follow-redirects-false + (let [client (fn [req] {:status 302 :body (:body req)}) + r-client (client/wrap-redirects client) + resp (r-client {:body "ok" :follow-redirects false})] + (is (= 302 (:status resp))) + (is (= "ok" (:body resp))) + (is (nil? (:trace-redirects resp))))) + +(deftest pass-on-follow-redirects-false-async + (let [client (fn [req respond raise] + (respond {:status 302 :body (:body req)})) + r-client (client/wrap-redirects client) + resp (promise) + exception (promise) + _ (r-client {:body "ok" :follow-redirects false} resp exception)] + (is (= 302 (:status @resp))) + (is (= "ok" (:body @resp))) + (is (nil? (:trace-redirects @resp))) + (is (not (realized? exception))))) + +(deftest throw-on-exceptional + (let [client (fn [req] {:status 500}) + e-client (client/wrap-exceptions client)] + (is (thrown-with-msg? Exception #"500" + (e-client {})))) + (let [client (fn [req] {:status 500 :body "foo"}) + e-client (client/wrap-exceptions client)] + (is (thrown-with-msg? Exception #":body" + (e-client {:throw-entire-message? true}))))) + +(deftest throw-on-custom-exceptional + (let [client (fn [req] {:status 201}) + e-client (client/wrap-exceptions client)] + (is (thrown-with-msg? Exception #"201" + (e-client {:unexceptional-status #{200}}))))) + +(deftest throw-type-field + (let [client (fn [req] {:status 500}) + e-client (client/wrap-exceptions client)] + (try+ + (e-client {}) + (catch [:type :clj-http.client/unexceptional-status] _ + (is true)) + (catch Object _ + (is false ":type selector was not caught."))))) + +(deftest throw-on-exceptional-async + (let [client (fn [req respond raise] + (try + (respond {:status 500}) + (catch Throwable ex + (raise ex)))) + e-client (client/wrap-exceptions client) + resp (promise) + exception (promise) + _ (e-client {} resp exception)] + (is (thrown-with-msg? Exception #"500" + (throw @exception)))) + (let [client (fn [req respond raise] + (try + (respond {:status 500 :body "foo"}) + (catch Throwable ex + (raise ex)))) + e-client (client/wrap-exceptions client) + resp (promise) + exception (promise) + _ (e-client {:throw-entire-message? true} resp exception)] + (is (thrown-with-msg? Exception #":body" + (throw @exception))))) + +(deftest pass-on-non-exceptional + (let [client (fn [req] {:status 200}) + e-client (client/wrap-exceptions client) + resp (e-client {})] + (is (= 200 (:status resp))))) + +(deftest pass-on-custom-non-exceptional + (let [client (fn [req] {:status 500}) + e-client (client/wrap-exceptions client) + resp (e-client {:unexceptional-status #{200 500}})] + (is (= 500 (:status resp))))) + +(deftest pass-on-non-exceptional-async + (let [client (fn [req respond raise] (respond {:status 200})) + e-client (client/wrap-exceptions client) + resp (promise) + exception (promise) + _ (e-client {} resp exception)] + (is (= 200 (:status @resp))) + (is (not (realized? exception))))) + +(deftest pass-on-exceptional-when-surpressed + (let [client (fn [req] {:status 500}) + e-client (client/wrap-exceptions client) + resp (e-client {:throw-exceptions false})] + (is (= 500 (:status resp))))) + +(deftest pass-on-exceptional-when-surpressed-async + (let [client (fn [req respond raise] (respond {:status 500})) + e-client (client/wrap-exceptions client) + resp (promise) + exception (promise) + _ (e-client {:throw-exceptions false} resp exception)] + (is (= 500 (:status @resp))) + (is (not (realized? exception))))) + +(deftest apply-on-compressed + (let [client (fn [req] + (is (= "gzip, deflate" + (get-in req [:headers "accept-encoding"]))) + {:body (util/gzip (util/utf8-bytes "foofoofoo")) + :headers {"content-encoding" "gzip"}}) + c-client (client/wrap-decompression client) + resp (c-client {})] + (is (= "foofoofoo" (util/utf8-string (:body resp)))) + (is (= "gzip" (:orig-content-encoding resp))) + (is (= nil (get-in resp [:headers "content-encoding"]))))) + +(deftest apply-on-compressed-async + (let [client (fn [req respond raise] + (is (= "gzip, deflate" + (get-in req [:headers "accept-encoding"]))) + (respond {:body (util/gzip (util/utf8-bytes "foofoofoo")) + :headers {"content-encoding" "gzip"}})) + c-client (client/wrap-decompression client) + resp (promise) + exception (promise) + _ (c-client {} resp exception)] + (is (= "foofoofoo" (util/utf8-string (:body @resp)))) + (is (= "gzip" (:orig-content-encoding @resp))) + (is (= nil (get-in @resp [:headers "content-encoding"]))))) + +(deftest apply-on-deflated + (let [client (fn [req] + (is (= "gzip, deflate" + (get-in req [:headers "accept-encoding"]))) + {:body (util/deflate (util/utf8-bytes "barbarbar")) + :headers {"content-encoding" "deflate"}}) + c-client (client/wrap-decompression client) + resp (c-client {})] + (is (= "barbarbar" (-> resp :body util/force-byte-array util/utf8-string)) + "string correctly inflated") + (is (= "deflate" (:orig-content-encoding resp))) + (is (= nil (get-in resp [:headers "content-encoding"]))))) + +(deftest apply-on-deflated-async + (let [client (fn [req respond raise] + (is (= "gzip, deflate" + (get-in req [:headers "accept-encoding"]))) + (respond {:body (util/deflate (util/utf8-bytes "barbarbar")) + :headers {"content-encoding" "deflate"}})) + c-client (client/wrap-decompression client) + resp (promise) + exception (promise) + _ (c-client {} resp exception)] + (is (= "barbarbar" (-> @resp :body util/force-byte-array util/utf8-string)) + "string correctly inflated") + (is (= "deflate" (:orig-content-encoding @resp))) + (is (= nil (get-in @resp [:headers "content-encoding"]))))) + +(deftest t-disabled-body-decompression + (let [client (fn [req] + (is (not= "gzip, deflate" + (get-in req [:headers "accept-encoding"]))) + {:body (util/deflate (util/utf8-bytes "barbarbar")) + :headers {"content-encoding" "deflate"}}) + c-client (client/wrap-decompression client) + resp (c-client {:decompress-body false})] + (is (= (slurp (util/inflate (util/deflate (util/utf8-bytes "barbarbar")))) + (slurp (util/inflate (-> resp :body util/force-byte-array)))) + "string not inflated") + (is (= nil (:orig-content-encoding resp))) + (is (= "deflate" (get-in resp [:headers "content-encoding"]))))) + +(deftest t-weird-non-known-compression + (let [client (fn [req] + (is (= "gzip, deflate" + (get-in req [:headers "accept-encoding"]))) + {:body (util/utf8-bytes "foofoofoo") + :headers {"content-encoding" "pig-latin"}}) + c-client (client/wrap-decompression client) + resp (c-client {})] + (is (= "foofoofoo" (util/utf8-string (:body resp)))) + (is (= "pig-latin" (:orig-content-encoding resp))) + (is (= "pig-latin" (get-in resp [:headers "content-encoding"]))))) + +(deftest pass-on-non-compressed + (let [c-client (client/wrap-decompression (fn [req] {:body "foo"})) + resp (c-client {:uri "/foo"})] + (is (= "foo" (:body resp))))) + +(deftest apply-on-accept + (is-applied client/wrap-accept + {:accept :json} + {:headers {"accept" "application/json"}}) + (is-applied client/wrap-accept + {:accept :transit+json} + {:headers {"accept" "application/transit+json"}}) + (is-applied client/wrap-accept + {:accept :transit+msgpack} + {:headers {"accept" "application/transit+msgpack"}})) + +(deftest apply-on-accept-async + (is-applied-async client/wrap-accept + {:accept :json} + {:headers {"accept" "application/json"}}) + (is-applied-async client/wrap-accept + {:accept :transit+json} + {:headers {"accept" "application/transit+json"}}) + (is-applied-async client/wrap-accept + {:accept :transit+msgpack} + {:headers {"accept" "application/transit+msgpack"}})) + +(deftest pass-on-no-accept + (is-passed client/wrap-accept + {:uri "/foo"})) + +(deftest pass-on-no-accept-async + (is-passed-async client/wrap-accept + {:uri "/foo"})) + +(deftest apply-on-accept-encoding + (is-applied client/wrap-accept-encoding + {:accept-encoding [:identity :gzip]} + {:headers {"accept-encoding" "identity, gzip"}})) + +(deftest apply-custom-accept-encoding + (testing "no custom encodings to accept" + (is-applied (comp client/wrap-accept-encoding + client/wrap-decompression) + {} + {:headers {"accept-encoding" "gzip, deflate"} + :orig-content-encoding nil})) + (testing "accept some custom encodings, but still include gzip and deflate" + (is-applied (comp client/wrap-accept-encoding + client/wrap-decompression) + {:accept-encoding [:foo :bar]} + {:headers {"accept-encoding" "foo, bar, gzip, deflate"} + :orig-content-encoding nil})) + (testing "accept some custom encodings, but exclude gzip and deflate" + (is-applied (comp client/wrap-accept-encoding + client/wrap-decompression) + {:accept-encoding [:foo :bar] :decompress-body false} + {:headers {"accept-encoding" "foo, bar"} + :decompress-body false}))) + +(deftest pass-on-no-accept-encoding + (is-passed client/wrap-accept-encoding + {:uri "/foo"})) + +(deftest apply-on-output-coercion + (let [client (fn [req] {:body (util/utf8-bytes "foo")}) + o-client (client/wrap-output-coercion client) + resp (o-client {:uri "/foo"})] + (is (= "foo" (:body resp))))) + +(deftest apply-on-output-coercion-async + (let [client (fn [req respond raise] + (respond {:body (util/utf8-bytes "foo")})) + o-client (client/wrap-output-coercion client) + resp (promise) + exception (promise) + _ (o-client {:uri "/foo"} resp exception)] + (is (= "foo" (:body @resp))) + (is (not (realized? exception))))) + +(deftest pass-on-no-output-coercion + (let [client (fn [req] {:body nil}) + o-client (client/wrap-output-coercion client) + resp (o-client {:uri "/foo"})] + (is (nil? (:body resp)))) + (let [the-stream (ByteArrayInputStream. (byte-array [])) + client (fn [req] {:body the-stream}) + o-client (client/wrap-output-coercion client) + resp (o-client {:uri "/foo" :as :stream})] + (is (= the-stream (:body resp)))) + (let [client (fn [req] {:body :thebytes}) + o-client (client/wrap-output-coercion client) + resp (o-client {:uri "/foo" :as :byte-array})] + (is (= :thebytes (:body resp))))) + +(deftest pass-on-no-output-coercion-async + (let [client (fn [req] {:body nil}) + o-client (client/wrap-output-coercion client) + resp (o-client {:uri "/foo"})] + (is (nil? (:body resp)))) + (let [the-stream (ByteArrayInputStream. (byte-array [])) + client (fn [req] {:body the-stream}) + o-client (client/wrap-output-coercion client) + resp (o-client {:uri "/foo" :as :stream})] + (is (= the-stream (:body resp)))) + (let [client (fn [req] {:body :thebytes}) + o-client (client/wrap-output-coercion client) + resp (o-client {:uri "/foo" :as :byte-array})] + (is (= :thebytes (:body resp))))) + +(deftest apply-on-input-coercion + (let [i-client (client/wrap-input-coercion identity) + resp (i-client {:body "foo"}) + resp2 (i-client {:body "foo2" :body-encoding "ASCII"}) + data (slurp (.getContent ^HttpEntity (:body resp))) + data2 (slurp (.getContent ^HttpEntity (:body resp2)))] + (is (= "UTF-8" (:character-encoding resp))) + (is (= "foo" data)) + (is (= "ASCII" (:character-encoding resp2))) + (is (= "foo2" data2)))) + +(deftest apply-on-input-coercion-async + (let [i-client (client/wrap-input-coercion (fn [request respond raise] + (respond request))) + resp (promise) + _ (i-client {:body "foo"} resp nil) + resp2 (promise) + _ (i-client {:body "foo2" :body-encoding "ASCII"} resp2 nil) + data (slurp (.getContent ^HttpEntity (:body @resp))) + data2 (slurp (.getContent ^HttpEntity (:body @resp2)))] + (is (= "UTF-8" (:character-encoding @resp))) + (is (= "foo" data)) + (is (= "ASCII" (:character-encoding @resp2))) + (is (= "foo2" data2)))) + +(deftest pass-on-no-input-coercion + (is-passed client/wrap-input-coercion + {:body nil})) + +(deftest pass-on-no-input-coercion + (is-passed-async client/wrap-input-coercion + {:body nil})) + +(deftest no-length-for-input-stream + (let [i-client (client/wrap-input-coercion identity) + resp1 (i-client {:body (ByteArrayInputStream. (util/utf8-bytes "foo"))}) + resp2 (i-client {:body (ByteArrayInputStream. (util/utf8-bytes "foo")) + :length 3}) + ^HttpEntity body1 (:body resp1) + ^HttpEntity body2 (:body resp2)] + (is (= -1 (.getContentLength body1))) + (is (= 3 (.getContentLength body2))))) + +(deftest apply-on-content-type + (is-applied client/wrap-content-type + {:content-type :json} + {:headers {"content-type" "application/json"} + :content-type :json}) + (is-applied client/wrap-content-type + {:content-type :json :character-encoding "UTF-8"} + {:headers {"content-type" "application/json; charset=UTF-8"} + :content-type :json :character-encoding "UTF-8"}) + (is-applied client/wrap-content-type + {:content-type :transit+json} + {:headers {"content-type" "application/transit+json"} + :content-type :transit+json}) + (is-applied client/wrap-content-type + {:content-type :transit+msgpack} + {:headers {"content-type" "application/transit+msgpack"} + :content-type :transit+msgpack})) + +(deftest apply-on-content-type-async + (is-applied-async client/wrap-content-type + {:content-type :json} + {:headers {"content-type" "application/json"} + :content-type :json}) + (is-applied-async client/wrap-content-type + {:content-type :json :character-encoding "UTF-8"} + {:headers {"content-type" "application/json; charset=UTF-8"} + :content-type :json :character-encoding "UTF-8"}) + (is-applied-async client/wrap-content-type + {:content-type :transit+json} + {:headers {"content-type" "application/transit+json"} + :content-type :transit+json}) + (is-applied-async client/wrap-content-type + {:content-type :transit+msgpack} + {:headers {"content-type" "application/transit+msgpack"} + :content-type :transit+msgpack})) + +(deftest pass-on-no-content-type + (is-passed client/wrap-content-type + {:uri "/foo"})) + +(deftest apply-on-query-params + (is-applied client/wrap-query-params + {:query-params {"foo" "bar" "dir" "<<"}} + {:query-string "foo=bar&dir=%3C%3C"}) + (is-applied client/wrap-query-params + {:query-string "foo=1" + :query-params {"foo" ["2" "3"]}} + {:query-string "foo=1&foo=2&foo=3"})) + +(deftest apply-on-query-params-async + (is-applied-async client/wrap-query-params + {:query-params {"foo" "bar" "dir" "<<"}} + {:query-string "foo=bar&dir=%3C%3C"}) + (is-applied-async client/wrap-query-params + {:query-string "foo=1" + :query-params {"foo" ["2" "3"]}} + {:query-string "foo=1&foo=2&foo=3"})) + +(deftest pass-on-no-query-params + (is-passed client/wrap-query-params + {:uri "/foo"})) + +(deftest apply-on-basic-auth + (is-applied client/wrap-basic-auth + {:basic-auth ["Aladdin" "open sesame"]} + {:headers {"authorization" + "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ=="}})) + +(deftest apply-on-basic-auth-async + (is-applied-async client/wrap-basic-auth + {:basic-auth ["Aladdin" "open sesame"]} + {:headers {"authorization" + "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ=="}})) + +(deftest pass-on-no-basic-auth + (is-passed client/wrap-basic-auth + {:uri "/foo"})) + +(deftest apply-on-oauth + (is-applied client/wrap-oauth + {:oauth-token "my-token"} + {:headers {"authorization" + "Bearer my-token"}})) + +(deftest apply-on-oauth-async + (is-applied-async client/wrap-oauth + {:oauth-token "my-token"} + {:headers {"authorization" + "Bearer my-token"}})) + +(deftest pass-on-no-oauth + (is-passed client/wrap-oauth + {:uri "/foo"})) + +(deftest apply-on-method + (let [m-client (client/wrap-method identity) + echo (m-client {:key :val :method :post})] + (is (= :val (:key echo))) + (is (= :post (:request-method echo))) + (is (not (:method echo))))) + +(deftest apply-on-method-async + (let [m-client (client/wrap-method async-identity-client) + echo (promise) + exception (promise) + _ (m-client {:key :val :method :post} echo exception)] + (is (= :val (:key @echo))) + (is (= :post (:request-method @echo))) + (is (not (:method @echo))))) + +(deftest pass-on-no-method + (let [m-client (client/wrap-method identity) + echo (m-client {:key :val})] + (is (= :val (:key echo))) + (is (not (:request-method echo))))) + +(deftest apply-on-url + (let [u-client (client/wrap-url identity) + resp (u-client {:url "http://google.com:8080/baz foo?bar=bat bit?"})] + (is (= :http (:scheme resp))) + (is (= "google.com" (:server-name resp))) + (is (= 8080 (:server-port resp))) + (is (= "/baz%20foo" (:uri resp))) + (is (= "bar=bat%20bit?" (:query-string resp))))) + +(deftest apply-on-url + (let [u-client (client/wrap-url async-identity-client) + resp (promise) + exception (promise) + _ (u-client {:url "http://google.com:8080/baz foo?bar=bat bit?"} + resp exception)] + (is (= :http (:scheme @resp))) + (is (= "google.com" (:server-name @resp))) + (is (= 8080 (:server-port @resp))) + (is (= "/baz%20foo" (:uri @resp))) + (is (= "bar=bat%20bit?" (:query-string @resp))) + (is (not (realized? exception))))) + +(deftest pass-on-no-url + (let [u-client (client/wrap-url identity) + resp (u-client {:uri "/foo"})] + (is (= "/foo" (:uri resp))))) + +(deftest provide-default-port + (is (= nil (-> "http://example.com/" client/parse-url :server-port))) + (is (= 8080 (-> "http://example.com:8080/" client/parse-url :server-port))) + (is (= nil (-> "https://example.com/" client/parse-url :server-port))) + (is (= 8443 (-> "https://example.com:8443/" client/parse-url :server-port))) + (is (= "https://example.com:8443/" + (-> "https://example.com:8443/" client/parse-url :url)))) + +(deftest decode-credentials-from-url + (is (= "fred's diner:fred's password" + (-> "http://fred%27s%20diner:fred%27s%20password@example.com/foo" + client/parse-url + :user-info)))) + +(deftest unparse-url + (is (= "http://fred's diner:fred's password@example.com/foo" + (-> "http://fred%27s%20diner:fred%27s%20password@example.com/foo" + client/parse-url client/unparse-url))) + (is (= "https://foo:bar@example.org:8080" + (-> "https://foo:bar@example.org:8080" + client/parse-url client/unparse-url))) + (is (= "ftp://example.org?foo" + (-> "ftp://example.org?foo" + client/parse-url client/unparse-url)))) + +(defrecord Point [x y]) + +(def write-point + "Write a point in Transit format." + (transit/write-handler + (constantly "point") + (fn [point] [(:x point) (:y point)]) + (constantly nil))) + +(def read-point + "Read a point in Transit format." + (transit/read-handler + (fn [[x y]] + (->Point x y)))) + +(def transit-opts + "Transit read and write options." + {:encode {:handlers {Point write-point}} + :decode {:handlers {"point" read-point}}}) + +(def transit-opts-deprecated + "Deprecated Transit read and write options." + {:handlers {Point write-point "point" read-point}}) + +(deftest apply-on-form-params + (testing "With form params" + (let [param-client (client/wrap-form-params identity) + resp (param-client {:request-method :post + :form-params (sorted-map :param1 "value1" + :param2 "value2")})] + (is (= "param1=value1¶m2=value2" (:body resp))) + (is (= "application/x-www-form-urlencoded" (:content-type resp))) + (is (not (contains? resp :form-params)))) + (let [param-client (client/wrap-form-params identity) + resp (param-client {:request-method :put + :form-params (sorted-map :param1 "value1" + :param2 "value2")})] + (is (= "param1=value1¶m2=value2" (:body resp))) + (is (= "application/x-www-form-urlencoded" (:content-type resp))) + (is (not (contains? resp :form-params))))) + + (testing "With json form params" + (let [param-client (client/wrap-form-params identity) + params {:param1 "value1" :param2 "value2"} + resp (param-client {:request-method :post + :content-type :json + :form-params params})] + (is (= (json/encode params) (:body resp))) + (is (= "application/json" (:content-type resp))) + (is (not (contains? resp :form-params)))) + (let [param-client (client/wrap-form-params identity) + params {:param1 "value1" :param2 "value2"} + resp (param-client {:request-method :put + :content-type :json + :form-params params})] + (is (= (json/encode params) (:body resp))) + (is (= "application/json" (:content-type resp))) + (is (not (contains? resp :form-params)))) + (let [param-client (client/wrap-form-params identity) + params {:param1 "value1" :param2 "value2"} + resp (param-client {:request-method :patch + :content-type :json + :form-params params})] + (is (= (json/encode params) (:body resp))) + (is (= "application/json" (:content-type resp))) + (is (not (contains? resp :form-params)))) + (let [param-client (client/wrap-form-params identity) + params {:param1 (java.util.Date. (long 0))} + resp (param-client {:request-method :put + :content-type :json + :form-params params + :json-opts {:date-format "yyyy-MM-dd"}})] + (is (= (json/encode params {:date-format "yyyy-MM-dd"}) (:body resp))) + (is (= "application/json" (:content-type resp))) + (is (not (contains? resp :form-params))))) + + (testing "With EDN form params" + (doseq [method [:post :put :patch]] + (let [param-client (client/wrap-form-params identity) + params {:param1 "value1" :param2 (Point. 1 2)} + resp (param-client {:request-method method + :content-type :edn + :form-params params})] + (is (= (pr-str params) (:body resp))) + (is (= "application/edn" (:content-type resp))) + (is (not (contains? resp :form-params)))))) + + (testing "With Transit/JSON form params" + (doseq [method [:post :put :patch]] + (let [param-client (client/wrap-form-params identity) + params {:param1 "value1" :param2 (Point. 1 2)} + resp (param-client {:request-method method + :content-type :transit+json + :form-params params + :transit-opts transit-opts})] + (is (= params (client/parse-transit + (ByteArrayInputStream. (:body resp)) + :json transit-opts))) + (is (= "application/transit+json" (:content-type resp))) + (is (not (contains? resp :form-params)))))) + + (testing "With Transit/MessagePack form params" + (doseq [method [:post :put :patch]] + (let [param-client (client/wrap-form-params identity) + params {:param1 "value1" :param2 "value2"} + resp (param-client {:request-method method + :content-type :transit+msgpack + :form-params params + :transit-opts transit-opts})] + (is (= params (client/parse-transit + (ByteArrayInputStream. (:body resp)) + :msgpack transit-opts))) + (is (= "application/transit+msgpack" (:content-type resp))) + (is (not (contains? resp :form-params)))))) + + (testing "With Transit/JSON form params and deprecated options" + (let [param-client (client/wrap-form-params identity) + params {:param1 "value1" :param2 (Point. 1 2)} + resp (param-client {:request-method :post + :content-type :transit+json + :form-params params + :transit-opts transit-opts-deprecated})] + (is (= params (client/parse-transit + (ByteArrayInputStream. (:body resp)) + :json transit-opts-deprecated))) + (is (= "application/transit+json" (:content-type resp))) + (is (not (contains? resp :form-params))))) + + (testing "Ensure it does not affect GET requests" + (let [param-client (client/wrap-form-params identity) + resp (param-client {:request-method :get + :body "untouched" + :form-params {:param1 "value1" + :param2 "value2"}})] + (is (= "untouched" (:body resp))) + (is (not (contains? resp :content-type))))) + + (testing "with no form params" + (let [param-client (client/wrap-form-params identity) + resp (param-client {:body "untouched"})] + (is (= "untouched" (:body resp))) + (is (not (contains? resp :content-type)))))) + +(deftest apply-on-form-params-async + (testing "With form params" + (let [param-client (client/wrap-form-params async-identity-client) + resp (promise) + exception (promise) + _ (param-client {:request-method :post + :form-params (sorted-map :param1 "value1" + :param2 "value2")} + resp exception)] + (is (= "param1=value1¶m2=value2" (:body @resp))) + (is (= "application/x-www-form-urlencoded" (:content-type @resp))) + (is (not (contains? @resp :form-params))) + (is (not (realized? exception)))) + (let [param-client (client/wrap-form-params async-identity-client) + resp (promise) + exception (promise) + _ (param-client {:request-method :put + :form-params (sorted-map :param1 "value1" + :param2 "value2")} + resp exception)] + (is (= "param1=value1¶m2=value2" (:body @resp))) + (is (= "application/x-www-form-urlencoded" (:content-type @resp))) + (is (not (contains? @resp :form-params))) + (is (not (realized? exception))))) + + (testing "Ensure it does not affect GET requests" + (let [param-client (client/wrap-form-params async-identity-client) + resp (promise) + exception (promise) + _ (param-client {:request-method :get + :body "untouched" + :form-params {:param1 "value1" + :param2 "value2"}} + resp exception)] + (is (= "untouched" (:body @resp))) + (is (not (contains? @resp :content-type))) + (is (not (realized? exception))))) + + (testing "with no form params" + (let [param-client (client/wrap-form-params async-identity-client) + resp (promise) + exception (promise) + _ (param-client {:body "untouched"} resp exception)] + (is (= "untouched" (:body @resp))) + (is (not (contains? @resp :content-type))) + (is (not (realized? exception)))))) + +(deftest apply-on-nested-params + (testing "nested parameter maps" + (is-applied (comp client/wrap-form-params + client/wrap-nested-params) + {:query-params {"foo" "bar"} + :form-params {"foo" "bar"} + :flatten-nested-keys [:query-params :form-params]} + {:query-params {"foo" "bar"} + :form-params {"foo" "bar"} + :flatten-nested-keys [:query-params :form-params]}) + (is-applied (comp client/wrap-form-params + client/wrap-nested-params) + {:query-params {"x" {"y" "z"}} + :form-params {"x" {"y" "z"}} + :flatten-nested-keys [:query-params]} + {:query-params {"x[y]" "z"} + :form-params {"x" {"y" "z"}} + :flatten-nested-keys [:query-params]}) + (is-applied (comp client/wrap-form-params + client/wrap-nested-params) + {:query-params {"a" {"b" {"c" "d"}}} + :form-params {"a" {"b" {"c" "d"}}} + :flatten-nested-keys [:form-params]} + {:query-params {"a" {"b" {"c" "d"}}} + :form-params {"a[b][c]" "d"} + :flatten-nested-keys [:form-params]}) + (is-applied (comp client/wrap-form-params + client/wrap-nested-params) + {:query-params {"a" {"b" {"c" "d"}}} + :form-params {"a" {"b" {"c" "d"}}} + :flatten-nested-keys [:query-params :form-params]} + {:query-params {"a[b][c]" "d"} + :form-params {"a[b][c]" "d"} + :flatten-nested-keys [:query-params :form-params]})) + + (testing "not creating empty param maps" + (is-applied client/wrap-query-params {} {}))) + +(deftest t-ignore-unknown-host + (is (thrown? UnknownHostException (client/get "http://example.invalid"))) + (is (nil? (client/get "http://example.invalid" + {:ignore-unknown-host? true})))) + +(deftest t-ignore-unknown-host-async + (let [resp (promise) exception (promise)] + (client/get "http://example.invalid" + {:async? true} resp exception) + (is (thrown? UnknownHostException (throw @exception)))) + (let [resp (promise) exception (promise)] + (client/get "http://example.invalid" + {:ignore-unknown-host? true + :async? true} resp exception) + (is (nil? @resp)))) + +(deftest test-status-predicates + (testing "2xx statuses" + (doseq [s (range 200 299)] + (is (client/success? {:status s})) + (is (not (client/redirect? {:status s}))) + (is (not (client/client-error? {:status s}))) + (is (not (client/server-error? {:status s}))))) + (testing "3xx statuses" + (doseq [s (range 300 399)] + (is (not (client/success? {:status s}))) + (is (client/redirect? {:status s})) + (is (not (client/client-error? {:status s}))) + (is (not (client/server-error? {:status s}))))) + (testing "4xx statuses" + (doseq [s (range 400 499)] + (is (not (client/success? {:status s}))) + (is (not (client/redirect? {:status s}))) + (is (client/client-error? {:status s})) + (is (not (client/server-error? {:status s}))))) + (testing "5xx statuses" + (doseq [s (range 500 599)] + (is (not (client/success? {:status s}))) + (is (not (client/redirect? {:status s}))) + (is (not (client/client-error? {:status s}))) + (is (client/server-error? {:status s})))) + (testing "409 Conflict" + (is (client/conflict? {:status 409})) + (is (not (client/conflict? {:status 201}))) + (is (not (client/conflict? {:status 404}))))) + +(deftest test-wrap-lower-case-headers + (is (= {:status 404} ((client/wrap-lower-case-headers + (fn [r] r)) {:status 404}))) + (is (= {:headers {"content-type" "application/json"}} + ((client/wrap-lower-case-headers + #(do (is (= {:headers {"accept" "application/json"}} %1)) + {:headers {"Content-Type" "application/json"}})) + {:headers {"Accept" "application/json"}})))) + +(deftest t-request-timing + (is (pos? (:request-time ((client/wrap-request-timing + (fn [r] (Thread/sleep 15) r)) {}))))) + +(deftest t-wrap-additional-header-parsing + (let [^String text (slurp (resource "header-test.html")) + client (fn [req] {:body (.getBytes text)}) + new-client (client/wrap-additional-header-parsing client) + resp (new-client {:decode-body-headers true}) + resp2 (new-client {:decode-body-headers false}) + resp3 ((client/wrap-additional-header-parsing + (fn [req] {:body nil})) {:decode-body-headers true}) + resp4 ((client/wrap-additional-header-parsing + (fn [req] {:headers {"content-type" "application/pdf"} + :body (.getBytes text)})) + {:decode-body-headers true})] + (is (= {"content-type" "text/html; charset=Shift_JIS" + "content-style-type" "text/css" + "content-script-type" "text/javascript"} + (:headers resp))) + (is (nil? (:headers resp2))) + (is (nil? (:headers resp3))) + (is (= {"content-type" "application/pdf"} (:headers resp4))))) + +(deftest t-wrap-additional-header-parsing-html5 + (let [^String text (slurp (resource "header-html5-test.html")) + client (fn [req] {:body (.getBytes text)}) + new-client (client/wrap-additional-header-parsing client) + resp (new-client {:decode-body-headers true})] + (is (= {"content-type" "text/html; charset=UTF-8"} + (:headers resp))))) + +(deftest ^:integration t-request-without-url-set + (run-server) + ;; roundtrip with scheme as a keyword + (let [resp (request {:uri "/redirect-to-get" + :method :get})] + (is (= 200 (:status resp))) + (is (= "close" (get-in resp [:headers "connection"]))) + (is (= "get" (:body resp))))) + +(deftest ^:integration t-reusable-conn-mgrs + (run-server) + (let [cm (conn/make-reusable-conn-manager {:timeout 10 :insecure? false}) + resp1 (request {:uri "/redirect-to-get" + :method :get + :connection-manager cm}) + resp2 (request {:uri "/redirect-to-get" + :method :get})] + (is (= 200 (:status resp1) (:status resp2))) + (is (nil? (get-in resp1 [:headers "connection"])) + "connection should remain open") + (is (= "close" (get-in resp2 [:headers "connection"])) + "connection should be closed") + (.shutdown cm))) + +(deftest ^:integration t-reusable-async-conn-mgrs + (run-server) + (let [cm (conn/make-reuseable-async-conn-manager {:timeout 10 :insecure? false}) + resp1 (promise) resp2 (promise) + exce1 (promise) exce2 (promise)] + (request {:async? true :uri "/redirect-to-get" :method :get :connection-manager cm} + resp1 + exce1) + (request {:async? true :uri "/redirect-to-get" :method :get} + resp2 + exce2) + (is (= 200 (:status @resp1) (:status @resp2))) + (is (nil? (get-in @resp1 [:headers "connection"])) + "connection should remain open") + (is (= "close" (get-in @resp2 [:headers "connection"])) + "connection should be closed") + (is (not (realized? exce2))) + (is (not (realized? exce1))) + (.shutdown cm))) + +(deftest ^:integration t-with-async-pool + (run-server) + (client/with-async-connection-pool {} + (let [resp1 (promise) resp2 (promise) + exce1 (promise) exce2 (promise)] + (request {:async? true :uri "/get" :method :get} resp1 exce1) + (request {:async? true :uri "/get" :method :get} resp2 exce2) + (is (= 200 (:status @resp1) (:status @resp2))) + (is (not (realized? exce2))) + (is (not (realized? exce1)))))) + +(deftest ^:integration t-with-async-pool-sleep + (run-server) + (client/with-async-connection-pool {} + (let [resp1 (promise) resp2 (promise) + exce1 (promise) exce2 (promise)] + (request {:async? true :uri "/get" :method :get} resp1 exce1) + (Thread/sleep 500) + (request {:async? true :uri "/get" :method :get} resp2 exce2) + (is (= 200 (:status @resp1) (:status @resp2))) + (is (not (realized? exce2))) + (is (not (realized? exce1)))))) + +(deftest ^:integration t-async-pool-wrap-exception + (run-server) + (client/with-async-connection-pool {} + (let [resp1 (promise) resp2 (promise) + exce1 (promise) exce2 (promise) count (atom 2)] + (request {:async? true :uri "/error" :method :get} resp1 exce1) + (Thread/sleep 500) + (request {:async? true :uri "/get" :method :get} resp2 exce2) + (is (realized? exce1)) + (is (not (realized? exce2))) + (is (= 200 (:status @resp2)))))) + +(deftest ^:integration t-async-pool-exception-when-start + (run-server) + (client/with-async-connection-pool {} + (let [resp1 (promise) resp2 (promise) + exce1 (promise) exce2 (promise) + middleware (fn [client] + (fn [req resp raise] (throw (Exception.))))] + (client/with-additional-middleware + [middleware] + (try (request {:async? true :uri "/error" :method :get} resp1 exce1) + (catch Throwable ex)) + (Thread/sleep 500) + (try (request {:async? true :uri "/get" :method :get} resp2 exce2) + (catch Throwable ex)) + (is (not (realized? exce1))) + (is (not (realized? exce2))) + (is (not (realized? resp1))) + (is (not (realized? resp2))))))) + +(deftest ^:integration t-reuse-async-pool + (run-server) + (client/with-async-connection-pool {} + (let [resp1 (promise) resp2 (promise) + exce1 (promise) exce2 (promise)] + (request {:async? true :uri "/get" :method :get} + (fn [resp] + (resp1 resp) + (request (client/reuse-pool + {:async? true + :uri "/get" + :method :get} + resp) + resp2 + exce2)) + exce1) + (is (= 200 (:status @resp1) (:status @resp2))) + (is (not (realized? exce2))) + (is (not (realized? exce1)))))) + +(deftest ^:integration t-async-pool-redirect-to-get + (run-server) + (client/with-async-connection-pool {} + (let [resp (promise) + exce (promise)] + (request {:async? true :uri "/redirect-to-get" + :method :get :redirect-strategy :default} resp exce) + (is (= 200 (:status @resp))) + (is (not (realized? exce)))))) + +(deftest ^:integration t-async-pool-max-redirect + (run-server) + (client/with-async-connection-pool {} + (let [resp (promise) + exce (promise)] + (request {:async? true :uri "/redirect" :method :get + :redirect-strategy :default + :throw-exceptions true} resp exce) + (is @exce) + (is (not (realized? resp)))))) + +(deftest test-url-encode-path + (is (= (client/url-encode-illegal-characters "?foo bar+baz[]75") + "?foo%20bar+baz%5B%5D75")) + (is (= {:uri (str "/:@-._~!$&'()*+,=" + ";" + ":@-._~!$&'()*+," + "=" + ":@-._~!$&'()*+,==") + :query-string (str "/?:@-._~!$'()*+,;" + "=" + "/?:@-._~!$'()*+,;==")} + ;; This URL sucks, yes, it's actually a valid URL + (select-keys (client/parse-url + (str "http://example.com/:@-._~!$&'()*+,=;:@-._~!$&'()*+" + ",=:@-._~!$&'()*+,==?/?:@-._~!$'()*+,;=/?:@-._~!$'(" + ")*+,;==#/?:@-._~!$&'()*+,;=")) + [:uri :query-string]))) + (let [all-chars (apply str (map char (range 256))) + all-legal (client/url-encode-illegal-characters all-chars)] + (is (= all-legal + (client/url-encode-illegal-characters all-legal))))) + +(defmethod client/coerce-response-body :json+ms949 + [req resp] + (client/coerce-json-body req resp true "MS949")) + +(deftest t-coercion-methods + (let [json-body (ByteArrayInputStream. (.getBytes "{\"foo\":\"bar\"}")) + json-ms949-body (ByteArrayInputStream. (.getBytes "{\"foo\":\"안뇽\"}" "MS949")) + auto-body (ByteArrayInputStream. (.getBytes "{\"foo\":\"bar\"}")) + edn-body (ByteArrayInputStream. (.getBytes "{:foo \"bar\"}")) + transit-json-body (ByteArrayInputStream. + (.getBytes "[\"^ \",\"~:foo\",\"bar\"]")) + transit-msgpack-body (->> (map byte [-127 -91 126 58 102 111 + 111 -93 98 97 114]) + (byte-array 11) + (ByteArrayInputStream.)) + www-form-urlencoded-body (ByteArrayInputStream. (.getBytes "foo=bar")) + auto-www-form-urlencoded-body + (ByteArrayInputStream. (.getBytes "foo=bar")) + json-resp {:body json-body :status 200 + :headers {"content-type" "application/json"}} + json-ms949-resp {:body json-ms949-body :status 200 + :headers {"content-type" "application/json; charset=ms949"}} + auto-resp {:body auto-body :status 200 + :headers {"content-type" "application/json"}} + edn-resp {:body edn-body :status 200 + :headers {"content-type" "application/edn"}} + transit-json-resp {:body transit-json-body :status 200 + :headers {"content-type" "application/transit-json"}} + transit-msgpack-resp {:body transit-msgpack-body :status 200 + :headers {"content-type" + "application/transit-msgpack"}} + www-form-urlencoded-resp + {:body www-form-urlencoded-body :status 200 + :headers {"content-type" + "application/x-www-form-urlencoded"}} + auto-www-form-urlencoded-resp + {:body auto-www-form-urlencoded-body :status 200 + :headers {"content-type" + "application/x-www-form-urlencoded"}}] + (is (= {:foo "bar"} + (:body (client/coerce-response-body {:as :json} json-resp)) + (:body (client/coerce-response-body {:as :clojure} edn-resp)) + (:body (client/coerce-response-body {:as :auto} auto-resp)) + (:body (client/coerce-response-body {:as :transit+json} + transit-json-resp)) + (:body (client/coerce-response-body {:as :transit+msgpack} + transit-msgpack-resp)) + (:body (client/coerce-response-body {:as :auto} + auto-www-form-urlencoded-resp)) + (:body (client/coerce-response-body {:as :x-www-form-urlencoded} + www-form-urlencoded-resp)))) + (is (= {:foo "안뇽"} + (:body (client/coerce-response-body {:as :json+ms949} json-ms949-resp)))) + + (testing "throws AssertionError when optional libraries are not loaded" + (with-redefs [client/json-enabled? false] + (is (thrown? AssertionError (client/coerce-response-body {:as :json} json-resp))) + (is (thrown? AssertionError (client/coerce-response-body {:as :auto} json-resp)))) + (with-redefs [client/transit-enabled? false] + (is (thrown? AssertionError (client/coerce-response-body {:as :transit+json} transit-json-resp))) + (is (thrown? AssertionError (client/coerce-response-body {:as :transit+msgpack} transit-msgpack-resp)))) + (with-redefs [client/ring-codec-enabled? false] + (is (thrown? AssertionError (client/coerce-response-body {:as :x-www-form-urlencoded} www-form-urlencoded-resp))) + (is (thrown? AssertionError (client/coerce-response-body {:as :auto} auto-www-form-urlencoded-resp))))))) + + +(deftest t-reader-coercion + (let [read-lines (fn [reader] (vec (take-while not-empty (repeatedly #(.readLine reader))))) + reader-body (ByteArrayInputStream. (.getBytes "foo\nbar\n")) + reader-resp {:body reader-body :status 200 :headers {"content-type" "text/plain; charset=utf-8"}} + encoded-body (ByteArrayInputStream. (byte-array [0xA9])) + encoded-resp {:body encoded-body :status 200 :headers {"content-type" "text/plain; charset=iso-8859-1"}} + utf8-body (ByteArrayInputStream. (byte-array [0xC2 0xA9])) + utf8-resp {:body utf8-body :status 200 :headers {"content-type" "text/plain; charset=utf-8"}}] + (is (= ["foo" "bar"] + (read-lines (:body (client/coerce-response-body {:as :reader} reader-resp))))) + + (is (= "©" + (.readLine (:body (client/coerce-response-body {:as :reader} encoded-resp))) + (.readLine (:body (client/coerce-response-body {:as :reader} utf8-resp))))))) + +(deftest ^:integration t-with-middleware + (run-server) + (is (:request-time (request {:uri "/get" :method :get}))) + (is (= client/*current-middleware* client/default-middleware)) + (client/with-middleware [client/wrap-url + client/wrap-method + #'client/wrap-request-timing] + (is (:request-time (request {:uri "/get" :method :get}))) + (is (= client/*current-middleware* [client/wrap-url + client/wrap-method + #'client/wrap-request-timing]))) + (client/with-middleware (->> client/default-middleware + (remove #{client/wrap-request-timing})) + (is (not (:request-time (request {:uri "/get" :method :get})))) + (is (not (contains? (set client/*current-middleware*) + client/wrap-request-timing))) + (is (contains? (set client/default-middleware) + client/wrap-request-timing)))) + +(deftest t-detect-charset-by-content-type + (is (= "UTF-8" (client/detect-charset nil))) + (is (= "UTF-8"(client/detect-charset "application/json"))) + (is (= "UTF-8"(client/detect-charset "text/html"))) + (is (= "GBK"(client/detect-charset "application/json; charset=GBK"))) + (is (= "ISO-8859-1" (client/detect-charset + "application/json; charset=ISO-8859-1"))) + (is (= "ISO-8859-1" (client/detect-charset + "application/json; charset = ISO-8859-1"))) + (is (= "GB2312" (client/detect-charset "text/html; Charset=GB2312")))) + +(deftest ^:integration customMethodTest + (run-server) + (let [resp (request {:uri "/propfind" :method "PROPFIND"})] + (is (= 200 (:status resp))) + (is (= "close" (get-in resp [:headers "connection"]))) + (is (= "propfind" (:body resp)))) + (let [resp (request {:uri "/propfind-with-body" + :method "PROPFIND" + :body "propfindbody"})] + (is (= 200 (:status resp))) + (is (= "close" (get-in resp [:headers "connection"]))) + (is (= "propfindbody" (:body resp))))) + +(deftest ^:integration status-line-parsing + (run-server) + (let [resp (request {:uri "/get" :method :get}) + protocol-version (:protocol-version resp)] + (is (= 200 (:status resp))) + (is (= "HTTP" (:name protocol-version))) + (is (= 1 (:major protocol-version))) + (is (= 1 (:minor protocol-version))) + (is (= "OK" (:reason-phrase resp))))) + +(deftest ^:integration multi-valued-query-params + (run-server) + (testing "default (repeating) multi-valued query params" + (let [resp (request {:uri "/query-string" + :method :get + :query-params {:a [1 2 3] + :b ["x" "y" "z"]}}) + query-string (-> resp :body form-decode-str)] + (is (= 200 (:status resp))) + (is (.contains query-string "a=1&a=2&a=3") query-string) + (is (.contains query-string "b=x&b=y&b=z") query-string))) + + (testing "multi-valued query params in indexed-style" + (let [resp (request {:uri "/query-string" + :method :get + :multi-param-style :indexed + :query-params {:a [1 2 3] + :b ["x" "y" "z"]}}) + query-string (-> resp :body form-decode-str)] + (is (= 200 (:status resp))) + (is (.contains query-string "a[0]=1&a[1]=2&a[2]=3") query-string) + (is (.contains query-string "b[0]=x&b[1]=y&b[2]=z") query-string))) + + (testing "multi-valued query params in array-style" + (let [resp (request {:uri "/query-string" + :method :get + :multi-param-style :array + :query-params {:a [1 2 3] + :b ["x" "y" "z"]}}) + query-string (-> resp :body form-decode-str)] + (is (= 200 (:status resp))) + (is (.contains query-string "a[]=1&a[]=2&a[]=3") query-string) + (is (.contains query-string "b[]=x&b[]=y&b[]=z") query-string))) + (testing "multi-valued query params in comma-separated" + (let [resp (request {:uri "/query-string" + :method :get + :multi-param-style :comma-separated + :query-params {:a [1 2 3] + :b ["x" "y" "z"]}}) + query-string (-> resp :body form-decode-str)] + (is (= 200 (:status resp))) + (is (.contains query-string "a=1,2,3") query-string) + (is (.contains query-string "b=x,y,z") query-string)))) + +(deftest t-wrap-flatten-nested-params + (is-applied client/wrap-flatten-nested-params + {} + {:flatten-nested-keys [:query-params]}) + (is-applied client/wrap-flatten-nested-params + {:flatten-nested-keys []} + {:flatten-nested-keys []}) + (is-applied client/wrap-flatten-nested-params + {:flatten-nested-keys [:foo]} + {:flatten-nested-keys [:foo]}) + (is-applied client/wrap-flatten-nested-params + {:ignore-nested-query-string true} + {:ignore-nested-query-string true + :flatten-nested-keys []}) + (is-applied client/wrap-flatten-nested-params + {} + {:flatten-nested-keys '(:query-params)}) + (is-applied client/wrap-flatten-nested-params + {:flatten-nested-form-params true} + {:flatten-nested-form-params true + :flatten-nested-keys '(:query-params :form-params)}) + (is-applied client/wrap-flatten-nested-params + {:flatten-nested-form-params true + :ignore-nested-query-string true} + {:ignore-nested-query-string true + :flatten-nested-form-params true + :flatten-nested-keys '(:form-params)}) + (try + ((client/wrap-flatten-nested-params identity) + {:flatten-nested-form-params true + :ignore-nested-query-string true + :flatten-nested-keys [:thing :bar]}) + (is false "should have thrown exception") + (catch IllegalArgumentException e + (is (= (.getMessage e) + (str "only :flatten-nested-keys or :ignore-nested-query-string/" + ":flatten-nested-keys may be specified, not both"))))) + (try + ((client/wrap-flatten-nested-params identity) + {:ignore-nested-query-string true + :flatten-nested-keys [:thing :bar]}) + (is false "should have thrown exception") + (catch IllegalArgumentException e + (is (= (.getMessage e) + (str "only :flatten-nested-keys or :ignore-nested-query-string/" + ":flatten-nested-keys may be specified, not both")))))) + +(defn transit-resp [body] + {:body body + :status 200 + :headers {"content-type" "application/transit-json"}}) + +(deftest issue-609-empty-transit-response + (testing "Body is available right away" + (is (= {:foo "bar"} + (:body (client/coerce-response-body + {:as :transit+json} + (transit-resp (ByteArrayInputStream. + (.getBytes "[\"^ \",\"~:foo\",\"bar\"]")))))))) + + (testing "Empty body is read as nil" + (is (nil? (:body (client/coerce-response-body + {:as :transit+json} + (transit-resp (ByteArrayInputStream. (.getBytes "")))))))) + + (testing "Body is read correctly even if the data becomes available later" + ;; Ensure both streams are closed (normally done inside future). + (with-open [o (PipedOutputStream.) + i (PipedInputStream.)] + (.connect i o) + (future + (Thread/sleep 10) + (.write o (.getBytes "[\"^ \",\"~:foo\",\"bar\"]")) + ;; Close right now, with-open will wait until test is done. + (.close o)) + (is (= {:foo "bar"} + (:body (client/coerce-response-body + {:as :transit+json} + (transit-resp i)))))))) diff --git a/test/aleph/http/clj_http/core_test.clj b/test/aleph/http/clj_http/core_test.clj new file mode 100644 index 00000000..bb83f83d --- /dev/null +++ b/test/aleph/http/clj_http/core_test.clj @@ -0,0 +1,1003 @@ +(ns aleph.http.clj-http.core-test + (:require [aleph.http.clj-http.util :refer [request]] + [cheshire.core :as json] + [clj-http.client :as client] + [clj-http.conn-mgr :as conn] + [clj-http.core :as core] + [clj-http.util :as util] + [clojure.java.io :refer [file]] + [clojure.test :refer :all] + [ring.adapter.jetty :as ring]) + (:import (java.io ByteArrayInputStream ByteArrayOutputStream FilterInputStream InputStream) + (java.net InetAddress SocketTimeoutException) + (java.util.concurrent TimeoutException TimeUnit) + (org.apache.http HttpConnection HttpInetConnection HttpRequest HttpResponse ProtocolException) + org.apache.http.client.config.RequestConfig + org.apache.http.client.params.ClientPNames + org.apache.http.client.protocol.HttpClientContext + org.apache.http.impl.conn.InMemoryDnsResolver + org.apache.http.impl.cookie.RFC6265CookieSpecProvider + [org.apache.http.message BasicHeader BasicHeaderIterator] + [org.apache.http.params CoreConnectionPNames CoreProtocolPNames] + [org.apache.http.protocol ExecutionContext HttpContext] + org.apache.logging.log4j.LogManager + sun.security.provider.certpath.SunCertPathBuilderException)) + +(set! *warn-on-reflection* false) + +(defonce logger (LogManager/getLogger "clj-http.test.core-test")) + +(defn handler [req] + (condp = [(:request-method req) (:uri req)] + [:get "/get"] + {:status 200 :body "get"} + [:get "/dont-cache"] + {:status 200 :body "nocache" + :headers {"cache-control" "private"}} + [:get "/empty"] + {:status 200 :body nil} + [:get "/empty-gzip"] + {:status 200 :body nil + :headers {"content-encoding" "gzip"}} + [:get "/clojure"] + {:status 200 :body "{:foo \"bar\" :baz 7M :eggplant {:quux #{1 2 3}}}" + :headers {"content-type" "application/clojure"}} + [:get "/edn"] + {:status 200 :body "{:foo \"bar\" :baz 7M :eggplant {:quux #{1 2 3}}}" + :headers {"content-type" "application/edn"}} + [:get "/clojure-bad"] + {:status 200 :body "{:foo \"bar\" :baz #=(+ 1 1)}" + :headers {"content-type" "application/clojure"}} + [:get "/json"] + {:status 200 :body "{\"foo\":\"bar\"}" + :headers {"content-type" "application/json"}} + [:get "/json-array"] + {:status 200 :body "[\"foo\", \"bar\"]" + :headers {"content-type" "application/json"}} + [:get "/json-large-array"] + {:status 200 :body (file "test-resources/big_array_json.json") + :headers {"content-type" "application/json"}} + [:get "/json-bad"] + {:status 400 :body "{\"foo\":\"bar\"}"} + [:get "/redirect"] + {:status 302 + :headers {"location" "http://localhost:18080/redirect"}} + [:get "/bad-redirect"] + {:status 301 :headers {"location" "https:///"}} + [:get "/redirect-to-get"] + {:status 302 + :headers {"location" "http://localhost:18080/get"}} + [:get "/unmodified-resource"] + {:status 304} + [:get "/transit-json"] + {:status 200 :body (str "[\"^ \",\"~:eggplant\",[\"^ \",\"~:quux\"," + "[\"~#set\",[1,3,2]]],\"~:baz\",\"~f7\"," + "\"~:foo\",\"bar\"]") + :headers {"content-type" "application/transit+json"}} + [:get "/transit-json-bad"] + {:status 400 :body "[\"^ \", \"~:foo\",\"bar\"]"} + [:get "/transit-json-empty"] + {:status 200 + :headers {"content-type" "application/transit+json"}} + [:get "/transit-msgpack"] + {:status 200 + :body (->> [-125 -86 126 58 101 103 103 112 108 97 110 116 -127 -90 126 + 58 113 117 117 120 -110 -91 126 35 115 101 116 -109 1 3 2 + -91 126 58 98 97 122 -93 126 102 55 -91 126 58 102 111 111 + -93 98 97 114] + (map byte) + (byte-array) + (ByteArrayInputStream.)) + :headers {"content-type" "application/transit+msgpack"}} + [:head "/head"] + {:status 200} + [:get "/content-type"] + {:status 200 :body (:content-type req)} + [:get "/header"] + {:status 200 :body (get-in req [:headers "x-my-header"])} + [:post "/post"] + {:status 200 :body (:body req)} + [:get "/error"] + {:status 500 :body "o noes"} + [:get "/timeout"] + (do + (Thread/sleep 10) + {:status 200 :body "timeout"}) + [:delete "/delete-with-body"] + {:status 200 :body "delete-with-body"} + [:post "/multipart"] + {:status 200 :body (:body req)} + [:head "/head-with-body"] + {:status 200 :headers {"body" (slurp (:body req))}} + [:get "/get-with-body"] + {:status 200 :body (:body req)} + [:options "/options"] + {:status 200 :body "options"} + [:copy "/copy"] + {:status 200 :body "copy"} + [:move "/move"] + {:status 200 :body "move"} + [:patch "/patch"] + {:status 200 :body "patch"} + [:get "/headers"] + {:status 200 :body (json/encode (:headers req))} + [:propfind "/propfind"] + {:status 200 :body "propfind"} + [:propfind "/propfind-with-body"] + {:status 200 :body (:body req)} + [:get "/query-string"] + {:status 200 :body (:query-string req)} + [:get "/cookie"] + {:status 200 :body "yay" :headers {"Set-Cookie" "foo=bar"}} + [:get "/bad-cookie"] + {:status 200 :body "yay" + :headers + {"Set-Cookie" + (str "DD-PSHARD=3; expires=\"Thu, 12-Apr-2018 06:40:25 GMT\"; " + "Max-Age=604800; Path=/; secure; HttpOnly")}})) + +(defn add-headers-if-requested [client] + (fn [req] + (let [resp (client req) + add-all (-> req :headers (get "add-headers")) + headers (:headers resp)] + (if add-all + (assoc resp :headers (assoc headers "got" (pr-str (:headers req)))) + resp)))) + +(defn run-server + [] + (defonce ^org.eclipse.jetty.server.Server server + (ring/run-jetty (add-headers-if-requested #'handler) {:port 18080 :join? false}))) + +(defn restart-server + [] + (.stop server) + (.start server)) + +(defn localhost [path] + (str "http://localhost:18080" path)) + +(def base-req + {:scheme :http + :server-name "localhost" + :server-port 18080}) + +(defn slurp-body [req] + (slurp (:body req))) + + +(deftest ^:integration makes-get-request + (run-server) + (let [resp (request {:request-method :get :uri "/get"})] + (is (= 200 (:status resp))) + (is (= "get" (slurp-body resp))))) + +(deftest ^:integration dns-resolver + (run-server) + (let [custom-dns-resolver (doto (InMemoryDnsResolver.) + (.add "foo.bar.com" (into-array[(InetAddress/getByAddress (byte-array [127 0 0 1]))]))) + resp (request {:request-method :get :uri "/get" + :server-name "foo.bar.com" + :dns-resolver custom-dns-resolver})] + (is (= 200 (:status resp))) + (is (= "get" (slurp-body resp))))) + +(deftest ^:integration dns-resolver-unknown-host + (run-server) + (let [custom-dns-resolver (doto (InMemoryDnsResolver.) + (.add "foo.bar.com" (into-array[(InetAddress/getByAddress (byte-array [127 0 0 1]))])))] + (is (thrown? java.net.UnknownHostException (request {:request-method :get :uri "/get" + :server-name "www.google.com" + :dns-resolver custom-dns-resolver}))))) + +(deftest ^:integration dns-resolver-reusable-connection-manager + (run-server) + (let [custom-dns-resolver (doto (InMemoryDnsResolver.) + (.add "totallynonexistant.google.com" + (into-array[(InetAddress/getByAddress (byte-array [127 0 0 1]))]))) + cm (conn/make-reuseable-async-conn-manager {:dns-resolver custom-dns-resolver}) + hc (core/build-async-http-client {} cm)] + (client/get "http://totallynonexistant.google.com:18080/json" + {:connection-manager cm + :http-client hc + :as :json + :async true} + (fn [resp] + (is (= 200 (:status resp))) + (is (= {:foo "bar"} (:body resp)))) + (fn [e] (is false (str "failed with " e))))) + (let [custom-dns-resolver (doto (InMemoryDnsResolver.) + (.add "nonexistant.google.com" (into-array[(InetAddress/getByAddress (byte-array [127 0 0 1]))]))) + cm (conn/make-reusable-conn-manager {:dns-resolver custom-dns-resolver}) + hc (:http-client (client/get "http://nonexistant.google.com:18080/get" + {:connection-manager cm})) + resp (client/get "http://nonexistant.google.com:18080/json" + {:connection-manager cm + :http-client hc + :as :json})] + (is (= 200 (:status resp))) + (is (= {:foo "bar"} (:body resp))))) + +(deftest ^:integration save-request-option + (run-server) + (let [resp (request {:request-method :post + :uri "/post" + :body (util/utf8-bytes "contents") + :save-request? true})] + (is (= "/post" (-> resp :request :uri))))) + +(deftest ^:integration makes-head-request + (run-server) + (let [resp (request {:request-method :head :uri "/head"})] + (is (= 200 (:status resp))) + (is (nil? (:body resp))))) + +(deftest ^:integration sets-content-type-with-charset + (run-server) + (let [resp (client/request {:scheme :http + :server-name "localhost" + :server-port 18080 + :request-method :get :uri "/content-type" + :content-type "text/plain" + :character-encoding "UTF-8"})] + (is (= "text/plain; charset=UTF-8" (:body resp))))) + +(deftest ^:integration sets-content-type-without-charset + (run-server) + (let [resp (client/request {:scheme :http + :server-name "localhost" + :server-port 18080 + :request-method :get :uri "/content-type" + :content-type "text/plain"})] + (is (= "text/plain" (:body resp))))) + +(deftest ^:integration sets-arbitrary-headers + (run-server) + (let [resp (request {:request-method :get :uri "/header" + :headers {"x-my-header" "header-val"}})] + (is (= "header-val" (slurp-body resp))))) + +(deftest ^:integration sends-and-returns-byte-array-body + (run-server) + (let [resp (request {:request-method :post :uri "/post" + :body (util/utf8-bytes "contents")})] + (is (= 200 (:status resp))) + (is (= "contents" (slurp-body resp))))) + +(deftest ^:integration returns-arbitrary-headers + (run-server) + (let [resp (request {:request-method :get :uri "/get"})] + (is (string? (get-in resp [:headers "date"]))) + (is (nil? (get-in resp [:headers "Date"]))))) + +(deftest ^:integration returns-status-on-exceptional-responses + (run-server) + (let [resp (request {:request-method :get :uri "/error"})] + (is (= 500 (:status resp))))) + +(deftest ^:integration sets-socket-timeout + (run-server) + (try + (is (thrown? SocketTimeoutException + (client/request {:scheme :http + :server-name "localhost" + :server-port 18080 + :request-method :get :uri "/timeout" + :socket-timeout 1}))))) + +(deftest ^:integration delete-with-body + (run-server) + (let [resp (request {:request-method :delete :uri "/delete-with-body" + :body (.getBytes "foo bar")})] + (is (= 200 (:status resp))))) + +;; Module issue exporting SunCertPathBuilderException +#_ +(deftest ^:integration self-signed-ssl-get + (let [server (ring/run-jetty handler + {:port 8081 :ssl-port 18082 + :ssl? true + :join? false + :keystore "test-resources/keystore" + :key-password "keykey"})] + (try + (is (thrown? SunCertPathBuilderException + (client/request {:scheme :https + :server-name "localhost" + :server-port 18082 + :request-method :get :uri "/get"}))) + (let [resp (request {:request-method :get :uri "/get" :server-port 18082 + :scheme :https :insecure? true})] + (is (= 200 (:status resp))) + (is (= "get" (String. (util/force-byte-array (:body resp)))))) + (finally + (.stop server))))) + +(deftest ^:integration multipart-form-uploads + (run-server) + (let [bytes (util/utf8-bytes "byte-test") + stream (ByteArrayInputStream. bytes) + resp (request {:request-method :post :uri "/multipart" + :multipart [{:name "a" :content "testFINDMEtest" + :encoding "UTF-8" + :mime-type "application/text"} + {:name "b" :content bytes + :mime-type "application/json"} + {:name "d" + :content (file "test-resources/keystore") + :encoding "UTF-8" + :mime-type "application/binary"} + {:name "c" :content stream + :mime-type "application/json"} + {:name "e" :part-name "eggplant" + :content "content" + :mime-type "application/text"}]}) + resp-body (apply str (map #(try (char %) (catch Exception _ "")) + (util/force-byte-array (:body resp))))] + (is (= 200 (:status resp))) + (is (re-find #"testFINDMEtest" resp-body)) + (is (re-find #"application/json" resp-body)) + (is (re-find #"application/text" resp-body)) + (is (re-find #"UTF-8" resp-body)) + (is (re-find #"byte-test" resp-body)) + (is (re-find #"name=\"c\"" resp-body)) + (is (re-find #"name=\"d\"" resp-body)) + (is (re-find #"name=\"eggplant\"" resp-body)) + (is (re-find #"content" resp-body)))) + +(deftest ^:integration multipart-inputstream-length + (run-server) + (let [bytes (util/utf8-bytes "byte-test") + stream (ByteArrayInputStream. bytes) + resp (request {:request-method :post :uri "/multipart" + :multipart [{:name "c" :content stream :length 9 + :mime-type "application/json"}]}) + resp-body (apply str (map #(try (char %) (catch Exception _ "")) + (util/force-byte-array (:body resp))))] + (is (= 200 (:status resp))) + (is (re-find #"byte-test" resp-body)))) + +(deftest parse-headers + (are [headers expected] + (let [iterator (BasicHeaderIterator. + (into-array BasicHeader + (map (fn [[name value]] + (BasicHeader. name value)) + headers)) nil)] + (is (= (core/parse-headers iterator) expected))) + + [] {} + + [["Set-Cookie" "one"]] {"set-cookie" "one"} + + [["Set-Cookie" "one"] ["set-COOKIE" "two"]] + {"set-cookie" ["one" "two"]} + + [["Set-Cookie" "one"] ["serVer" "some-server"] ["set-cookie" "two"]] + {"set-cookie" ["one" "two"] "server" "some-server"})) + +(deftest ^:integration t-streaming-response + (run-server) + (let [stream (:body (request {:request-method :get :uri "/get" :as :stream})) + body (slurp stream)] + (is (= "get" body)))) + + +(deftest ^:integration throw-on-too-many-redirects + (run-server) + (let [resp (client/get (localhost "/redirect") + {:max-redirects 2 :throw-exceptions false + :redirect-strategy :none + :allow-circular-redirects true})] + (is (= 302 (:status resp)))) + + (let [resp (client/get (localhost "/redirect") + {:max-redirects 3 + :redirect-strategy :graceful + :allow-circular-redirects true})] + (is (= 302 (:status resp))) + (is (= 3 (count (:trace-redirects resp)))) + (is (= ["http://localhost:18080/redirect" + "http://localhost:18080/redirect" + "http://localhost:18080/redirect"] + (:trace-redirects resp)))) + + (is (thrown-with-msg? Exception #"Maximum redirects \(2\) exceeded" + (client/get (localhost "/redirect") + {:max-redirects 2 + :throw-exceptions true + :allow-circular-redirects true}))) + (is (thrown-with-msg? Exception #"Maximum redirects \(50\) exceeded" + (client/get (localhost "/redirect") + {:throw-exceptions true + :allow-circular-redirects true})))) + +(deftest ^:integration get-with-body + (run-server) + (let [resp (request {:request-method :get :uri "/get-with-body" + :body (.getBytes "foo bar")})] + (is (= 200 (:status resp))) + (is (= "foo bar" (String. (util/force-byte-array (:body resp))))))) + +(deftest ^:integration head-with-body + (run-server) + (let [resp (request {:request-method :head :uri "/head-with-body" + :body (.getBytes "foo")})] + (is (= 200 (:status resp))) + (is (= "foo" (get-in resp [:headers "body"]))))) + +(deftest ^:integration t-clojure-output-coercion + (run-server) + (let [resp (client/get (localhost "/clojure") {:as :clojure})] + (is (= 200 (:status resp))) + (is (= {:foo "bar" :baz 7M :eggplant {:quux #{1 2 3}}} (:body resp)))) + (let [clj-resp (client/get (localhost "/clojure") {:as :auto}) + edn-resp (client/get (localhost "/edn") {:as :auto})] + (is (= 200 (:status clj-resp) (:status edn-resp))) + (is (= {:foo "bar" :baz 7M :eggplant {:quux #{1 2 3}}} + (:body clj-resp) + (:body edn-resp))))) + +(deftest ^:integration t-transit-output-coercion + (run-server) + (let [transit-json-resp (client/get (localhost "/transit-json") {:as :auto}) + transit-msgpack-resp (client/get (localhost "/transit-msgpack") + {:as :auto}) + bad-status-resp-default + (client/get (localhost "/transit-json-bad") + {:throw-exceptions false :as :transit+json}) + bad-status-resp-always + (client/get (localhost "/transit-json-bad") + {:throw-exceptions false :as :transit+json + :coerce :always}) + bad-status-resp-exceptional + (client/get (localhost "/transit-json-bad") + {:throw-exceptions false :as :transit+json + :coerce :exceptional}) + empty-resp (client/get (localhost "/transit-json-empty") + {:throw-exceptions false :as :transit+json})] + (is (= 200 + (:status transit-json-resp) + (:status transit-msgpack-resp) + (:status empty-resp))) + (is (= 400 + (:status bad-status-resp-default) + (:status bad-status-resp-always) + (:status bad-status-resp-exceptional))) + (is (= {:foo "bar" :baz 7M :eggplant {:quux #{1 2 3}}} + (:body transit-json-resp) + (:body transit-msgpack-resp))) + + (is (nil? (:body empty-resp))) + + (is (= "[\"^ \", \"~:foo\",\"bar\"]" + (:body bad-status-resp-default))) + (is (= {:foo "bar"} + (:body bad-status-resp-always))) + (is (= {:foo "bar"} + (:body bad-status-resp-exceptional))))) + +(deftest ^:integration t-json-output-coercion + (run-server) + (let [resp (client/get (localhost "/json") {:as :json}) + resp-array (client/get (localhost "/json-array") {:as :json}) + resp-array-strict (client/get (localhost "/json-array") {:as :json-strict}) + resp-large-array (client/get (localhost "/json-large-array") {:as :json}) + resp-large-array-strict (client/get (localhost "/json-large-array") {:as :json-strict}) + resp-str (client/get (localhost "/json") + {:as :json :coerce :exceptional}) + resp-str-keys (client/get (localhost "/json") {:as :json-string-keys}) + resp-strict-str-keys (client/get (localhost "/json") + {:as :json-strict-string-keys}) + resp-auto (client/get (localhost "/json") {:as :auto}) + bad-resp (client/get (localhost "/json-bad") + {:throw-exceptions false :as :json}) + bad-resp-json (client/get (localhost "/json-bad") + {:throw-exceptions false :as :json + :coerce :always}) + bad-resp-json2 (client/get (localhost "/json-bad") + {:throw-exceptions false :as :json + :coerce :unexceptional})] + (is (= 200 + (:status resp) + (:status resp-array) + (:status resp-array-strict) + (:status resp-large-array) + (:status resp-large-array-strict) + (:status resp-str) + (:status resp-str-keys) + (:status resp-strict-str-keys) + (:status resp-auto))) + (is (= {:foo "bar"} + (:body resp) + (:body resp-auto))) + (is (= ["foo", "bar"] + (:body resp-array))) + (is (= {"foo" "bar"} + (:body resp-strict-str-keys) + (:body resp-str-keys))) + ;; '("foo" "bar") and ["foo" "bar"] compare as equal with =. + (is (vector? (:body resp-array))) + (is (vector? (:body resp-array-strict))) + (is (= "{\"foo\":\"bar\"}" (:body resp-str))) + (is (= 400 + (:status bad-resp) + (:status bad-resp-json) + (:status bad-resp-json2))) + (is (= "{\"foo\":\"bar\"}" (:body bad-resp)) + "don't coerce on bad response status by default") + (is (= {:foo "bar"} (:body bad-resp-json))) + (is (= "{\"foo\":\"bar\"}" (:body bad-resp-json2))) + + (testing "lazily parsed stream completes parsing." + (is (= 100 (count (:body resp-large-array))))) + (is (= 100 (count (:body resp-large-array-strict)))))) + +(deftest ^:integration t-ipv6 + (run-server) + (let [resp (client/get "http://[::1]:18080/get")] + (is (= 200 (:status resp))) + (is (= "get" (:body resp))))) + +(deftest t-custom-retry-handler + (let [called? (atom false)] + (is (thrown? Exception + (client/post "http://localhost" + {:multipart [{:name "title" :content "Foo"} + {:name "Content/type" + :content "text/plain"} + {:name "file" + :content (file "/tmp/missingfile")}] + :retry-handler (fn [ex try-count http-context] + (reset! called? true) + false)}))) + (is @called?))) + +;; super-basic test for methods that aren't used that often +(deftest ^:integration t-copy-options-move + (run-server) + (let [resp1 (client/options (localhost "/options")) + resp2 (client/move (localhost "/move")) + resp3 (client/copy (localhost "/copy")) + resp4 (client/patch (localhost "/patch"))] + (is (= #{200} (set (map :status [resp1 resp2 resp3 resp4])))) + (is (= "options" (:body resp1))) + (is (= "move" (:body resp2))) + (is (= "copy" (:body resp3))) + (is (= "patch" (:body resp4))))) + +(deftest ^:integration t-json-encoded-form-params + (run-server) + (let [params {:param1 "value1" :param2 {:foo "bar"}} + resp (client/post (localhost "/post") {:content-type :json + :form-params params})] + (is (= 200 (:status resp))) + (is (= (json/encode params) (:body resp))))) + +(deftest ^:integration t-request-interceptor + (run-server) + (let [req-ctx (atom []) + {:keys [status trace-redirects] :as resp} + (client/get + (localhost "/get") + {:request-interceptor + (fn [^HttpRequest req ^HttpContext ctx] + (reset! req-ctx {:method (.getMethod req) :uri (.getURI req)}))})] + (is (= 200 status)) + (is (= "GET" (:method @req-ctx))) + (is (= "/get" (.getPath (:uri @req-ctx)))))) + +(deftest ^:integration t-response-interceptor + (run-server) + (let [saved-ctx (atom []) + {:keys [status trace-redirects] :as resp} + (client/get + (localhost "/redirect-to-get") + {:response-interceptor + (fn [^HttpResponse resp ^HttpContext ctx] + (let [^HttpInetConnection conn + (.getAttribute ctx ExecutionContext/HTTP_CONNECTION)] + (swap! saved-ctx conj {:remote-port (.getRemotePort conn) + :http-conn conn})))})] + (is (= 200 status)) + (is (= 2 (count @saved-ctx))) + #_(is (= (count trace-redirects) (count @saved-ctx))) + (is (every? #(= 18080 (:remote-port %)) @saved-ctx)) + (is (every? #(instance? HttpConnection (:http-conn %)) @saved-ctx)))) + +(deftest ^:integration t-send-input-stream-body + (run-server) + (let [b1 (:body (client/post "http://localhost:18080/post" + {:body (ByteArrayInputStream. (.getBytes "foo")) + :length 3})) + b2 (:body (client/post "http://localhost:18080/post" + {:body (ByteArrayInputStream. + (.getBytes "foo"))})) + b3 (:body (client/post "http://localhost:18080/post" + {:body (ByteArrayInputStream. + (.getBytes "apple")) + :length 2}))] + (is (= b1 "foo")) + (is (= b2 "foo")) + (is (= b3 "ap")))) + +;; (deftest t-add-client-params +;; (testing "Using add-client-params!" +;; (let [ps {"http.conn-manager.timeout" 100 +;; "http.socket.timeout" 250 +;; "http.protocol.allow-circular-redirects" false +;; "http.protocol.version" HttpVersion/HTTP_1_0 +;; "http.useragent" "clj-http"} +;; setps (.getParams (doto (DefaultHttpClient.) +;; (core/add-client-params! ps)))] +;; (doseq [[k v] ps] +;; (is (= v (.getParameter setps k))))))) + +;; Regression, get notified if something changes +(deftest ^:integration t-known-client-params-are-unchanged + (let [params ["http.socket.timeout" CoreConnectionPNames/SO_TIMEOUT + "http.connection.timeout" + CoreConnectionPNames/CONNECTION_TIMEOUT + "http.protocol.version" CoreProtocolPNames/PROTOCOL_VERSION + "http.useragent" CoreProtocolPNames/USER_AGENT + "http.conn-manager.timeout" ClientPNames/CONN_MANAGER_TIMEOUT + "http.protocol.allow-circular-redirects" + ClientPNames/ALLOW_CIRCULAR_REDIRECTS]] + (doseq [[plaintext constant] (partition 2 params)] + (is (= plaintext constant))))) + +;; If you don't explicitly set a :cookie-policy, use +;; CookiePolicy/BROWSER_COMPATIBILITY +;; (deftest t-add-client-params-default-cookie-policy +;; (testing "Using add-client-params! to get a default cookie policy" +;; (let [setps (.getParams (doto (DefaultHttpClient.) +;; (core/add-client-params! {})))] +;; (is (= CookiePolicy/BROWSER_COMPATIBILITY +;; (.getParameter setps ClientPNames/COOKIE_POLICY)))))) + +;; If you set a :cookie-policy, the name of the policy is registered +;; as (str (type cookie-policy)) +;; (deftest t-add-client-params-cookie-policy +;; (testing "Using add-client-params! to get an explicitly set :cookie-policy" +;; (let [setps (.getParams (doto (DefaultHttpClient.) +;; (core/add-client-params! +;; {:cookie-policy (constantly nil)})))] +;; (is (.startsWith ^String +;; (.getParameter setps ClientPNames/COOKIE_POLICY) +;; "class "))))) + + +;; This relies on connections to writequit.org being slower than 10ms, if this +;; fails, you must have very nice internet. +(deftest ^:integration sets-connection-timeout + (run-server) + (try + (is (thrown? SocketTimeoutException + (client/request {:scheme :http + :server-name "writequit.org" + :server-port 80 + :request-method :get :uri "/" + :connection-timeout 10}))))) + +(deftest ^:integration connection-pool-timeout + (run-server) + (client/with-connection-pool {:threads 1 :default-per-route 1} + (let [async-request #(future (client/request {:scheme :http + :server-name "localhost" + :server-port 18080 + :request-method :get + :connection-timeout 1 + :connection-request-timeout 1 + :uri "/timeout"})) + is-pool-timeout-error? + (fn [req-fut] + (instance? org.apache.http.conn.ConnectionPoolTimeoutException + (try @req-fut (catch Exception e (.getCause e))))) + req1 (async-request) + req2 (async-request) + timeout-error1 (is-pool-timeout-error? req1) + timeout-error2 (is-pool-timeout-error? req2)] + (is (or timeout-error1 timeout-error2))))) + +(deftest ^:integration t-header-collections + (run-server) + (let [headers (-> (client/get "http://localhost:18080/headers" + {:headers {"foo" ["bar" "baz"] + "eggplant" "quux"}}) + :body + json/decode)] + (is (= {"eggplant" "quux" "foo" "bar,baz"} + (select-keys headers ["foo" "eggplant"]))))) + +(deftest ^:integration t-clojure-no-read-eval + (run-server) + (is (thrown? Exception (client/get (localhost "/clojure-bad") {:as :clojure})) + "Should throw an exception when reading clojure eval components")) + +(deftest ^:integration t-numeric-headers + (run-server) + (client/request {:method :get :url (localhost "/get") :headers {"foo" 2}})) + +(deftest ^:integration t-empty-response-coercion + (run-server) + (let [resp (client/get (localhost "/empty") {:as :clojure})] + (is (= (:body resp) nil))) + (let [resp (client/get (localhost "/empty") {:as :json})] + (is (= (:body resp) nil))) + (let [resp (client/get (localhost "/empty-gzip") + {:as :clojure})] + (is (= (:body resp) nil))) + (let [resp (client/get (localhost "/empty-gzip") + {:as :json})] + (is (= (:body resp) nil)))) + +(deftest ^:integration t-trace-redirects + (run-server) + (let [resp-with-redirects + (client/request {:method :get + :url (localhost "/redirect-to-get")}) + + resp-with-graceful-redirects + (client/request {:method :get + :url (localhost "/redirect-to-get") + :redirect-strategy :graceful}) + + resp-without-redirects + (client/request {:method :get + :url (localhost "/redirect-to-get") + :redirect-strategy :none})] + + (is (= (:trace-redirects resp-with-redirects) + ["http://localhost:18080/get"])) + + (is (= (:trace-redirects resp-with-graceful-redirects) + ["http://localhost:18080/get"])) + + (is (= (:trace-redirects resp-without-redirects) [])))) + +(deftest t-request-config + (let [params {:conn-timeout 100 ;; deprecated + :connection-timeout 200 ;; takes precedence over `:conn-timeout` + :conn-request-timeout 300 ;; deprecated + :connection-request-timeout 400 ;; takes precedence over `:conn-request-timeout` + :socket-timeout 500 + :max-redirects 600 + :cookie-spec "foo" + :normalize-uri false} + request-config (core/request-config params)] + (is (= 200 (.getConnectTimeout request-config))) + (is (= 400 (.getConnectionRequestTimeout request-config))) + (is (= 500 (.getSocketTimeout request-config))) + (is (= 600 (.getMaxRedirects request-config))) + (is (= core/CUSTOM_COOKIE_POLICY (.getCookieSpec request-config))) + (is (false? (.isNormalizeUri request-config))))) + +(deftest ^:integration t-override-request-config + (run-server) + (let [called-args (atom []) + real-http-client core/build-http-client + http-context (HttpClientContext/create) + request-config (.build (RequestConfig/custom))] + (with-redefs + [core/build-http-client + (fn [& args] + (proxy [org.apache.http.impl.client.CloseableHttpClient] [] + (execute [http-req context] + (swap! called-args conj [http-req context]) + (.execute (apply real-http-client args) http-req context))))] + (client/request {:method :get + :url "http://localhost:18080/get" + :http-client-context http-context + :http-request-config request-config}) + + (let [context-for-request (last (last @called-args))] + (is (= http-context context-for-request)) + (is (= request-config (.getRequestConfig context-for-request))))))) + +(deftest ^:integration test-custom-http-builder-fns + (run-server) + (let [resp (client/get (localhost "/get") + {:headers {"add-headers" "true"} + :http-builder-fns + [(fn [builder req] + (.setDefaultHeaders builder (:hdrs req)))] + :hdrs [(BasicHeader. "foo" "bar")]})] + (is (= 200 (:status resp))) + (is (.contains (get-in resp [:headers "got"]) "\"foo\" \"bar\"") + "Headers should have included the new default headers")) + (let [resp (promise) + error (promise) + f (client/get (localhost "/get") + {:async true + :headers {"add-headers" "true"} + :async-http-builder-fns + [(fn [builder req] + (.setDefaultHeaders builder (:hdrs req)))] + :hdrs [(BasicHeader. "foo" "bar")]} + resp error)] + (.get f) + (is (= 200 (:status @resp))) + (is (.contains (get-in @resp [:headers "got"]) "\"foo\" \"bar\"") + "Headers should have included the new default headers") + (is (not (realized? error))))) + +(deftest ^:integration test-custom-http-client-builder + (run-server) + (let [methods (atom nil) + resp (client/get + (localhost "/get") + {:http-client-builder + (-> (org.apache.http.impl.client.HttpClientBuilder/create) + (.setRequestExecutor + (proxy [org.apache.http.protocol.HttpRequestExecutor] [] + (execute [request connection context] + (->> request + .getRequestLine + .getMethod + (swap! methods conj)) + (proxy-super execute request connection context)))))})] + (is (= ["GET"] @methods)))) + +(deftest ^:integration test-bad-redirects + (run-server) + (try + (client/get (localhost "/bad-redirect")) + (is false "should have thrown an exception") + (catch ProtocolException e + (is (.contains + (.getMessage e) + "Redirect URI does not specify a valid host name: https:///")))) + ;; async version + (let [e (atom nil) + latch (promise)] + (try + (.get + (client/get (localhost "/bad-redirect") {:async true} + (fn [resp] + (is false + (str "should not have been called but got" resp))) + (fn [err] + (reset! e err) + (deliver latch true) + nil))) + (catch Exception error + (is (.contains + (.getMessage error) + "Redirect URI does not specify a valid host name: https:///")))) + @latch + (is (.contains + (.getMessage @e) + "Redirect URI does not specify a valid host name: https:///"))) + (try + (.get (client/get + (localhost "/bad-redirect") + {:async true + :validate-redirects false} + (fn [resp] + (is false + (str "should not have been called but got" resp))) + (fn [err] + (is false + (str "should not have been called but got" err)))) + 1 TimeUnit/SECONDS) + (is false "should have thrown a timeout exception") + (catch TimeoutException te))) + +(deftest ^:integration test-reusable-http-client + (run-server) + (let [cm (conn/make-reuseable-async-conn-manager {}) + hc (core/build-async-http-client {} cm)] + (client/get (localhost "/json") + {:connection-manager cm + :http-client hc + :as :json + :async true} + (fn [resp] + (is (= 200 (:status resp))) + (is (= {:foo "bar"} (:body resp))) + (is (= hc (:http-client resp)) + "http-client is correctly reused")) + (fn [e] (is false (str "failed with " e))))) + (let [cm (conn/make-reusable-conn-manager {}) + hc (:http-client (client/get (localhost "/get") + {:connection-manager cm})) + resp (client/get (localhost "/json") + {:connection-manager cm + :http-client hc + :as :json})] + (is (= 200 (:status resp))) + (is (= {:foo "bar"} (:body resp))) + (is (= hc (:http-client resp)) + "http-client is correctly reused"))) + +(deftest ^:integration t-cookies-spec + (run-server) + (try + (client/get (localhost "/bad-cookie")) + (is false "should have failed") + (catch org.apache.http.cookie.MalformedCookieException e)) + (client/get (localhost "/bad-cookie") {:decode-cookies false}) + (let [validated (atom false) + spec-provider (RFC6265CookieSpecProvider.) + resp (client/get (localhost "/cookie") + {:cookie-spec + (fn [http-context] + (proxy [org.apache.http.impl.cookie.CookieSpecBase] [] + ;; Version and version header + (getVersion [] 0) + (getVersionHeader [] nil) + ;; parse headers into cookie objects + (parse [header cookie-origin] + (.parse (.create spec-provider http-context) + header cookie-origin)) + ;; Validate a cookie, throwing MalformedCookieException if the + ;; cookies isn't valid + (validate [cookie cookie-origin] + (reset! validated true)) + ;; Determine if a cookie matches the target location + (match [cookie cookie-origin] true) + ;; Format a list of cookies into a list of headers + (formatCookies [cookies] (java.util.ArrayList.))))})] + (is (= @validated true)))) + + +(deftest t-cache-config + (let [cc (core/build-cache-config + {:cache-config {:allow-303-caching true + :asynchronous-worker-idle-lifetime-secs 10 + :asynchronous-workers-core 2 + :asynchronous-workers-max 3 + :heuristic-caching-enabled true + :heuristic-coefficient 1.5 + :heuristic-default-lifetime 12 + :max-cache-entries 100 + :max-object-size 123 + :max-update-retries 3 + :revalidation-queue-size 2 + :shared-cache false + :weak-etag-on-put-delete-allowed true}})] + (is (= true (.is303CachingEnabled cc))) + (is (= 10 (.getAsynchronousWorkerIdleLifetimeSecs cc))) + (is (= 2 (.getAsynchronousWorkersCore cc))) + (is (= 3 (.getAsynchronousWorkersMax cc))) + (is (= true (.isHeuristicCachingEnabled cc))) + (is (= 1.5 (.getHeuristicCoefficient cc))) + (is (= 12 (.getHeuristicDefaultLifetime cc))) + (is (= 100 (.getMaxCacheEntries cc))) + (is (= 123 (.getMaxObjectSize cc))) + (is (= 3 (.getMaxUpdateRetries cc))) + (is (= 2 (.getRevalidationQueueSize cc))) + (is (= false (.isSharedCache cc))) + (is (= true (.isWeakETagOnPutDeleteAllowed cc))))) + +(deftest ^:integration t-client-caching + (run-server) + (let [cm (conn/make-reusable-conn-manager {}) + r1 (client/get (localhost "/get") + {:connection-manager cm :cache true}) + client (:http-client r1) + r2 (client/get (localhost "/get") + {:connection-manager cm :http-client client :cache true}) + r3 (client/get (localhost "/get") + {:connection-manager cm :http-client client :cache true}) + r4 (client/get (localhost "/get") + {:connection-manager cm :http-client client :cache true})] + (is (= :CACHE_MISS (:cached r1))) + (is (= :VALIDATED (:cached r2))) + (is (= :VALIDATED (:cached r3))) + (is (= :VALIDATED (:cached r4)))) + (let [cm (conn/make-reusable-conn-manager {}) + r1 (client/get (localhost "/dont-cache") + {:connection-manager cm :cache true}) + client (:http-client r1) + r2 (client/get (localhost "/dont-cache") + {:connection-manager cm :http-client client :cache true}) + r3 (client/get (localhost "/dont-cache") + {:connection-manager cm :http-client client :cache true}) + r4 (client/get (localhost "/dont-cache") + {:connection-manager cm :http-client client :cache true})] + (is (= :CACHE_MISS (:cached r1))) + (is (= :CACHE_MISS (:cached r2))) + (is (= :CACHE_MISS (:cached r3))) + (is (= :CACHE_MISS (:cached r4))))) diff --git a/test/aleph/http/clj_http/util.clj b/test/aleph/http/clj_http/util.clj new file mode 100644 index 00000000..92ec757d --- /dev/null +++ b/test/aleph/http/clj_http/util.clj @@ -0,0 +1,87 @@ +(ns aleph.http.clj-http.util + (:require + [aleph.http :as http] + [clj-commons.byte-streams :as bs] + [clj-http.core :as core] + [clojure.set :as set] + [clojure.test :refer :all]) + (:import + (java.io ByteArrayInputStream + ByteArrayOutputStream + FilterInputStream + InputStream))) + +(def base-req + {:scheme :http + :server-name "localhost" + :server-port 18080}) + +(def uninteresting-headers ["date" "connection" "server"]) + +(defn header-keys + "Returns a set of headers of interest" + [m] + (-> (apply dissoc m uninteresting-headers) + (keys) + (set))) + +(defn is-headers= + "Are the two header maps equal? + + Additional Aleph headers are ignored" + [clj-http-headers aleph-headers] + (let [clj-http-ks (header-keys clj-http-headers) + aleph-ks (header-keys aleph-headers)] + (is (set/superset? aleph-ks clj-http-ks)) + (let [ks-intersection (set/intersection aleph-ks clj-http-ks) + clj-http-common-headers (select-keys clj-http-headers ks-intersection) + aleph-common-headers (select-keys aleph-headers ks-intersection)] + (is (= clj-http-common-headers aleph-common-headers))))) + +(defn is-input-stream= + "Are the two body InputStreams equal? + + Returns a new ByteArrayInputStream based on the consumed original" + [^InputStream clj-http-body ^InputStream aleph-body] + (if clj-http-body + (do + (is (some? aleph-body) "Why is aleph body nil? It should be an empty InputStream for now...") + (let [baos (ByteArrayOutputStream.)] + (.transferTo clj-http-body baos) ; not avail until JDK 9 + + (let [clj-http-body-bytes (.toByteArray baos)] + (is (= (count clj-http-body-bytes) + (.available aleph-body))) + (is (bs/bytes= clj-http-body-bytes aleph-body)) + + (proxy [FilterInputStream] + [^InputStream (ByteArrayInputStream. clj-http-body-bytes)] + (close [] + (.close clj-http-body) + (proxy-super close)))))) + (do + (is (= clj-http-body aleph-body)) + clj-http-body))) + +(defn request + "Modified version of original, that also sends request via Aleph, and + tests the responses for equality." + ([req] + (request req nil nil)) + ([req respond raise] + #_(core/request (merge base-req req) respond raise) ; no aleph + + (if (or respond raise) + ;; do not attempt to compare when using async clj-http...for now + (let [ring-map (merge base-req req)] + (core/request ring-map respond raise)) + + (let [ring-map (merge base-req req) + ;;_ (clojure.pprint/pprint ring-map) + clj-http-resp (core/request ring-map) + ;;_ (clojure.pprint/pprint ring-map) + aleph-resp @(http/request ring-map)] + (is (= (:status clj-http-resp) (:status aleph-resp))) + (is-headers= (:headers clj-http-resp) (:headers aleph-resp)) + (let [new-clj-http-body (is-input-stream= (:body clj-http-resp) (:body aleph-resp))] + (assoc clj-http-resp :body new-clj-http-body)))))) From 071fac48d2905d7f2a1f811237656d17e353c78b Mon Sep 17 00:00:00 2001 From: Matthew Davidson Date: Wed, 14 Sep 2022 18:27:39 +0800 Subject: [PATCH 02/13] Remove unwanted default middleware for clj-http core comparison --- test/aleph/http/clj_http/util.clj | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/aleph/http/clj_http/util.clj b/test/aleph/http/clj_http/util.clj index 92ec757d..3ca7d4ff 100644 --- a/test/aleph/http/clj_http/util.clj +++ b/test/aleph/http/clj_http/util.clj @@ -11,6 +11,9 @@ FilterInputStream InputStream))) +;; turn off default middleware for the core tests +(def no-middleware-pool (http/connection-pool {:middleware identity})) + (def base-req {:scheme :http :server-name "localhost" @@ -80,7 +83,7 @@ ;;_ (clojure.pprint/pprint ring-map) clj-http-resp (core/request ring-map) ;;_ (clojure.pprint/pprint ring-map) - aleph-resp @(http/request ring-map)] + aleph-resp @(http/request (assoc ring-map :pool no-middleware-pool))] (is (= (:status clj-http-resp) (:status aleph-resp))) (is-headers= (:headers clj-http-resp) (:headers aleph-resp)) (let [new-clj-http-body (is-input-stream= (:body clj-http-resp) (:body aleph-resp))] From 7676c4423069f07229881c876b5ffd42d38007b5 Mon Sep 17 00:00:00 2001 From: Matthew Davidson Date: Tue, 20 Sep 2022 15:30:28 +0800 Subject: [PATCH 03/13] Move clj-http deps to dev profile for ease of dev --- project.clj | 64 ++++++++++++++++++++++++++--------------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/project.clj b/project.clj index 7e96f6b4..00cd38e7 100644 --- a/project.clj +++ b/project.clj @@ -2,7 +2,7 @@ (defproject aleph (or (System/getenv "PROJECT_VERSION") "0.5.0") :description "A framework for asynchronous communication" - :repositories {"jboss" "https://repository.jboss.org/nexus/content/groups/public/" + :repositories {"jboss" "https://repository.jboss.org/nexus/content/groups/public/" "sonatype-oss-public" "https://oss.sonatype.org/content/groups/public/"} :url "https://github.com/clj-commons/aleph" :license {:name "MIT License"} @@ -21,47 +21,47 @@ [io.netty/netty-handler-proxy ~netty-version] [io.netty/netty-resolver ~netty-version] [io.netty/netty-resolver-dns ~netty-version]] - :profiles {:dev {:dependencies [[org.clojure/clojure "1.11.1"] - [criterium "0.4.6"] - [cheshire "5.10.0"] - [org.slf4j/slf4j-simple "1.7.30"] - [com.cognitect/transit-clj "1.0.324"] - [spootnik/signal "0.2.4"] - [me.mourjo/dynamic-redef "0.1.0"]]} + :profiles {:dev {:dependencies [[org.clojure/clojure "1.11.1"] + [criterium "0.4.6"] + [cheshire "5.10.0"] + [org.slf4j/slf4j-simple "1.7.30"] + [com.cognitect/transit-clj "1.0.324"] + [spootnik/signal "0.2.4"] + [me.mourjo/dynamic-redef "0.1.0"] + + ;; for testing clj-http parity + [clj-http "3.12.3"] + [ring/ring-jetty-adapter "1.9.3"] + [org.apache.logging.log4j/log4j-api "2.17.1"] + [org.apache.logging.log4j/log4j-core "2.17.1"] + [org.apache.logging.log4j/log4j-1.2-api "2.17.1"]]} :lein-to-deps {:source-paths ["deps"]} ;; This is for self-generating certs for testing ONLY: - :test {:dependencies [[org.bouncycastle/bcprov-jdk15on "1.69"] - [org.bouncycastle/bcpkix-jdk15on "1.69"] - - ;; for testing clj-http parity - [clj-http "3.12.3"] - [ring/ring-jetty-adapter "1.9.3"] - [org.apache.logging.log4j/log4j-api "2.17.1"] - [org.apache.logging.log4j/log4j-core "2.17.1"] - [org.apache.logging.log4j/log4j-1.2-api "2.17.1"]] - :javac-options ^:replace ["--release" "12"] ; necessary for some tests - :jvm-opts ["-Dorg.slf4j.simpleLogger.defaultLogLevel=off"]}} - :codox {:src-dir-uri "https://github.com/ztellman/aleph/tree/master/" + :test {:dependencies [[org.bouncycastle/bcprov-jdk15on "1.69"] + [org.bouncycastle/bcpkix-jdk15on "1.69"]] + :javac-options ^:replace ["--release" "12"] ; necessary for some tests + :jvm-opts ["-Dorg.slf4j.simpleLogger.defaultLogLevel=off"]}} + :codox {:src-dir-uri "https://github.com/ztellman/aleph/tree/master/" :src-linenum-anchor-prefix "L" - :defaults {:doc/format :markdown} - :include [aleph.tcp - aleph.udp - aleph.http - aleph.flow] - :output-dir "doc"} + :defaults {:doc/format :markdown} + :include [aleph.tcp + aleph.udp + aleph.http + aleph.flow] + :output-dir "doc"} :plugins [[lein-codox "0.10.7"] [lein-marginalia "0.9.1"] [lein-pprint "1.3.2"] [ztellman/lein-cljfmt "0.1.10"]] :java-source-paths ["src/aleph/utils"] :cljfmt {:indents {#".*" [[:inner 0]]}} - :test-selectors {:default #(not - (some #{:benchmark :stress :integration} - (cons (:tag %) (keys %)))) - :benchmark :benchmark + :test-selectors {:default #(not + (some #{:benchmark :stress :integration} + (cons (:tag %) (keys %)))) + :benchmark :benchmark :integration :integration - :stress :stress - :all (constantly true)} + :stress :stress + :all (constantly true)} :jvm-opts ^:replace ["-server" "-Xmx2g" "-XX:+HeapDumpOnOutOfMemoryError" From a2beb80a7618ab37f3a4e4e9edf46f5695b69485 Mon Sep 17 00:00:00 2001 From: Matthew Davidson Date: Tue, 20 Sep 2022 15:32:14 +0800 Subject: [PATCH 04/13] Replace clj-http core tests Accidentally used unreleased tests; head-with-body relied on unreleased code --- test/aleph/http/clj_http/core_test.clj | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/test/aleph/http/clj_http/core_test.clj b/test/aleph/http/clj_http/core_test.clj index bb83f83d..40145d63 100644 --- a/test/aleph/http/clj_http/core_test.clj +++ b/test/aleph/http/clj_http/core_test.clj @@ -107,8 +107,6 @@ {:status 200 :body "delete-with-body"} [:post "/multipart"] {:status 200 :body (:body req)} - [:head "/head-with-body"] - {:status 200 :headers {"body" (slurp (:body req))}} [:get "/get-with-body"] {:status 200 :body (:body req)} [:options "/options"] @@ -150,11 +148,6 @@ (defonce ^org.eclipse.jetty.server.Server server (ring/run-jetty (add-headers-if-requested #'handler) {:port 18080 :join? false}))) -(defn restart-server - [] - (.stop server) - (.start server)) - (defn localhost [path] (str "http://localhost:18080" path)) @@ -166,7 +159,6 @@ (defn slurp-body [req] (slurp (:body req))) - (deftest ^:integration makes-get-request (run-server) (let [resp (request {:request-method :get :uri "/get"})] @@ -422,10 +414,8 @@ (deftest ^:integration head-with-body (run-server) - (let [resp (request {:request-method :head :uri "/head-with-body" - :body (.getBytes "foo")})] - (is (= 200 (:status resp))) - (is (= "foo" (get-in resp [:headers "body"]))))) + (let [resp (request {:request-method :head :uri "/head" :body "foo"})] + (is (= 200 (:status resp))))) (deftest ^:integration t-clojure-output-coercion (run-server) From 0254c6ec337677e036ea52f39733a1fc20e2f294 Mon Sep 17 00:00:00 2001 From: Matthew Davidson Date: Tue, 20 Sep 2022 15:33:15 +0800 Subject: [PATCH 05/13] Minor refactoring and cleanup --- test/aleph/http/clj_http/util.clj | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/test/aleph/http/clj_http/util.clj b/test/aleph/http/clj_http/util.clj index 3ca7d4ff..54773b97 100644 --- a/test/aleph/http/clj_http/util.clj +++ b/test/aleph/http/clj_http/util.clj @@ -2,7 +2,7 @@ (:require [aleph.http :as http] [clj-commons.byte-streams :as bs] - [clj-http.core :as core] + [clj-http.core :as clj-http] [clojure.set :as set] [clojure.test :refer :all]) (:import @@ -19,12 +19,12 @@ :server-name "localhost" :server-port 18080}) -(def uninteresting-headers ["date" "connection" "server"]) +(def ignored-headers ["date" "connection" "server"]) (defn header-keys "Returns a set of headers of interest" [m] - (-> (apply dissoc m uninteresting-headers) + (-> (apply dissoc m ignored-headers) (keys) (set))) @@ -72,18 +72,15 @@ ([req] (request req nil nil)) ([req respond raise] - #_(core/request (merge base-req req) respond raise) ; no aleph - (if (or respond raise) ;; do not attempt to compare when using async clj-http...for now (let [ring-map (merge base-req req)] - (core/request ring-map respond raise)) + (clj-http/request ring-map respond raise)) - (let [ring-map (merge base-req req) - ;;_ (clojure.pprint/pprint ring-map) - clj-http-resp (core/request ring-map) - ;;_ (clojure.pprint/pprint ring-map) - aleph-resp @(http/request (assoc ring-map :pool no-middleware-pool))] + (let [clj-http-ring-map (merge base-req req) + aleph-ring-map (merge base-req req {:pool no-middleware-pool}) + clj-http-resp (clj-http/request clj-http-ring-map) + aleph-resp @(http/request aleph-ring-map)] (is (= (:status clj-http-resp) (:status aleph-resp))) (is-headers= (:headers clj-http-resp) (:headers aleph-resp)) (let [new-clj-http-body (is-input-stream= (:body clj-http-resp) (:body aleph-resp))] From d1de95cf47e63936fe90ac0183d6118821d033b6 Mon Sep 17 00:00:00 2001 From: Matthew Davidson Date: Tue, 20 Sep 2022 15:33:43 +0800 Subject: [PATCH 06/13] Remove unreleased tests from clj-http client-test --- test/aleph/http/clj_http/client_test.clj | 42 ------------------------ 1 file changed, 42 deletions(-) diff --git a/test/aleph/http/clj_http/client_test.clj b/test/aleph/http/clj_http/client_test.clj index ba5c33ff..932ddf5b 100644 --- a/test/aleph/http/clj_http/client_test.clj +++ b/test/aleph/http/clj_http/client_test.clj @@ -13,8 +13,6 @@ [ring.util.codec :refer [form-decode-str]] [slingshot.slingshot :refer [try+]]) (:import java.io.ByteArrayInputStream - java.io.PipedInputStream - java.io.PipedOutputStream java.net.UnknownHostException org.apache.http.HttpEntity org.apache.logging.log4j.LogManager)) @@ -28,13 +26,6 @@ :server-name "localhost" :server-port 18080}) -#_ -(defn request - ([req] - (client/request (merge base-req req))) - ([req respond raise] - (client/request (merge base-req req) respond raise))) - (defn parse-form-params [s] (->> (str/split (form-decode-str s) #"&") (map #(str/split % #"=")) @@ -1760,36 +1751,3 @@ (is (= (.getMessage e) (str "only :flatten-nested-keys or :ignore-nested-query-string/" ":flatten-nested-keys may be specified, not both")))))) - -(defn transit-resp [body] - {:body body - :status 200 - :headers {"content-type" "application/transit-json"}}) - -(deftest issue-609-empty-transit-response - (testing "Body is available right away" - (is (= {:foo "bar"} - (:body (client/coerce-response-body - {:as :transit+json} - (transit-resp (ByteArrayInputStream. - (.getBytes "[\"^ \",\"~:foo\",\"bar\"]")))))))) - - (testing "Empty body is read as nil" - (is (nil? (:body (client/coerce-response-body - {:as :transit+json} - (transit-resp (ByteArrayInputStream. (.getBytes "")))))))) - - (testing "Body is read correctly even if the data becomes available later" - ;; Ensure both streams are closed (normally done inside future). - (with-open [o (PipedOutputStream.) - i (PipedInputStream.)] - (.connect i o) - (future - (Thread/sleep 10) - (.write o (.getBytes "[\"^ \",\"~:foo\",\"bar\"]")) - ;; Close right now, with-open will wait until test is done. - (.close o)) - (is (= {:foo "bar"} - (:body (client/coerce-response-body - {:as :transit+json} - (transit-resp i)))))))) From 7cd65ef3feaa9424712031ff729108f94e330e12 Mon Sep 17 00:00:00 2001 From: Matthew Davidson Date: Tue, 4 Oct 2022 17:41:20 +0800 Subject: [PATCH 07/13] Better middleware testing Mimic clj-http tests' use of middleware Translate clj-http's middleware to Aleph's when possible Split default middleware into client and request parts - most middleware changes req maps, but wrap-exceptions and wrap-request-timing change the client fn. Add missing :reason-phrase resp key Add missing :url wrap-url key Directly compare response bodies as bytes when not InputStreams Ignore DSN resolution tests --- project.clj | 2 +- src/aleph/http.clj | 2 +- src/aleph/http/client_middleware.clj | 57 ++++--- src/aleph/http/core.clj | 5 +- test/aleph/http/clj_http/client_test.clj | 4 +- test/aleph/http/clj_http/core_test.clj | 10 +- test/aleph/http/clj_http/util.clj | 182 ++++++++++++++++++----- 7 files changed, 190 insertions(+), 72 deletions(-) diff --git a/project.clj b/project.clj index 00cd38e7..7a6eb53c 100644 --- a/project.clj +++ b/project.clj @@ -56,7 +56,7 @@ :java-source-paths ["src/aleph/utils"] :cljfmt {:indents {#".*" [[:inner 0]]}} :test-selectors {:default #(not - (some #{:benchmark :stress :integration} + (some #{:benchmark :stress :integration :ignore} (cons (:tag %) (keys %)))) :benchmark :benchmark :integration :integration diff --git a/src/aleph/http.clj b/src/aleph/http.clj index 060f58f1..44ba97e8 100644 --- a/src/aleph/http.clj +++ b/src/aleph/http.clj @@ -102,7 +102,7 @@ | `max-queue-size` | the maximum number of pending acquires from the pool that are allowed before `acquire` will start to throw a `java.util.concurrent.RejectedExecutionException`, defaults to `65536` | `control-period` | the interval, in milliseconds, between use of the controller to adjust the size of the pool, defaults to `60000` | `dns-options` | an optional map with async DNS resolver settings, for more information check `aleph.netty/dns-resolver-group`. When set, ignores `name-resolver` setting from `connection-options` in favor of shared DNS resolver instance - | `middleware` | a function to modify request before sending, defaults to `aleph.http.client-middleware/wrap-request` + | `middleware` | a function to modify clients/requests before sending, defaults to `aleph.http.client-middleware/wrap-request` | `pool-builder-fn` | an optional one arity function which returns a `io.aleph.dirigiste.IPool` from a map containing the following keys: `generate`, `destroy`, `control-period`, `max-queue-length` and `stats-callback`. | `pool-controller-builder-fn` | an optional zero arity function which returns a `io.aleph.dirigiste.IPool$Controller`. diff --git a/src/aleph/http/client_middleware.clj b/src/aleph/http/client_middleware.clj index f9082de6..4a4c1c34 100644 --- a/src/aleph/http/client_middleware.clj +++ b/src/aleph/http/client_middleware.clj @@ -155,6 +155,7 @@ {:scheme (keyword (.getProtocol url-parsed)) :server-name (.getHost url-parsed) :server-port (when-pos (.getPort url-parsed)) + :url url :uri (url-encode-illegal-characters (.getPath url-parsed)) :user-info (when-let [user-info (.getUserInfo url-parsed)] (URLDecoder/decode user-info)) @@ -243,12 +244,13 @@ (if (unexceptional-status? status) rsp (cond - (false? (opt req :throw-exceptions)) rsp (instance? InputStream body) - (d/chain' (d/future (bs/to-byte-array body)) + (d/chain' + (d/future + (bs/to-byte-array body)) (fn [body] (d/error-deferred (ex-info @@ -563,9 +565,8 @@ [req] (if-let [url (:url req)] (-> req - (dissoc :url) - (assoc :request-url url) - (merge (parse-url url))) + (assoc :request-url url) + (merge (parse-url url))) req)) (defn wrap-request-timing @@ -918,7 +919,13 @@ (opt req :save-request) (assoc :aleph/request req')))) +(def default-client-middleware + "Default middleware that takes a client fn" + [wrap-exceptions + wrap-request-timing]) + (def default-middleware + "Default middleware that takes a request map" [wrap-method wrap-url wrap-nested-params @@ -937,22 +944,24 @@ "Returns a batteries-included HTTP request function corresponding to the given core client. See default-middleware for the middleware wrappers that are used by default" - [client] - (let [client' (-> client - wrap-exceptions - wrap-request-timing)] - (fn [req] - (let [executor (ex/executor)] - (if (:aleph.http.client/close req) - (client req) - - (let [req' (reduce #(%2 %1) req default-middleware)] - (d/chain' (client' req') - - ;; coerce the response body - (fn [{:keys [body] :as rsp}] - (let [rsp' (handle-response-debug req' rsp)] - (if (and (some? body) (some? (:as req'))) - (d/future-with (or executor (ex/wait-pool)) - (coerce-response-body req' rsp')) - rsp')))))))))) + ([client] + (wrap-request client default-client-middleware default-middleware)) + ([client client-middleware middleware] + (let [client' (reduce #(%2 %1) + client + client-middleware)] + (fn [req] + (let [executor (ex/executor)] + (if (:aleph.http.client/close req) + (client req) + + (let [req' (reduce #(%2 %1) req middleware)] + (d/chain' (client' req') + + ;; coerce the response body + (fn [{:keys [body] :as rsp}] + (let [rsp' (handle-response-debug req' rsp)] + (if (and (some? body) (some? (:as req'))) + (d/future-with (or executor (ex/wait-pool)) + (coerce-response-body req' rsp')) + rsp'))))))))))) diff --git a/src/aleph/http/core.clj b/src/aleph/http/core.clj index 493a41bb..bba08e27 100644 --- a/src/aleph/http/core.clj +++ b/src/aleph/http/core.clj @@ -224,9 +224,10 @@ :remote-addr (netty/channel-remote-address ch)) (p/def-derived-map NettyResponse [^HttpResponse rsp complete body] - :status (-> rsp .status .code) + :status (-> rsp (.status) (.code)) + :reason-phrase (-> rsp (.status) (.reasonPhrase)) :aleph/keep-alive? (HttpUtil/isKeepAlive rsp) - :headers (-> rsp .headers headers->map) + :headers (-> rsp (.headers) headers->map) :aleph/complete complete :body body) diff --git a/test/aleph/http/clj_http/client_test.clj b/test/aleph/http/clj_http/client_test.clj index 932ddf5b..715fe0b9 100644 --- a/test/aleph/http/clj_http/client_test.clj +++ b/test/aleph/http/clj_http/client_test.clj @@ -1,6 +1,6 @@ (ns aleph.http.clj-http.client-test (:require [aleph.http.clj-http.core-test :refer [run-server]] - [aleph.http.clj-http.util :refer [request]] + [aleph.http.clj-http.util :refer [make-request]] [cheshire.core :as json] [clj-http.client :as client] [clj-http.conn-mgr :as conn] @@ -26,6 +26,8 @@ :server-name "localhost" :server-port 18080}) +(def request (make-request #'client/request {:using-middleware? true})) + (defn parse-form-params [s] (->> (str/split (form-decode-str s) #"&") (map #(str/split % #"=")) diff --git a/test/aleph/http/clj_http/core_test.clj b/test/aleph/http/clj_http/core_test.clj index 40145d63..47004bf6 100644 --- a/test/aleph/http/clj_http/core_test.clj +++ b/test/aleph/http/clj_http/core_test.clj @@ -1,5 +1,5 @@ (ns aleph.http.clj-http.core-test - (:require [aleph.http.clj-http.util :refer [request]] + (:require [aleph.http.clj-http.util :refer [make-request]] [cheshire.core :as json] [clj-http.client :as client] [clj-http.conn-mgr :as conn] @@ -156,6 +156,8 @@ :server-name "localhost" :server-port 18080}) +(def request (make-request #'core/request {:using-middleware? false})) + (defn slurp-body [req] (slurp (:body req))) @@ -165,7 +167,7 @@ (is (= 200 (:status resp))) (is (= "get" (slurp-body resp))))) -(deftest ^:integration dns-resolver +(deftest ^:ignore dns-resolver (run-server) (let [custom-dns-resolver (doto (InMemoryDnsResolver.) (.add "foo.bar.com" (into-array[(InetAddress/getByAddress (byte-array [127 0 0 1]))]))) @@ -175,7 +177,7 @@ (is (= 200 (:status resp))) (is (= "get" (slurp-body resp))))) -(deftest ^:integration dns-resolver-unknown-host +(deftest ^:ignore dns-resolver-unknown-host (run-server) (let [custom-dns-resolver (doto (InMemoryDnsResolver.) (.add "foo.bar.com" (into-array[(InetAddress/getByAddress (byte-array [127 0 0 1]))])))] @@ -183,7 +185,7 @@ :server-name "www.google.com" :dns-resolver custom-dns-resolver}))))) -(deftest ^:integration dns-resolver-reusable-connection-manager +(deftest ^:ignore dns-resolver-reusable-connection-manager (run-server) (let [custom-dns-resolver (doto (InMemoryDnsResolver.) (.add "totallynonexistant.google.com" diff --git a/test/aleph/http/clj_http/util.clj b/test/aleph/http/clj_http/util.clj index 54773b97..35318387 100644 --- a/test/aleph/http/clj_http/util.clj +++ b/test/aleph/http/clj_http/util.clj @@ -1,10 +1,14 @@ (ns aleph.http.clj-http.util (:require [aleph.http :as http] + [aleph.http.client-middleware :as aleph.mid] [clj-commons.byte-streams :as bs] [clj-http.core :as clj-http] + [clj-http.client] [clojure.set :as set] - [clojure.test :refer :all]) + [clojure.string :as str] + [clojure.test :refer :all] + [clojure.tools.logging :as log]) (:import (java.io ByteArrayInputStream ByteArrayOutputStream @@ -41,47 +45,147 @@ aleph-common-headers (select-keys aleph-headers ks-intersection)] (is (= clj-http-common-headers aleph-common-headers))))) -(defn is-input-stream= - "Are the two body InputStreams equal? +(defn bodies= + "Are the two bodies equal? clj-http's client/request fn coerces to strings by default, + while the core/request leaves the body an InputStream. + Aleph, in keeping with it's stream-based nature, leaves as an InputStream by default. - Returns a new ByteArrayInputStream based on the consumed original" - [^InputStream clj-http-body ^InputStream aleph-body] + If an InputStream, returns a new ByteArrayInputStream based on the consumed original" + [clj-http-body ^InputStream aleph-body] (if clj-http-body - (do - (is (some? aleph-body) "Why is aleph body nil? It should be an empty InputStream for now...") - (let [baos (ByteArrayOutputStream.)] - (.transferTo clj-http-body baos) ; not avail until JDK 9 - - (let [clj-http-body-bytes (.toByteArray baos)] - (is (= (count clj-http-body-bytes) - (.available aleph-body))) - (is (bs/bytes= clj-http-body-bytes aleph-body)) - - (proxy [FilterInputStream] - [^InputStream (ByteArrayInputStream. clj-http-body-bytes)] - (close [] - (.close clj-http-body) - (proxy-super close)))))) + (condp instance? clj-http-body + InputStream + (do + (is (some? aleph-body) "Why is aleph body nil? It should be an empty InputStream for now...") + (let [baos (ByteArrayOutputStream.)] + (.transferTo clj-http-body baos) ; not avail until JDK 9 + + (let [clj-http-body-bytes (.toByteArray baos)] + (is (= (count clj-http-body-bytes) + (.available aleph-body))) + (is (bs/bytes= clj-http-body-bytes aleph-body)) + + (proxy [FilterInputStream] + [^InputStream (ByteArrayInputStream. clj-http-body-bytes)] + (close [] + (.close clj-http-body) + (proxy-super close)))))) + + (do + (is (bs/bytes= clj-http-body aleph-body)) + clj-http-body)) (do (is (= clj-http-body aleph-body)) clj-http-body))) -(defn request - "Modified version of original, that also sends request via Aleph, and - tests the responses for equality." - ([req] - (request req nil nil)) - ([req respond raise] - (if (or respond raise) - ;; do not attempt to compare when using async clj-http...for now - (let [ring-map (merge base-req req)] - (clj-http/request ring-map respond raise)) - - (let [clj-http-ring-map (merge base-req req) - aleph-ring-map (merge base-req req {:pool no-middleware-pool}) - clj-http-resp (clj-http/request clj-http-ring-map) - aleph-resp @(http/request aleph-ring-map)] - (is (= (:status clj-http-resp) (:status aleph-resp))) - (is-headers= (:headers clj-http-resp) (:headers aleph-resp)) - (let [new-clj-http-body (is-input-stream= (:body clj-http-resp) (:body aleph-resp))] - (assoc clj-http-resp :body new-clj-http-body)))))) + +(defn- defined-middleware + "Returns a set of symbols beginning with `wrap-` in the ns" + [ns] + (->> (ns-publics ns) + keys + (map str) + (filter #(str/starts-with? % "wrap-")) + (map symbol) + set)) + +(defn- aleph-test-conn-pool + "clj-http middleware is traditional fn-based middleware, using a 3-arity version to handle async. + + Aleph usually uses a more async-friendly interceptor-style, where the middleware transforms the request maps, + but does nothing about calling the next fn in the chain. + + Unfortunately, a couple middleware cannot be converted to interceptor-style, complicating things." + [middleware-list] + (let [missing-midw (set/difference + (defined-middleware 'clj-http.client) + (defined-middleware 'aleph.http.client-middleware))] + (when-not (seq missing-midw) + (println "clj-http is using middleware that aleph lacks:") + (prn missing-midw) + (log/warn "clj-http is using middleware that aleph lacks" + :missing-middleware missing-midw))) + (let [non-interceptor-middleware (set aleph.mid/default-client-middleware) + client-middleware (cond-> [] + (some #{clj-http.client/wrap-exceptions} middleware-list) + (conj aleph.mid/wrap-exceptions) + + (some #{clj-http.client/wrap-request-timing} middleware-list) + (conj aleph.mid/wrap-request-timing)) + middleware-list' (->> middleware-list + (map (fn [midw] + (-> midw + class + str + (str/split #"\$") + peek + (str/replace "_" "-") + (->> (symbol "aleph.http.client-middleware")) + requiring-resolve))) + (filter some?) + (map var-get) + (remove non-interceptor-middleware) + vec)] + ;;(println "Client-based middleware:") + ;;(prn client-middleware) + ;;(println "Regular middleware:") + ;;(prn middleware-list') + (http/connection-pool {:middleware #(aleph.mid/wrap-request % client-middleware middleware-list')}))) + + +(defn- print-middleware-list + [middleware-list] + (prn (mapv (fn [midw] + (-> midw + class + str + (str/split #"\$") + peek + (str/replace "_" "-") + symbol)) + middleware-list))) + +(defn make-request + "Need to switch between clj-http's core/request and client/request. + + Modified version of original request fns, that also sends requests + via Aleph, and tests the responses for equality." + [clj-http-request {:keys [using-middleware?]}] + (fn compare-request + ([req] + (compare-request req nil nil)) + ([req respond raise] + (if (or respond raise) + ;; do not attempt to compare when using async clj-http...for now + (let [ring-map (merge base-req req)] + (clj-http-request ring-map respond raise)) + + (let [clj-http-ring-map (merge base-req req) + ;;_ (prn clj-http-ring-map) + clj-http-middleware (if using-middleware? clj-http.client/*current-middleware* []) + ;;_ (print-middleware-list clj-http.client/*current-middleware*) + aleph-ring-map (merge base-req req {:pool (aleph-test-conn-pool clj-http-middleware)}) + ;;_ (prn aleph-ring-map) + clj-http-resp (clj-http-request clj-http-ring-map) + aleph-resp @(http/request aleph-ring-map)] + (is (= (:status clj-http-resp) (:status aleph-resp))) + + + #_(when (not= (:status clj-http-resp) (:status aleph-resp)) + (println "clj-http req:") + (prn clj-http-ring-map) + (println) + (println "clj-http resp:") + (prn clj-http-resp) + (println) + (println) + (println "aleph req:") + (prn aleph-ring-map) + (println) + (println "aleph resp:") + (prn aleph-resp)) + + (is-headers= (:headers clj-http-resp) (:headers aleph-resp)) + (is (instance? InputStream (:body aleph-resp))) + (let [new-clj-http-body (bodies= (:body clj-http-resp) (:body aleph-resp))] + (assoc clj-http-resp :body new-clj-http-body))))))) From 04a71cd63aa3311cdf0697eccd2c7cabce41fb58 Mon Sep 17 00:00:00 2001 From: Matthew Davidson Date: Tue, 4 Oct 2022 18:07:17 +0800 Subject: [PATCH 08/13] Add clj-http test predicate --- project.clj | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/project.clj b/project.clj index 7a6eb53c..28fe4831 100644 --- a/project.clj +++ b/project.clj @@ -61,6 +61,10 @@ :benchmark :benchmark :integration :integration :stress :stress + :clj-http [(fn clj-http-ns-pred [namespc & _] + (.contains (str namespc) "clj-http")) + (fn clj-http-test-pred [m & _] + (not (:ignore m)))] :all (constantly true)} :jvm-opts ^:replace ["-server" "-Xmx2g" From 6c402f97bfce01cc691e2e5e9292f843d57e4a99 Mon Sep 17 00:00:00 2001 From: Matthew Davidson Date: Thu, 6 Oct 2022 21:36:50 +0800 Subject: [PATCH 09/13] WIP on supporting client/* --- test/aleph/http/clj_http/client_test.clj | 2 +- test/aleph/http/clj_http/core_test.clj | 8 +++++++- test/aleph/http/clj_http/util.clj | 20 ++++++++++++++------ 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/test/aleph/http/clj_http/client_test.clj b/test/aleph/http/clj_http/client_test.clj index 715fe0b9..52a35f48 100644 --- a/test/aleph/http/clj_http/client_test.clj +++ b/test/aleph/http/clj_http/client_test.clj @@ -26,7 +26,7 @@ :server-name "localhost" :server-port 18080}) -(def request (make-request #'client/request {:using-middleware? true})) +(def request (make-request client/request {:using-middleware? true})) (defn parse-form-params [s] (->> (str/split (form-decode-str s) #"&") diff --git a/test/aleph/http/clj_http/core_test.clj b/test/aleph/http/clj_http/core_test.clj index 47004bf6..7e2fea77 100644 --- a/test/aleph/http/clj_http/core_test.clj +++ b/test/aleph/http/clj_http/core_test.clj @@ -156,7 +156,13 @@ :server-name "localhost" :server-port 18080}) -(def request (make-request #'core/request {:using-middleware? false})) + +(def request (make-request core/request {:using-middleware? false})) + +(use-fixtures :once + (fn [f] + (binding [client/request (make-request client/request {:using-middleware? true})] + (f)))) (defn slurp-body [req] (slurp (:body req))) diff --git a/test/aleph/http/clj_http/util.clj b/test/aleph/http/clj_http/util.clj index 35318387..d20cfecb 100644 --- a/test/aleph/http/clj_http/util.clj +++ b/test/aleph/http/clj_http/util.clj @@ -28,9 +28,10 @@ (defn header-keys "Returns a set of headers of interest" [m] - (-> (apply dissoc m ignored-headers) - (keys) - (set))) + (->> (apply dissoc m ignored-headers) + (keys) + (map str/lower-case) + (set))) (defn is-headers= "Are the two header maps equal? @@ -71,9 +72,15 @@ (.close clj-http-body) (proxy-super close)))))) - (do - (is (bs/bytes= clj-http-body aleph-body)) - clj-http-body)) + (try + (do + (is (bs/bytes= clj-http-body aleph-body)) + clj-http-body) + (catch Exception e + (println "clj-http body class: " (class clj-http-body)) + (prn clj-http-body) + (flush) + (throw e)))) (do (is (= clj-http-body aleph-body)) clj-http-body))) @@ -170,6 +177,7 @@ aleph-resp @(http/request aleph-ring-map)] (is (= (:status clj-http-resp) (:status aleph-resp))) + (prn aleph-resp) #_(when (not= (:status clj-http-resp) (:status aleph-resp)) (println "clj-http req:") From 9478392771c0750b528e85970f43ad4e16bca27e Mon Sep 17 00:00:00 2001 From: Arnaud Geiser Date: Sat, 8 Oct 2022 15:03:26 +0200 Subject: [PATCH 10/13] Add missing m.txt file --- test-resources/m.txt | 4 ++++ test/aleph/http/clj_http/client_test.clj | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 test-resources/m.txt diff --git a/test-resources/m.txt b/test-resources/m.txt new file mode 100644 index 00000000..8e6ce25e --- /dev/null +++ b/test-resources/m.txt @@ -0,0 +1,4 @@ +this +is +some +file. diff --git a/test/aleph/http/clj_http/client_test.clj b/test/aleph/http/clj_http/client_test.clj index 715fe0b9..5fb5e524 100644 --- a/test/aleph/http/clj_http/client_test.clj +++ b/test/aleph/http/clj_http/client_test.clj @@ -155,8 +155,8 @@ :content (clojure.java.io/file "test-resources/m.txt")}]} resp - exception - )] + exception)] + (is (= 200 (:status @resp))) (is (not (realized? exception))) #_(when (realized? exception) (prn @exception))) From a94bc5f6e65b8ce5b08b249d88552f8a956b169f Mon Sep 17 00:00:00 2001 From: Matthew Davidson Date: Sat, 22 Oct 2022 18:16:30 +0800 Subject: [PATCH 11/13] Compare multipart requests --- src/aleph/http/multipart.clj | 2 +- test/aleph/http/clj_http/core_test.clj | 4 +- test/aleph/http/clj_http/util.clj | 118 +++++++++++++++++++++++-- 3 files changed, 115 insertions(+), 9 deletions(-) diff --git a/src/aleph/http/multipart.clj b/src/aleph/http/multipart.clj index 0f615385..4025fe57 100644 --- a/src/aleph/http/multipart.clj +++ b/src/aleph/http/multipart.clj @@ -201,7 +201,7 @@ (defn decode-request "Takes a ring request and returns a manifold stream which yields - parts of the mutlipart/form-data encoded body. In case the size of + parts of the multipart/form-data encoded body. In case the size of a part content exceeds `:memory-limit` limit (16KB by default), corresponding payload would be written to a temp file. Check `:memory?` flag to know whether content might be read directly from `:content` or diff --git a/test/aleph/http/clj_http/core_test.clj b/test/aleph/http/clj_http/core_test.clj index 7e2fea77..88a726f9 100644 --- a/test/aleph/http/clj_http/core_test.clj +++ b/test/aleph/http/clj_http/core_test.clj @@ -106,7 +106,8 @@ [:delete "/delete-with-body"] {:status 200 :body "delete-with-body"} [:post "/multipart"] - {:status 200 :body (:body req)} + {:status 200 :body (:body req) + :headers {"x-original-content-type" (get-in req [:headers "content-type"] "not found")}} [:get "/get-with-body"] {:status 200 :body (:body req)} [:options "/options"] @@ -335,6 +336,7 @@ :mime-type "application/text"}]}) resp-body (apply str (map #(try (char %) (catch Exception _ "")) (util/force-byte-array (:body resp))))] + #_(println "clj-http resp-body:\n>>>>>>>>>>>\n" resp-body "\n>>>>>>>>>>\n") (is (= 200 (:status resp))) (is (re-find #"testFINDMEtest" resp-body)) (is (re-find #"application/json" resp-body)) diff --git a/test/aleph/http/clj_http/util.clj b/test/aleph/http/clj_http/util.clj index d20cfecb..8ac9cc42 100644 --- a/test/aleph/http/clj_http/util.clj +++ b/test/aleph/http/clj_http/util.clj @@ -1,6 +1,7 @@ (ns aleph.http.clj-http.util (:require [aleph.http :as http] + [aleph.http.core :as http.core] [aleph.http.client-middleware :as aleph.mid] [clj-commons.byte-streams :as bs] [clj-http.core :as clj-http] @@ -13,7 +14,8 @@ (java.io ByteArrayInputStream ByteArrayOutputStream FilterInputStream - InputStream))) + InputStream) + (java.util.regex Pattern))) ;; turn off default middleware for the core tests (def no-middleware-pool (http/connection-pool {:middleware identity})) @@ -24,6 +26,7 @@ :server-port 18080}) (def ignored-headers ["date" "connection" "server"]) +(def multipart-related-headers ["content-length" "x-original-content-type"]) (defn header-keys "Returns a set of headers of interest" @@ -46,10 +49,24 @@ aleph-common-headers (select-keys aleph-headers ks-intersection)] (is (= clj-http-common-headers aleph-common-headers))))) +(defn- tee-output-stream + "Return the byte array contents of a stream, and a new, unconsumed stream" + [^InputStream in] + (let [baos (ByteArrayOutputStream.)] + (.transferTo in baos) ; not avail until JDK 9 + + (let [in-bytes (.toByteArray baos)] + {:bytes in-bytes + :stream (proxy [FilterInputStream] + [^InputStream (ByteArrayInputStream. in-bytes)] + (close [] + (.close in) + (proxy-super close)))}))) + (defn bodies= "Are the two bodies equal? clj-http's client/request fn coerces to strings by default, while the core/request leaves the body an InputStream. - Aleph, in keeping with it's stream-based nature, leaves as an InputStream by default. + Aleph, in keeping with its stream-based nature, leaves it as an InputStream by default. If an InputStream, returns a new ByteArrayInputStream based on the consumed original" [clj-http-body ^InputStream aleph-body] @@ -85,6 +102,66 @@ (is (= clj-http-body aleph-body)) clj-http-body))) +(defn- parse-multipart-boundary + [s] + (->> s + (re-find #"boundary=([^ ;]*)") + (second))) + +;;(defn- decode-multipart-body +;; [req body] +;; (let [req' (http.core/ring-request->netty-request req) +;; factory (DefaultHttpDataFactory. (long 1e6)) +;; decoder (HttpPostRequestDecoder. factory req') +;; baos (ByteArrayOutputStream.)])) + +(defn multipart-resp= + "Compares multipart responses from /multipart, which echoes the orig multipart bodies. + + Splits based on boundaries, and compares the parts. Whole-byte comparison is impossible + since the boundary strings are chosen randomly. + + Does not compare part headers for now, since they differ in case and order, and clj-http + adds Content-Length headers, which are uncommon, can cause problems, and may be completely + unknown for streaming requests." + [clj-http-resp aleph-resp] + (let [clj-http-headers (:headers clj-http-resp) + aleph-headers (:headers aleph-resp) + clj-http-boundary (parse-multipart-boundary (get clj-http-headers "x-original-content-type")) + aleph-boundary (parse-multipart-boundary (get aleph-headers "x-original-content-type")) + {clj-http-bytes :bytes clj-http-stream :stream} (tee-output-stream (:body clj-http-resp)) + aleph-bytes (-> aleph-resp :body tee-output-stream :bytes) + + ;; unlikely to be a problem, but let's make the regex literal, just to be safe + clj-http-boundary-regex (Pattern/compile clj-http-boundary (bit-or Pattern/LITERAL Pattern/MULTILINE)) + aleph-boundary-regex (Pattern/compile aleph-boundary (bit-or Pattern/LITERAL Pattern/MULTILINE)) + + clj-http-contents (-> ^bytes clj-http-bytes + (String.) + (str/split clj-http-boundary-regex)) + aleph-contents (-> ^bytes aleph-bytes + (String.) + (str/split aleph-boundary-regex))] + #_(do + (println "aleph bytes") + (bs/print-bytes aleph-bytes) + + (println "clj-http bytes") + (bs/print-bytes clj-http-bytes)) + + (is (= (count clj-http-contents) (count aleph-contents)) + "Unequal number of parts found!") + (doseq [[^String clj-http-part ^String aleph-part] (partition 2 (interleave clj-http-contents aleph-contents))] + (let [[clj-http-part-headers clj-http-part-body] (str/split clj-http-part #"\r\n\r\n") + [aleph-part-headers aleph-part-body] (str/split aleph-part #"\r\n\r\n")] + #_ (println "headers:>>>>>>>>>>>>\n" clj-http-part-headers "\n>>>>>>>>>>>>>>>>\n" aleph-part-headers) + #_ (println ">>>>>>>>>>\nbodies:\n" clj-http-part-body "\n>>>>>>>>>>>>>>>>\n" aleph-part-body) + (is (or (and (nil? clj-http-part-body) (nil? aleph-part-body)) + (.equalsIgnoreCase clj-http-part-body aleph-part-body)) + (str "clj-part:\n>>>>>>>>>>\n" clj-http-part-body "\n>>>>>>>>>>\naleph-part:\n>>>>>>>>>>\n" aleph-part-body "\n>>>>>>>>>>\n")))) + + clj-http-stream)) + (defn- defined-middleware "Returns a set of symbols beginning with `wrap-` in the ns" @@ -173,11 +250,12 @@ ;;_ (print-middleware-list clj-http.client/*current-middleware*) aleph-ring-map (merge base-req req {:pool (aleph-test-conn-pool clj-http-middleware)}) ;;_ (prn aleph-ring-map) + is-multipart (contains? clj-http-ring-map :multipart) clj-http-resp (clj-http-request clj-http-ring-map) aleph-resp @(http/request aleph-ring-map)] (is (= (:status clj-http-resp) (:status aleph-resp))) - (prn aleph-resp) + #_(when (not= (:status clj-http-resp) (:status aleph-resp)) (println "clj-http req:") @@ -193,7 +271,33 @@ (println "aleph resp:") (prn aleph-resp)) - (is-headers= (:headers clj-http-resp) (:headers aleph-resp)) - (is (instance? InputStream (:body aleph-resp))) - (let [new-clj-http-body (bodies= (:body clj-http-resp) (:body aleph-resp))] - (assoc clj-http-resp :body new-clj-http-body))))))) + (is (instance? InputStream (:body aleph-resp))) ; non-nil, for now... + + (if is-multipart + (do + ;;(println "multipart resps") + ;;(prn clj-http-resp) + ;;(prn aleph-resp) + ;;(println) + + (do + (println "clj-http req:") + (prn clj-http-ring-map) + (println) + (println "clj-http resp:") + (prn clj-http-resp) + (println) + (println) + (println "aleph req:") + (prn aleph-ring-map) + (println) + (println "aleph resp:") + (prn aleph-resp)) + + (is-headers= (apply dissoc (:headers clj-http-resp) multipart-related-headers) + (apply dissoc (:headers aleph-resp) multipart-related-headers)) + (assoc clj-http-resp :body (multipart-resp= clj-http-resp aleph-resp))) + (do + (is-headers= (:headers clj-http-resp) (:headers aleph-resp)) + (let [new-clj-http-body (bodies= (:body clj-http-resp) (:body aleph-resp) is-multipart)] + (assoc clj-http-resp :body new-clj-http-body))))))))) From 718fcce081334acba57c9a836e29a4d753dc9da9 Mon Sep 17 00:00:00 2001 From: Matthew Davidson Date: Wed, 26 Oct 2022 15:29:32 +0800 Subject: [PATCH 12/13] Copy body and multipart content streams Copy streams used in request maps, so consuming one doesn't break the other --- test/aleph/http/clj_http/util.clj | 65 ++++++++++++++++++++++++------- 1 file changed, 50 insertions(+), 15 deletions(-) diff --git a/test/aleph/http/clj_http/util.clj b/test/aleph/http/clj_http/util.clj index 8ac9cc42..575f87d4 100644 --- a/test/aleph/http/clj_http/util.clj +++ b/test/aleph/http/clj_http/util.clj @@ -229,6 +229,41 @@ symbol)) middleware-list))) +(defn- bais-clone + "Clones a ByteArrayInputStream and resets the original's pos, so it can be read again" + [^ByteArrayInputStream bais] + (.mark bais 0) + (let [new-bais (ByteArrayInputStream. (.readAllBytes bais))] + (.reset bais) + new-bais)) + +(defn build-aleph-ring-map + "Constructs an aleph ring map, based on the clj-http ring map. + + Adds corresponding middleware, and copies request ByteArrayInputStreams, + since they can't be read more than once by default." + [clj-http-ring-map clj-http-middleware] + (let [clone-bais-val (fn [m k] + (if (= ByteArrayInputStream (-> m k class)) + (assoc m k (bais-clone (k m))) + m)) + middleware-ring-map (merge clj-http-ring-map {:pool (aleph-test-conn-pool clj-http-middleware)})] + (cond-> middleware-ring-map + + (contains? clj-http-ring-map :body) + (clone-bais-val :body) + + (contains? clj-http-ring-map :multipart) + (update-in [:multipart] + (fn [parts] + (into [] + (map #(clone-bais-val % :content) + #_(fn [part] + (if (= ByteArrayInputStream (-> part :content class)) + (assoc part :content (bais-clone (:content part))) + part))) + parts)))))) + (defn make-request "Need to switch between clj-http's core/request and client/request. @@ -248,7 +283,7 @@ ;;_ (prn clj-http-ring-map) clj-http-middleware (if using-middleware? clj-http.client/*current-middleware* []) ;;_ (print-middleware-list clj-http.client/*current-middleware*) - aleph-ring-map (merge base-req req {:pool (aleph-test-conn-pool clj-http-middleware)}) + aleph-ring-map (build-aleph-ring-map clj-http-ring-map clj-http-middleware) ;;_ (prn aleph-ring-map) is-multipart (contains? clj-http-ring-map :multipart) clj-http-resp (clj-http-request clj-http-ring-map) @@ -280,24 +315,24 @@ ;;(prn aleph-resp) ;;(println) - (do - (println "clj-http req:") - (prn clj-http-ring-map) - (println) - (println "clj-http resp:") - (prn clj-http-resp) - (println) - (println) - (println "aleph req:") - (prn aleph-ring-map) - (println) - (println "aleph resp:") - (prn aleph-resp)) + ;;(do + ;; (println "clj-http req:") + ;; (prn clj-http-ring-map) + ;; (println) + ;; (println "clj-http resp:") + ;; (prn clj-http-resp) + ;; (println) + ;; (println) + ;; (println "aleph req:") + ;; (prn aleph-ring-map) + ;; (println) + ;; (println "aleph resp:") + ;; (prn aleph-resp)) (is-headers= (apply dissoc (:headers clj-http-resp) multipart-related-headers) (apply dissoc (:headers aleph-resp) multipart-related-headers)) (assoc clj-http-resp :body (multipart-resp= clj-http-resp aleph-resp))) (do (is-headers= (:headers clj-http-resp) (:headers aleph-resp)) - (let [new-clj-http-body (bodies= (:body clj-http-resp) (:body aleph-resp) is-multipart)] + (let [new-clj-http-body (bodies= (:body clj-http-resp) (:body aleph-resp))] (assoc clj-http-resp :body new-clj-http-body))))))))) From 3abf9c84464430fe02f11afe7e62435c1c8fcf27 Mon Sep 17 00:00:00 2001 From: Matthew Davidson Date: Wed, 26 Oct 2022 16:08:52 +0800 Subject: [PATCH 13/13] Add some more tests to ignore list clj-http options that interop with Apache-specific stuff can't be tested --- test/aleph/http/clj_http/core_test.clj | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/aleph/http/clj_http/core_test.clj b/test/aleph/http/clj_http/core_test.clj index 88a726f9..c0920026 100644 --- a/test/aleph/http/clj_http/core_test.clj +++ b/test/aleph/http/clj_http/core_test.clj @@ -795,7 +795,7 @@ (is (= http-context context-for-request)) (is (= request-config (.getRequestConfig context-for-request))))))) -(deftest ^:integration test-custom-http-builder-fns +(deftest ^:integration ^:ignore test-custom-http-builder-fns (run-server) (let [resp (client/get (localhost "/get") {:headers {"add-headers" "true"} @@ -884,7 +884,7 @@ (is false "should have thrown a timeout exception") (catch TimeoutException te))) -(deftest ^:integration test-reusable-http-client +(deftest ^:integration ^:ignore test-reusable-http-client (run-server) (let [cm (conn/make-reuseable-async-conn-manager {}) hc (core/build-async-http-client {} cm)] @@ -942,7 +942,7 @@ (is (= @validated true)))) -(deftest t-cache-config +(deftest ^:ignore t-cache-config (let [cc (core/build-cache-config {:cache-config {:allow-303-caching true :asynchronous-worker-idle-lifetime-secs 10 @@ -971,7 +971,7 @@ (is (= false (.isSharedCache cc))) (is (= true (.isWeakETagOnPutDeleteAllowed cc))))) -(deftest ^:integration t-client-caching +(deftest ^:integration ^:ignore t-client-caching (run-server) (let [cm (conn/make-reusable-conn-manager {}) r1 (client/get (localhost "/get")