Merge commit 'f6803676e703d69224e988231311530ac369edfb'

NAMESPACE

@@ -67,6 +67,7 @@ export(req_cookies_set)
 export(req_dry_run)
 export(req_error)
 export(req_headers)
+export(req_headers_redacted)
 export(req_method)
 export(req_oauth)
 export(req_oauth_auth_code)
@@ -116,10 +117,12 @@ export(resp_headers)
 export(resp_is_error)
 export(resp_link_url)
 export(resp_raw)
+export(resp_request)
 export(resp_retry_after)
 export(resp_status)
 export(resp_status_desc)
 export(resp_stream_aws)
+export(resp_stream_is_complete)
 export(resp_stream_lines)
 export(resp_stream_raw)
 export(resp_stream_sse)
@@ -145,7 +148,11 @@ export(signal_total_pages)
 export(throttle_status)
 export(url_build)
 export(url_modify)
+export(url_modify_query)
+export(url_modify_relative)
 export(url_parse)
+export(url_query_build)
+export(url_query_parse)
 export(with_mock)
 export(with_mocked_responses)
 export(with_verbosity)

NEWS.md

@@ -1,16 +1,19 @@
 # httr2 (development version)
 
 * `curl_translate()` now translates cookie headers to `req_cookies_set()` (#431).
-* New `url_modify()` makes it easier to modify an existing url (#464).
+* `resp_stream_is_complete()` tells you if there is still data remaining to be streamed (#559).
+* New `url_modify()`, `url_modify_query()`, and `url_modify_relative()` make it easier to modify an existing url (#464).
+* New `url_query_parse()` and `url_query_build()` allow you to parse and build a query string (#425).
+* `req_url_query()` gains the ability to control how spaces are encoded (#432).
+* New `resp_request()` aids debugging by returning the request associated with a response (#604).
+* `print.request()` now correctly escapes `{}` in headers (#586).
+* New `req_headers_redacted()` provides a user-friendlier way to set redacted headers (#561).
+* `resp_link_url()` now works if there are multiple `Link` headers (#587).
 * New `req_url_relative()` for constructing relative urls (#449).
 * `url_parse()` gains `base_url` argument so you can also use it to parse relative URLs (#449).
 * `url_parse()` now uses `curl::curl_parse_url()` which is much faster and more correct (#577).
-* `req_retry()` now defaults to `max_tries = 2` with a message.
-  Set to `max_tries = 1` to disable retries.
-
-* Errors thrown during the parsing of an OAuth response now have a dedicated
-  `httr2_oauth_parse` error class that includes the original response object
-  (@atheriel, #596).
+* `req_retry()` now defaults to `max_tries = 2` with a message. Set to `max_tries = 1` to disable retries.
+* Errors thrown during the parsing of an OAuth response now have a dedicated `httr2_oauth_parse` error class that includes the original response object (@atheriel, #596).
 
 # httr2 1.0.7
 

R/curl.R

@@ -63,7 +63,9 @@ curl_translate <- function(cmd, simplify_headers = TRUE) {
 
   # Content type set with data
   type <- data$headers$`Content-Type`
-  data$headers$`Content-Type` <- NULL
+  if (!identical(data$data, "")) {
+    data$headers$`Content-Type` <- NULL
+  }
 
   headers <- curl_simplify_headers(data$headers, simplify_headers)
   steps <- add_curl_step(steps, "req_headers", dots = headers)

R/headers.R

@@ -1,15 +1,11 @@
 as_headers <- function(x, error_call = caller_env()) {
   if (is.character(x) || is.raw(x)) {
-    headers <- curl::parse_headers(x)
-    headers <- headers[grepl(":", headers, fixed = TRUE)]
+    parsed <- curl::parse_headers(x)
+    valid <- parsed[grepl(":", parsed, fixed = TRUE)]
+    halves <- parse_in_half(valid, ":")
 
-    equals <- regexpr(":", headers, fixed = TRUE)
-    pieces <- regmatches(headers, equals, invert = TRUE)
-
-    names <- map_chr(pieces, "[[", 1)
-    values <- as.list(trimws(map_chr(pieces, "[[", 2)))
-
-    new_headers(set_names(values, names), error_call = error_call)
+    headers <- set_names(trimws(halves$right), halves$left)
+    new_headers(as.list(headers), error_call = error_call)
   } else if (is.list(x)) {
     new_headers(x, error_call = error_call)
   } else {

R/oauth-flow-auth-code.R

@@ -365,7 +365,7 @@ oauth_flow_auth_code_listen <- function(redirect_uri = "http://localhost:1410")
 # https://www.w3.org/TR/html401/interact/forms.html#h-17.13.4.1
 # Spaces are first replaced by +
 parse_form_urlencoded <- function(query) {
-  query <- query_parse(query)
+  query <- url_query_parse(query)
   query[] <- gsub("+", " ", query, fixed = TRUE)
   query
 }

R/req-auth-aws.R

@@ -129,7 +129,7 @@ aws_v4_signature <- function(method,
     CanonicalQueryString <- ""
   } else {
     sorted_query <- url$query[order(names(url$query))]
-    CanonicalQueryString <- query_build(CanonicalQueryString)
+    CanonicalQueryString <- url_query_build(CanonicalQueryString)
   }
 
   headers$host <- url$hostname

R/req-body.R

@@ -227,7 +227,7 @@ req_body_get <- function(req) {
     raw = req$body$data,
     form = {
       data <- unobfuscate(req$body$data)
-      query_build(data)
+      url_query_build(data)
     },
     json = exec(jsonlite::toJSON, req$body$data, !!!req$body$params),
     cli::cli_abort("Unsupported request body type {.str {req$body$type}}.")

R/req-headers.R

@@ -1,7 +1,12 @@
 #' Modify request headers
 #'
+#' @description
 #' `req_headers()` allows you to set the value of any header.
 #'
+#' `req_headers_redacted()` is a variation that adds "redacted" headers, which
+#' httr2 avoids printing on the console. This is good practice for
+#' authentication headers to avoid accidentally leaking them in log files.
+#'
 #' @param .req A [request].
 #' @param ... <[`dynamic-dots`][rlang::dyn-dots]> Name-value pairs of headers
 #'   and their values.
@@ -46,13 +51,16 @@
 #' # If you have headers in a list, use !!!
 #' headers <- list(HeaderOne = "one", HeaderTwo = "two")
 #' req |>
-#'    req_headers(!!!headers, HeaderThree = "three") |>
-#'    req_dry_run()
-#'
-#' # Use `.redact` to hide a header in the output
-#' req |>
-#'   req_headers(Secret = "this-is-private", Public = "but-this-is-not", .redact = "Secret") |>
+#'   req_headers(!!!headers, HeaderThree = "three") |>
 #'   req_dry_run()
+#'
+#' # Use `req_headers_redacted()`` to hide a header in the output
+#' req_secret <- req |>
+#'   req_headers_redacted(Secret = "this-is-private") |>
+#'   req_headers(Public = "but-this-is-not")
+#'
+#' req_secret
+#' req_secret |> req_dry_run()
 req_headers <- function(.req, ..., .redact = NULL) {
   check_request(.req)
 
@@ -68,3 +76,12 @@ req_headers <- function(.req, ..., .redact = NULL) {
 
   .req
 }
+
+#' @export
+#' @rdname req_headers
+req_headers_redacted <- function(.req, ...) {
+  check_request(.req)
+
+  dots <- list(...)
+  req_headers(.req, !!!dots, .redact = names(dots))
+}

R/req-options.R

@@ -207,7 +207,7 @@ verbose_header <- function(prefix, x, redact = TRUE, to_redact = NULL) {
   lines <- unlist(strsplit(x, "\r?\n", useBytes = TRUE))
 
   for (line in lines) {
-    if (grepl(":", line, fixed = TRUE)) {
+    if (grepl("^[-a-zA-z0-9]+:", line)) {
       header <- headers_redact(as_headers(line), redact, to_redact = to_redact)
       cli::cat_line(prefix, cli::style_bold(names(header)), ": ", header)
     } else {

R/req-url.R

@@ -6,6 +6,9 @@
 #' * `req_url_path()` modifies the path
 #' * `req_url_path_append()` adds to the path
 #'
+#' Alternatively, to modify only a URL without creating a request,
+#' you can instead use [url_modify()] and friends.
+#'
 #' @inheritParams req_perform
 #' @param url New URL; completely replaces existing.
 #' @param ... For `req_url_query()`: <[`dynamic-dots`][rlang::dyn-dots]>
@@ -56,34 +59,19 @@ req_url <- function(req, url) {
 #' @rdname req_url
 req_url_relative <- function(req, url) {
   check_request(req)
-
-  new_url <- url_parse(url, base_url = req$url)
-  req_url(req, url_build(new_url))
+  req_url(req, url_modify_relative(req$url, url))
 }
 
 #' @export
 #' @rdname req_url
-#' @param .multi Controls what happens when an element of `...` is a vector
-#'   containing multiple values:
-#'
-#'   * `"error"`, the default, throws an error.
-#'   * `"comma"`, separates values with a `,`, e.g. `?x=1,2`.
-#'   * `"pipe"`, separates values with a `|`, e.g. `?x=1|2`.
-#'   * `"explode"`, turns each element into its own parameter, e.g. `?x=1&x=2`
-#'
-#'   If none of these options work for your needs, you can instead supply a
-#'   function that takes a character vector of argument values and returns a
-#'   a single string.
+#' @inheritParams url_modify_query
 req_url_query <- function(.req,
                           ...,
-                          .multi = c("error", "comma", "pipe", "explode")) {
+                          .multi = c("error", "comma", "pipe", "explode"),
+                          .space = c("percent", "form")) {
   check_request(.req)
-
-  dots <- multi_dots(..., .multi = .multi)
-
-  url <- url_parse(.req$url)
-  url$query <- modify_list(url$query, !!!dots)
-  req_url(.req, url_build(url))
+  url <- url_modify_query(.req$url, ..., .multi = .multi, .space = .space)
+  req_url(.req, url)
 }
 
 #' @export

R/resp-headers.R

@@ -173,7 +173,9 @@ resp_link_url <- function(resp, rel) {
     return()
   }
 
-  links <- parse_link(resp_header(resp, "Link"))
+  headers <- resp_headers(resp)
+  link_headers <- headers[names(headers) == "Link"]
+  links <- unlist(lapply(link_headers, parse_link), recursive = FALSE)
   sel <- map_lgl(links, ~ .$rel == rel)
   if (sum(sel) != 1L) {
     return()

R/resp-request.R

@@ -0,0 +1,16 @@
+#' Find the request responsible for a response
+#'
+#' To make debugging easier, httr2 includes the request that was used to
+#' generate every response. You can use this function to access it.
+#'
+#' @inheritParams resp_header
+#' @export
+#' @examples
+#' req <- request(example_url())
+#' resp <- req_perform(req)
+#' resp_request(resp)
+resp_request <- function(resp) {
+  check_response(resp)
+
+  resp$request
+}

R/resp-stream.R

@@ -10,6 +10,9 @@
 #' * `resp_stream_aws()` retrieves a single event from an AWS stream
 #'   (i.e. mime type `application/vnd.amazon.eventstream``).
 #'
+#' Use `resp_stream_is_complete()` to determine if there is further data
+#' waiting on the stream.
+#'
 #' @returns
 #' * `resp_stream_raw()`: a raw vector.
 #' * `resp_stream_lines()`: a character vector.
@@ -80,6 +83,14 @@ resp_stream_sse <- function(resp, max_size = Inf) {
   }
 }
 
+#' @export
+#' @rdname resp_stream_raw
+resp_stream_is_complete <- function(resp) {
+  check_response(resp)
+
+  !isIncomplete(resp$body)
+}
+
 #' @export
 #' @param ... Not used; included for compatibility with generic.
 #' @rdname resp_stream_raw

R/test.R

@@ -50,7 +50,7 @@ example_url <- function() {
   env_cache(the, "test_app",
     webfakes::new_app_process(
       app,
-      opts = webfakes::server_opts(num_threads = 2)
+      opts = webfakes::server_opts(num_threads = 6, enable_keep_alive = TRUE)
     )
   )
   the$test_app$url()