purrr map2 can't find a function while nested inside another function

[ercbk]

library(mirai)

x <- c(1, 2, 3)
y <- c(10, 20, 30)


some_fun1 <- function(x, y) {
  sum_fun1 <- function(a, b) {
    a + b
  }
  get_sum1 <- 
    purrr::map2(
      x,
      y,
      \(x1, x2) {
        sum_fun1(x1, x2)
      },
      .parallel = TRUE
    )
  return(get_sum1)
}

daemons(3)
#> [1] 3
some_fun1(x, y)
#> ✔ Automatically crated `.f`: 1.29 kB
#> Error in `purrr::map2()`:
#> ℹ In index: 1.
#> Caused by error in `sum_fun1()`:
#> ! could not find function "sum_fun1"
daemons(0)
#> [1] 0

some_fun2 <- function(x, y) {
  sum_fun2 <- function(a, b) {
    a + b
  }
  get_sum2 <- 
    purrr::map2(
      x,
      y,
      \(x1, x2) {
        sum_fun2(x1, x2)
      }
    )
  return(get_sum2)
}

some_fun2(x, y)
#> [[1]]
#> [1] 11
#> 
#> [[2]]
#> [1] 22
#> 
#> [[3]]
#> [1] 33

^{Created on 2025-02-10 with reprex v2.1.1}

Session Info

sessioninfo::session_info()
─ Session info ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
 setting  value
 version  R version 4.4.2 (2024-10-31 ucrt)
 os       Windows 10 x64 (build 19045)
 system   x86_64, mingw32
 ui       RStudio
 language (EN)
 collate  English_United States.utf8
 ctype    English_United States.utf8
 tz       America/New_York
 date     2025-02-10
 rstudio  2024.12.0+467 Kousa Dogwood (desktop)
 pandoc   NA

─ Packages ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
 package     * version    date (UTC) lib source
 carrier       0.1.1      2023-04-28 [1] RSPM
 cli           3.6.3      2024-06-21 [1] CRAN (R 4.4.2)
 glue          1.8.0      2024-09-30 [1] RSPM
 lifecycle     1.0.4      2023-11-07 [1] CRAN (R 4.4.2)
 lobstr        1.1.2      2022-06-22 [1] RSPM
 magrittr      2.0.3      2022-03-30 [1] CRAN (R 4.4.2)
 mirai       * 2.1.0.9000 2025-02-07 [1] https://shikokuchuo.r-universe.dev (R 4.4.2)
 nanonext      1.5.0      2025-01-28 [1] RSPM (R 4.4.0)
 pillar        1.10.1     2025-01-07 [1] RSPM (R 4.4.0)
 prettyunits   1.2.0      2023-09-24 [1] RSPM
 purrr         1.0.4.9000 2025-02-10 [1] Github (tidyverse/purrr@9c8beb4)
 rlang         1.1.5      2025-01-17 [1] RSPM (R 4.4.0)
 rstudioapi    0.17.1     2024-10-22 [1] CRAN (R 4.4.2)
 sessioninfo   1.2.2      2021-12-06 [1] CRAN (R 4.4.2)
 vctrs         0.6.5      2023-12-01 [1] RSPM

 [1] C:/Users/tbats/AppData/Local/R/win-library/4.4
 [2] C:/Program Files/R/R-4.4.2/library

────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────

[shikokuchuo]

This would be the consequence of purrr's auto-crating behaviour. See 'Crating a function' under ?parallelization.

It's designed in part to guard against shipping large objects in a function's enclosing environment when they're not even needed by it (can easily happen when dealing with nested functions). You would need to crate your anonymous function in the following way:

some_fun1 <- function(x, y) {
  sum_fun1 <- function(a, b) {
    a + b
  }
  get_sum1 <- 
    purrr::map2(
      x,
      y,
      carrier::crate(
        \(x1, x2) {
          sum_fun1(x1, x2)
        },
        sum_fun1 = sum_fun1
      ),
      .parallel = TRUE
    )
  return(get_sum1)
}

Purrr uses the carrier package, but it's a similar concept to using the ... argument in mirai::mirai_map() itself to create functions which are self-contained.

[ercbk]

Is there any way to use ...?

library(mirai)

x <- c(1, 2, 3)
y <- c(10, 20, 30)


some_fun1 <- function(x, y, ...) {
  sum_fun1 <- function(a, b, ...) {
    a + b
  }
  get_sum1 <- 
    purrr::map2(
      x,
      y,
      carrier::crate(
        \(x1, x2) {
          sum_fun1(x1, x2, ...)
        },
        sum_fun1 = sum_fun1
      ),
      .parallel = TRUE
    )
  return(get_sum1)
}

daemons(3)
#> [1] 3
some_fun1(x, y)
#> Error in `purrr::map2()`:
#> ℹ In index: 1.
#> Caused by error:
#> ! '...' used in an incorrect context
daemons(0)
#> [1] 0

^{Created on 2025-02-10 with reprex v2.1.1}

[ercbk]

I think this gets me pretty close to where I want to be.

library(mirai)

daemons(3)
#> [1] 3

x <- c(1, 2, 3)
y <- c(10, 20, 30)


some_fun1 <- function(x, y, ...) {
  sum_fun1 <- function(a, b, ...) {
    args <- list(a = a, b = b)
    dots <- list(...)
    if (length(dots) != 0) {
      args <- append(args, dots)
    }
    args
  }
  dots <- list(...)
  get_sum1 <- 
    purrr::map2(
      x,
      y,
      carrier::crate(
        \(x1, x2) {
          sum_fun1(x1, x2, !!!dots)
        },
        sum_fun1 = sum_fun1
      ),
      .parallel = TRUE
    )
  return(get_sum1)
}

some_fun1(x, y)
#> [[1]]
#> [[1]]$a
#> [1] 1
#> 
#> [[1]]$b
#> [1] 10
#> 
#> 
#> [[2]]
#> [[2]]$a
#> [1] 2
#> 
#> [[2]]$b
#> [1] 20
#> 
#> 
#> [[3]]
#> [[3]]$a
#> [1] 3
#> 
#> [[3]]$b
#> [1] 30
some_fun1(x, y, z = 10)
#> [[1]]
#> [[1]]$a
#> [1] 1
#> 
#> [[1]]$b
#> [1] 10
#> 
#> [[1]]$z
#> [1] 10
#> 
#> 
#> [[2]]
#> [[2]]$a
#> [1] 2
#> 
#> [[2]]$b
#> [1] 20
#> 
#> [[2]]$z
#> [1] 10
#> 
#> 
#> [[3]]
#> [[3]]$a
#> [1] 3
#> 
#> [[3]]$b
#> [1] 30
#> 
#> [[3]]$z
#> [1] 10
some_fun1(x, y, z = 10, m = 20)
#> [[1]]
#> [[1]]$a
#> [1] 1
#> 
#> [[1]]$b
#> [1] 10
#> 
#> [[1]]$z
#> [1] 10
#> 
#> [[1]]$m
#> [1] 20
#> 
#> 
#> [[2]]
#> [[2]]$a
#> [1] 2
#> 
#> [[2]]$b
#> [1] 20
#> 
#> [[2]]$z
#> [1] 10
#> 
#> [[2]]$m
#> [1] 20
#> 
#> 
#> [[3]]
#> [[3]]$a
#> [1] 3
#> 
#> [[3]]$b
#> [1] 30
#> 
#> [[3]]$z
#> [1] 10
#> 
#> [[3]]$m
#> [1] 20

daemons(0)
#> [1] 0

^{Created on 2025-02-12 with reprex v2.1.1}

[ercbk]

Got it. There are two solutions:

Bundle all the inputs that aren't being iterated into a list

some_fun1 <- function(x, y, n = NULL, ...) {
  sum_fun1 <- function(a, b, n, ...) {
    args <- list(a = a, b = b, n = n)
    dots <- list(...)
    if (length(dots) != 0) {
      args <- append(args, dots)
    }

    args
  }
  args <- list(n = n)
  dots <- list(...)
  if (length(dots) != 0) {
    args <- append(args, dots)
  }

  get_sum1 <- 
    purrr::map2(
      x,
      y,
      carrier::crate(
        \(x1, x2) {
          sum_fun1(x1, x2, !!!args)
        },
        sum_fun1 = sum_fun1
      ),
      .parallel = TRUE
    )
  return(get_sum1)
}

Or unquote the inputs not being iterated individually

some_fun1 <- function(x, y, n = NULL, ...) {
  sum_fun1 <- function(a, b, n, ...) {
    args <- list(a = a, b = b, n = n)
    dots <- list(...)
    if (length(dots) != 0) {
      args <- append(args, dots)
    }

    args
  }

  dots <- list(...)

  get_sum1 <- 
    purrr::map2(
      x,
      y,
      carrier::crate(
        \(x1, x2) {
          sum_fun1(x1, x2, n = !!n, !!!dots)
        },
        sum_fun1 = sum_fun1
      ),
      .parallel = TRUE
    )
  return(get_sum1)
}

some_fun1(x, y)
some_fun1(x, y, n = 5)
some_fun1(x, y, n = 5, z = 10)
some_fun1(x, y, n = 5, z = 10, m = 20)

[ercbk]

Even only doing 2 iterations with the function I using this for, I get a 41%(?) speed-up 👍.
parallel: 17.01 sec
not parallel: 28.78 sec

... with 4 iterations, that's a 67% speed-up I think 👍👍
parallel: 19.25 sec
not parallel: 58.50 sec