diff --git a/DESCRIPTION b/DESCRIPTION index a38eb60..d8fe672 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -25,7 +25,11 @@ Authors@R: person(given = "Mark", family = "Edmondson", role = "ctb", - email = "r@sunholo.com")) + email = "r@sunholo.com"), + person(given = "Hong", + family = "Ooi", + role = "ctb", + email = "hongooi73@gmail.com")) Description: Cache the results of a function so that when you call it again with the same arguments it returns the pre-computed value. @@ -41,6 +45,8 @@ Suggests: covr, googleAuthR, googleCloudStorageR, + AzureStor (>= 3.0.0), + AzureAuth, httr, testthat Remotes: diff --git a/NAMESPACE b/NAMESPACE index 82acdd3..1acdd7b 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,6 +1,7 @@ # Generated by roxygen2: do not edit by hand S3method(print,memoised) +export(cache_azure) export(cache_filesystem) export(cache_gcs) export(cache_memory) diff --git a/R/cache_azure.R b/R/cache_azure.R new file mode 100644 index 0000000..30f09c7 --- /dev/null +++ b/R/cache_azure.R @@ -0,0 +1,104 @@ +#' Azure Storage Cache +#' +#' Azure Storage backed cache, for remote caching. File, blob and ADLSgen2 storage are all supported. +#' +#' @examples +#' +#' \dontrun{ +#' library(AzureStor) +#' +#' # use Azure blob storage for the cache +#' stor <- storage_endpoint("https://blob.accountname.core.windows.net", key="storage-key") +#' azcache <- cache_azure("cache_container", stor) +#' mem_runif <- memoise(runif, cache = azcache) +#' +#' +#' # you can also pass the endpoint URL and key to cache_azure: +#' azcache <- cache_azure("cache_container", "https://blob.accountname.core.windows.net", +#' key = "storage-key") +#' +#' # a better alternative to a master key: OAuth 2.0 authentication via AAD +#' token <- AzureAuth::get_azure_token( +#' "https://storage.azure.com", +#' tenant = "mytenant", +#' app = "app_id" +#' ) +#' azcache <- cache_azure("cache_container", "https://blob.accountname.core.windows.net", +#' token = token) +#' } +#' +#' @param cache_name Name of the storage container for storing cache files. +#' @param endpoint The storage account endpoint for the cache. This should be an object of class \code{AzureStor::storage_endpoint}, or inheriting from it. Alternatively, you can provide the endpoint URL as a string, and pass any authentication arguments in `...`. +#' @param compress Argument passed to \code{saveRDS}. One of FALSE, "gzip", +#' "bzip2" or "xz". Default: FALSE. +#' @param ... Further arguments that will be passed to \code{AzureStor::storage_endpoint}, if \code{endpoint} is a URL. +#' @export +cache_azure <- function(cache_name, endpoint, compress = FALSE, ...) { + if(!requireNamespace("AzureStor")) { stop("Package `AzureStor` must be installed for `cache_azure()`.") } # nocov + + if(is.character(endpoint)) { + endpoint <- AzureStor::storage_endpoint(endpoint, ...) + } else if(!inherits(endpoint, "storage_endpoint")) { + stop("Must provide either the URL of a storage account endpoint, or a `storage_endpoint` object") + } + + path <- tempfile("memoise-") + dir.create(path) + + # create container if it doesn't exist + try(AzureStor::create_storage_container(endpoint, cache_name), silent = TRUE) + cache <- AzureStor::storage_container(endpoint, cache_name) + + cache_reset <- function() { + keys <- cache_keys() + lapply(keys, function(key) { + AzureStor::delete_storage_file(cache, key, confirm = FALSE) + }) + } + + cache_set <- function(key, value) { + rds <- file.path(path, key) + saveRDS(value, rds, compress = compress) + opts <- options(azure_storage_progress_bar = FALSE) + on.exit({ + unlink(rds) + options(opts) + }) + AzureStor::storage_upload(cache, rds, key) + } + + cache_get <- function(key) { + rds <- file.path(path, key) + opts <- options(azure_storage_progress_bar = FALSE) + on.exit({ + unlink(rds) + options(opts) + }) + res <- try(AzureStor::storage_download(cache, key, rds, overwrite = TRUE), silent = TRUE) + if(inherits(res, "try-error")) { + return(cachem::key_missing()) + } + readRDS(rds) + } + + cache_has_key <- function(key) { + key %in% cache_keys() + } + + cache_drop_key <- function(key) { + AzureStor::delete_storage_file(cache, key, confirm = FALSE) + } + + cache_keys <- function() { + AzureStor::list_storage_files(cache, info = "name") + } + + list( + reset = cache_reset, + set = cache_set, + get = cache_get, + exists = cache_has_key, + remove = cache_drop_key, + keys = cache_keys + ) +} diff --git a/R/cache_filesystem.R b/R/cache_filesystem.R index 153f8d4..aaef39a 100644 --- a/R/cache_filesystem.R +++ b/R/cache_filesystem.R @@ -21,6 +21,12 @@ #' #' mem_runif <- memoise(runif, cache = gd) #' +#' # Use with OneDrive (on Windows) +#' +#' od <- cache_filesystem("~/../OneDrive") +#' +#' mem_runif <- memoise(runif, cache = od) +#' #' } #' #' @export diff --git a/man/cache_azure.Rd b/man/cache_azure.Rd new file mode 100644 index 0000000..d19686a --- /dev/null +++ b/man/cache_azure.Rd @@ -0,0 +1,47 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/cache_azure.R +\name{cache_azure} +\alias{cache_azure} +\title{Azure Storage Cache} +\usage{ +cache_azure(cache_name, endpoint, compress = FALSE, ...) +} +\arguments{ +\item{cache_name}{Name of the storage container for storing cache files.} + +\item{endpoint}{The storage account endpoint for the cache. This should be an object of class \code{AzureStor::storage_endpoint}, or inheriting from it. Alternatively, you can provide the endpoint URL as a string, and pass any authentication arguments in `...`.} + +\item{compress}{Argument passed to \code{saveRDS}. One of FALSE, "gzip", +"bzip2" or "xz". Default: FALSE.} + +\item{...}{Further arguments that will be passed to \code{AzureStor::storage_endpoint}, if \code{endpoint} is a URL.} +} +\description{ +Azure Storage backed cache, for remote caching. File, blob and ADLSgen2 storage are all supported. +} +\examples{ + +\dontrun{ +library(AzureStor) + +# use Azure blob storage for the cache +stor <- storage_endpoint("https://blob.accountname.core.windows.net", key="storage-key") +azcache <- cache_azure("cache_container", stor) +mem_runif <- memoise(runif, cache = azcache) + + +# you can also pass the endpoint URL and key to cache_azure: +azcache <- cache_azure("cache_container", "https://blob.accountname.core.windows.net", + key = "storage-key") + +# a better alternative to a master key: OAuth 2.0 authentication via AAD +token <- AzureAuth::get_azure_token( + "https://storage.azure.com", + tenant = "mytenant", + app = "app_id" +) +azcache <- cache_azure("cache_container", "https://blob.accountname.core.windows.net", + token = token) +} + +} diff --git a/man/cache_filesystem.Rd b/man/cache_filesystem.Rd index a2c678b..b9981fa 100644 --- a/man/cache_filesystem.Rd +++ b/man/cache_filesystem.Rd @@ -33,6 +33,12 @@ gd <- cache_filesystem("~/Google Drive/.rcache") mem_runif <- memoise(runif, cache = gd) +# Use with OneDrive (on Windows) + +od <- cache_filesystem("~/../OneDrive") + +mem_runif <- memoise(runif, cache = od) + } } diff --git a/tests/testthat/helper.R b/tests/testthat/helper.R index af8164c..006a624 100644 --- a/tests/testthat/helper.R +++ b/tests/testthat/helper.R @@ -23,3 +23,14 @@ skip_on_travis_pr <- function() { invisible(TRUE) } + +skip_without_azure_credentials <- function() { + storage_url <- Sys.getenv("AZ_TEST_STORAGE_MEMOISE_URL") + storage_key <- Sys.getenv("AZ_TEST_STORAGE_MEMOISE_KEY") + + if (storage_url == "" || storage_key == "") { + testthat::skip("No Azure storage credentials") + } else { + invisible(TRUE) + } +} diff --git a/tests/testthat/test-azure.R b/tests/testthat/test-azure.R new file mode 100644 index 0000000..b270057 --- /dev/null +++ b/tests/testthat/test-azure.R @@ -0,0 +1,41 @@ +context("azure storage") + +skip_on_cran() +skip_on_travis_pr() +skip_without_azure_credentials() + +storage_url <- Sys.getenv("AZ_TEST_STORAGE_MEMOISE_URL") +storage_key <- Sys.getenv("AZ_TEST_STORAGE_MEMOISE_KEY") + +bl_endp <- AzureStor::storage_endpoint(storage_url, key=storage_key) +cache_name <- paste0(sample(letters, 20, TRUE), collapse = "") + +test_that("using an Azure storage cache works", { + + azcache <- cache_azure(cache_name, bl_endp) + i <- 0 + fn <- function() { i <<- i + 1; i } + fnm <- memoise(fn, cache = azcache) + on.exit(forget(fnm)) + + expect_equal(fn(), 1) + expect_equal(fn(), 2) + expect_equal(fnm(), 3) + expect_equal(fnm(), 3) + expect_equal(fn(), 4) + expect_equal(fnm(), 3) + + expect_false(forget(fn)) + expect_true(forget(fnm)) + expect_equal(fnm(), 5) + + expect_true(drop_cache(fnm)()) + expect_equal(fnm(), 6) + + expect_true(is.memoised(fnm)) + expect_false(is.memoised(fn)) +}) + +teardown({ + AzureStor::delete_blob_container(bl_endp, cache_name, confirm = FALSE) +})