diff --git a/.lintr b/.lintr index 6fa3c0ff..49971ef5 100644 --- a/.lintr +++ b/.lintr @@ -5,6 +5,6 @@ linters: linters_with_defaults( object_name_linter = object_name_linter(c("snake_case", "CamelCase")), # only allow snake case and camel case object names cyclocomp_linter = NULL, # do not check function complexity commented_code_linter = NULL, # allow code in comments - line_length_linter = line_length_linter(120), + line_length_linter = line_length_linter(200), indentation_linter(indent = 2L, hanging_indent_style = "never") ) diff --git a/DESCRIPTION b/DESCRIPTION index 5cc99bee..85932f0c 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -39,6 +39,7 @@ License: LGPL-3 URL: https://mlr3mbo.mlr-org.com, https://github.com/mlr-org/mlr3mbo BugReports: https://github.com/mlr-org/mlr3mbo/issues Depends: + mlr3 (>= 1.2.0), mlr3tuning (>= 1.4.0), R (>= 3.1.0) Imports: @@ -56,8 +57,9 @@ Suggests: emoa, fastGHQuad, lhs, + libcmaesr, + mlr3learners (>= 0.12.0), mirai, - mlr3learners (>= 0.7.0), mlr3pipelines (>= 0.5.2), nloptr, ranger, @@ -66,14 +68,17 @@ Suggests: redux, rush, stringi, - testthat (>= 3.0.0) + testthat (>= 3.0.0), +Remotes: + mlr-org/bbotk, + mlr-org/libcmaesr ByteCompile: no Encoding: UTF-8 Config/testthat/edition: 3 Config/testthat/parallel: false NeedsCompilation: yes Roxygen: list(markdown = TRUE, r6 = TRUE) -RoxygenNote: 7.3.2 +RoxygenNote: 7.3.3 Collate: 'mlr_acqfunctions.R' 'AcqFunction.R' @@ -92,10 +97,16 @@ Collate: 'AcqFunctionStochasticCB.R' 'AcqFunctionStochasticEI.R' 'AcqOptimizer.R' + 'AcqOptimizerCmaes.R' + 'AcqOptimizerDirect.R' + 'AcqOptimizerLbfgsb.R' + 'AcqOptimizerLocalSearch.R' + 'AcqOptimzerRandomSearch.R' 'mlr_input_trafos.R' 'InputTrafo.R' 'InputTrafoUnitcube.R' 'aaa.R' + 'LearnerRegrRangerMbo.R' 'OptimizerADBO.R' 'OptimizerAsyncMbo.R' 'OptimizerMbo.R' @@ -108,8 +119,10 @@ Collate: 'ResultAssignerArchive.R' 'ResultAssignerSurrogate.R' 'Surrogate.R' + 'SurrogateGP.R' 'SurrogateLearner.R' 'SurrogateLearnerCollection.R' + 'SurrogateRF.R' 'TunerADBO.R' 'TunerAsyncMbo.R' 'TunerMbo.R' diff --git a/NAMESPACE b/NAMESPACE index b4710198..58a83d8d 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -5,6 +5,7 @@ S3method(as.data.table,DictionaryInputTrafo) S3method(as.data.table,DictionaryLoopFunction) S3method(as.data.table,DictionaryOutputTrafo) S3method(as.data.table,DictionaryResultAssigner) +S3method(default_values,LearnerRegrRangerMbo) S3method(print,loop_function) export(AcqFunction) export(AcqFunctionAEI) @@ -22,8 +23,14 @@ export(AcqFunctionSmsEgo) export(AcqFunctionStochasticCB) export(AcqFunctionStochasticEI) export(AcqOptimizer) +export(AcqOptimizerCmaes) +export(AcqOptimizerDirect) +export(AcqOptimizerLbfgsb) +export(AcqOptimizerLocalSearch) +export(AcqOptimizerRandomSearch) export(InputTrafo) export(InputTrafoUnitcube) +export(LearnerRegrRangerMbo) export(OptimizerADBO) export(OptimizerAsyncMbo) export(OptimizerMbo) @@ -34,8 +41,10 @@ export(ResultAssigner) export(ResultAssignerArchive) export(ResultAssignerSurrogate) export(Surrogate) +export(SurrogateGP) export(SurrogateLearner) export(SurrogateLearnerCollection) +export(SurrogateRF) export(TunerADBO) export(TunerAsyncMbo) export(TunerMbo) diff --git a/R/AcqOptimizerCmaes.R b/R/AcqOptimizerCmaes.R new file mode 100644 index 00000000..a179dd85 --- /dev/null +++ b/R/AcqOptimizerCmaes.R @@ -0,0 +1,142 @@ +#' @title CMA-ES Acquisition Function Optimizer +#' +#' @include AcqOptimizer.R +#' +#' @description +#' CMA-ES acquisition function optimizer. +#' Calls `cmaes()` from \CRANpkg{libcmaesr}. +#' The default algorithm is `"abipop"` with unlimited restarts and a budget of `100 * D^2` function evaluations, where `D` is the dimension of the search space. +#' For the meaning of the control parameters, see `libcmaesr::cmaes_control()`. +#' +#' @section Termination Parameters: +#' The following termination parameters can be used. +#' +#' \describe{ +#' \item{`max_fevals`}{`integer(1)`\cr +#' Maximum number of function evaluations. +#' Deactivate with `NA`. +#' Default is `100 * D^2`, where `D` is the dimension of the search space.} +#' \item{`max_iter`}{`integer(1)`\cr +#' Maximum number of iterations. +#' Deactivate with `NA`.} +#' \item{`ftarget`}{`numeric(1)`\cr +#' Target function value. +#' Deactivate with `NA`.} +#' \item{`f_tolerance`}{`numeric(1)`\cr +#' Function tolerance. +#' Deactivate with `NA`.} +#' \item{`x_tolerance`}{`numeric(1)`\cr +#' Parameter tolerance. +#' Deactivate with `NA`.} +#' } +#' +#' @export +AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", + inherit = AcqOptimizer, + public = list( + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + #' + #' @param acq_function (`NULL` | [AcqFunction]). + initialize = function(acq_function = NULL) { + self$acq_function = assert_r6(acq_function, "AcqFunction", null.ok = TRUE) + param_set = ps( + algo = p_fct(init = "abipop", levels = c( + "cmaes", + "ipop", + "bipop", + "acmaes", + "aipop", + "abipop", + "sepcmaes", + "sepipop", + "sepbipop", + "sepacmaes", + "sepaipop", + "sepabipop", + "vdcma", + "vdipopcma", + "vdbipopcma")), + lambda = p_int(lower = 1L, default = NA_integer_, special_vals = list(NA_integer_)), + sigma = p_dbl(default = NA_real_, special_vals = list(NA_real_)), + max_restarts = p_int(lower = 1L, special_vals = list(NA), init = 1e5L), + tpa = p_int(default = NA_integer_, special_vals = list(NA_integer_)), + tpa_dsigma = p_dbl(default = NA_real_, special_vals = list(NA_real_)), + seed = p_int(default = NA_integer_, special_vals = list(NA_integer_)), + quiet = p_lgl(default = FALSE), + # internal termination criteria + max_fevals = p_int(lower = 1L, special_vals = list(NA_integer_)), + max_iter = p_int(lower = 1L, default = NA_integer_, special_vals = list(NA_integer_)), + ftarget = p_dbl(default = NA_real_, special_vals = list(NA_real_)), + f_tolerance = p_dbl(default = NA_real_, special_vals = list(NA_real_)), + x_tolerance = p_dbl(default = NA_real_, special_vals = list(NA_real_)), + catch_errors = p_lgl(init = TRUE) + ) + private$.param_set = param_set + }, + + #' @description + #' Optimize the acquisition function. + #' + #' @return [data.table::data.table()] with 1 row per candidate. + optimize = function() { + pv = self$param_set$values + + if (is.null(pv$max_fevals)) { + pv$max_fevals = 100 * self$acq_function$domain$length^2 + } + + fun = get_private(self$acq_function)$.fun + constants = self$acq_function$constants$values + direction = self$acq_function$codomain$direction + + control = invoke(libcmaesr::cmaes_control, maximize = direction == -1L, .args = pv[names(pv) %in% formalArgs(libcmaesr::cmaes_control)]) + + wrapper = function(xmat) { + xdt = set_names(as.data.table(xmat), self$acq_function$domain$ids()) + mlr3misc::invoke(fun, xdt = xdt, .args = constants)[[1]] + } + + lower = self$acq_function$domain$lower + upper = self$acq_function$domain$upper + x0 = as.numeric(self$acq_function$archive$best()[, self$acq_function$domain$ids(), with = FALSE]) + + # add saveguard_epsilon to x0 + saveguard_epsilon = 1e-5 + x0[x0 < lower] = x0[x0 < lower] + saveguard_epsilon + x0[x0 > upper] = x0[x0 > upper] - saveguard_epsilon + + optimize = function() { + libcmaesr::cmaes( + objective = wrapper, + x0 = x0, + lower = lower, + upper = upper, + batch = TRUE, + control = control) + } + + if (pv$catch_errors) { + tryCatch({ + res = optimize() + }, error = function(error_condition) { + lg$warn(error_condition$message) + stop(set_class(list(message = error_condition$message, call = NULL), classes = c("acq_optimizer_error", "mbo_error", "error", "condition"))) + }) + } else { + res = optimize() + } + as.data.table(as.list(set_names(c(res$x, res$y * direction), c(self$acq_function$domain$ids(), self$acq_function$codomain$ids())))) + } + ), + + active = list( + #' @template field_print_id + print_id = function(rhs) { + assert_ro_binding(rhs) + "(OptimizerBatchCmaes)" + } + ) +) + diff --git a/R/AcqOptimizerDirect.R b/R/AcqOptimizerDirect.R new file mode 100644 index 00000000..8f6256a3 --- /dev/null +++ b/R/AcqOptimizerDirect.R @@ -0,0 +1,171 @@ +#' @title Direct Acquisition Function Optimizer +#' +#' @include AcqOptimizer.R +#' +#' @description +#' Direct acquisition function optimizer. +#' Calls `nloptr()` from \CRANpkg{nloptr}. +#' In its default setting, the algorithm restarts `5 * D` times and runs at most for `100 * D^2` function evaluations, where `D` is the dimension of the search space. +#' Each run stops when the relative tolerance of the parameters is less than `10^-4`. +#' The first iteration starts with the best point in the archive and the next iterations start from a random point. +#' +#' @section Parameters: +#' \describe{ +#' \item{`restart_strategy`}{`character(1)`\cr +#' Restart strategy. +#' Can be `"none"` or `"random"`. +#' Default is `"none"`. +#' } +#' \item{`max_restarts`}{`integer(1)`\cr +#' Maximum number of restarts. +#' Default is `5 * D` (Default).} +#' +#' @note +#' If the restart strategy is `"none"`, the optimizer starts with the best point in the archive. +#' The optimization stops when one of the stopping criteria is met. +#' +#' If `restart_strategy` is `"random"`, the optimizer runs at least for `maxeval` iterations. +#' The first iteration starts with the best point in the archive and stops when one of the stopping criteria is met. +#' The next iterations start from a random point. +#' +#' @section Termination Parameters: +#' The following termination parameters can be used. +#' +#' \describe{ +#' \item{`stopval`}{`numeric(1)`\cr +#' Stop value. +#' Deactivate with `-Inf` (Default).} +#' \item{`maxtime`}{`integer(1)`\cr +#' Maximum time. +#' Deactivate with `-1L` (Default).} +#' \item{`maxeval`}{`integer(1)`\cr +#' Maximum number of evaluations. +#' Default is `100 * D^2`, where `D` is the dimension of the search space. +#' Deactivate with `-1L`.} +#' \item{`xtol_rel`}{`numeric(1)`\cr +#' Relative tolerance of the parameters. +#' Default is `10^-4`. +#' Deactivate with `-1`.} +#' \item{`xtol_abs`}{`numeric(1)`\cr +#' Absolute tolerance of the parameters. +#' Deactivate with `-1` (Default).} +#' \item{`ftol_rel`}{`numeric(1)`\cr +#' Relative tolerance of the objective function. +#' Deactivate with `-1`. (Default).} +#' \item{`ftol_abs`}{`numeric(1)`\cr +#' Absolute tolerance of the objective function. +#' Deactivate with `-1` (Default).} +#' } +#' +#' @export +AcqOptimizerDirect = R6Class("AcqOptimizerDirect", + inherit = AcqOptimizer, + public = list( + + #' @field state (`list()`)\cr + #' List of [nloptr::nloptr()] results. + state = NULL, + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + #' + #' @param acq_function (`NULL` | [AcqFunction]). + initialize = function(acq_function = NULL) { + self$acq_function = assert_r6(acq_function, "AcqFunction", null.ok = TRUE) + param_set = ps( + maxeval = p_int(lower = 1, init = 1000L, special_vals = list(-1)), + stopval = p_dbl(default = -Inf, lower = -Inf, upper = Inf), + xtol_rel = p_dbl(default = 1e-04, lower = 0, upper = Inf, special_vals = list(-1)), + xtol_abs = p_dbl(default = 0, lower = 0, upper = Inf, special_vals = list(-1)), + ftol_rel = p_dbl(default = 0, lower = 0, upper = Inf, special_vals = list(-1)), + ftol_abs = p_dbl(default = 0, lower = 0, upper = Inf, special_vals = list(-1)), + minf_max = p_dbl(default = -Inf), + restart_strategy = p_fct(levels = c("none", "random"), init = "random"), + max_restarts = p_int(lower = 0L), + catch_errors = p_lgl(init = TRUE) + ) + private$.param_set = param_set + }, + + #' @description + #' Optimize the acquisition function. + #' + #' @return [data.table::data.table()] with 1 row per candidate. + optimize = function() { + pv = self$param_set$values + restart_strategy = pv$restart_strategy + max_restarts = pv$max_restarts + pv$max_restarts = NULL + pv$restart_strategy = NULL + + wrapper = function(x, fun, constants, direction) { + xdt = as.data.table(as.list(set_names(x, self$acq_function$domain$ids()))) + res = mlr3misc::invoke(fun, xdt = xdt, .args = constants)[[1]] + res * direction + } + + fun = get_private(self$acq_function)$.fun + constants = self$acq_function$constants$values + direction = self$acq_function$codomain$direction + + y = Inf + n = 0L + i = 0L + maxeval = if (pv$maxeval == -1) Inf else pv$maxeval + while (n < maxeval && i <= max_restarts) { + i = i + 1L + + x0 = if (i == 1L) { + as.numeric(self$acq_function$archive$best()[, self$acq_function$domain$ids(), with = FALSE]) + } else { + # random restart + as.numeric(generate_design_random(self$acq_function$domain, n = 1)$data) + } + + optimize = function() { + invoke(nloptr::nloptr, + eval_f = wrapper, + lb = self$acq_function$domain$lower, + ub = self$acq_function$domain$upper, + opts = insert_named(pv, list(algorithm = "NLOPT_GN_DIRECT_L", maxeval = maxeval - n)), + eval_grad_f = NULL, + x0 = x0, + fun = fun, + constants = constants, + direction = direction) + } + + if (pv$catch_errors) { + tryCatch({ + res = optimize() + }, error = function(error_condition) { + lg$warn(error_condition$message) + stop(set_class(list(message = error_condition$message, call = NULL), classes = c("acq_optimizer_error", "mbo_error", "error", "condition"))) + }) + } else { + res = optimize() + } + + if (res$objective < y) { + y = res$objective + x = res$solution + } + + n = n + res$iterations + + self$state = c(self$state, set_names(list(list(model = res, start = x0)), paste0("iteration_", i))) + + if (restart_strategy == "none") break + } + as.data.table(as.list(set_names(c(x, y * direction), c(self$acq_function$domain$ids(), self$acq_function$codomain$ids())))) + } + ), + + active = list( + #' @template field_print_id + print_id = function(rhs) { + assert_ro_binding(rhs) + "(OptimizerDirect)" + } + ) +) diff --git a/R/AcqOptimizerLbfgsb.R b/R/AcqOptimizerLbfgsb.R new file mode 100644 index 00000000..37f00543 --- /dev/null +++ b/R/AcqOptimizerLbfgsb.R @@ -0,0 +1,159 @@ +#' @title L-BFGS-B Acquisition Function Optimizer +#' +#' @description +#' If `restart_strategy` is `"random"`, the optimizer runs for `n_iterations` iterations. +#' Each iteration starts with a random search of size `random_restart_size`. +#' The best point is used as the start point for the L-BFGS-B optimization. +#' +#' If `restart_strategy` is `"none"`, the only the L-BFGS-B optimization is performed. +#' The start point is the best point in the archive. +#' +#' @export +#' @export +AcqOptimizerLbfgsb = R6Class("AcqOptimizerLbfgsb", + inherit = AcqOptimizer, + public = list( + + #' @field state (`list()`)\cr + #' List of [nloptr::nloptr()] results. + state = NULL, + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + #' + #' @param acq_function (`NULL` | [AcqFunction]). + initialize = function(acq_function = NULL) { + self$acq_function = assert_r6(acq_function, "AcqFunction", null.ok = TRUE) + param_set = ps( + maxeval = p_int(lower = 1, init = 1000L, special_vals = list(-1)), + stopval = p_dbl(default = -Inf, lower = -Inf, upper = Inf), + xtol_rel = p_dbl(default = 1e-04, lower = 0, upper = Inf, special_vals = list(-1)), + xtol_abs = p_dbl(default = 0, lower = 0, upper = Inf, special_vals = list(-1)), + ftol_rel = p_dbl(default = 0, lower = 0, upper = Inf, special_vals = list(-1)), + ftol_abs = p_dbl(default = 0, lower = 0, upper = Inf, special_vals = list(-1)), + minf_max = p_dbl(default = -Inf), + restart_strategy = p_fct(levels = c("none", "random"), init = "none"), + n_restarts = p_int(lower = 0L, init = 0L), + max_restarts = p_int(lower = 0L, init = 0L) + # n_candidates = p_int(lower = 1, default = 1L), + # logging_level = p_fct(levels = c("fatal", "error", "warn", "info", "debug", "trace"), default = "warn"), + # warmstart = p_lgl(default = FALSE), + # warmstart_size = p_int(lower = 1L, special_vals = list("all")), + # skip_already_evaluated = p_lgl(default = TRUE), + # catch_errors = p_lgl(default = TRUE) + ) + # ps$values = list(n_candidates = 1, logging_level = "warn", warmstart = FALSE, skip_already_evaluated = TRUE, catch_errors = TRUE) + # ps$add_dep("warmstart_size", on = "warmstart", cond = CondEqual$new(TRUE)) + private$.param_set = param_set + }, + + #' @description + #' Optimize the acquisition function. + #' + #' @return [data.table::data.table()] with 1 row per candidate. + optimize = function() { + pv = self$param_set$values + restart_strategy = pv$restart_strategy + max_restarts = pv$max_restarts + pv$max_restarts = NULL + pv$restart_strategy = NULL + + wrapper = function(x, fun, constants, direction) { + xdt = as.data.table(as.list(set_names(x, self$acq_function$domain$ids()))) + res = mlr3misc::invoke(fun, xdt = xdt, .args = constants)[[1]] + res * direction + } + + fun = get_private(self$acq_function)$.fun + constants = self$acq_function$constants$values + direction = self$acq_function$codomain$direction + + y = Inf + n = 0L + i = 0L + maxeval = if (pv$maxeval == -1) Inf else pv$maxeval + while (n < maxeval && i <= max_restarts) { + i = i + 1L + + x0 = if (i == 1L) { + as.numeric(self$acq_function$archive$best()[, self$acq_function$domain$ids(), with = FALSE]) + } else { + # random restart + as.numeric(generate_design_random(self$acq_function$domain, n = 1)$data) + } + + eval_grad_f = function(x, fun, constants, direction) { + invoke(nloptr::nl.grad, x0 = x, fn = wrapper, fun = fun, constants = constants, direction = direction) + } + saveguard_epsilon = 1e-5 + + # optimize with nloptr + res = invoke(nloptr::nloptr, + eval_f = wrapper, + lb = self$acq_function$domain$lower + saveguard_epsilon, + ub = self$acq_function$domain$upper - saveguard_epsilon, + opts = insert_named(pv, list(algorithm = "NLOPT_LD_LBFGS", maxeval = maxeval - n)), + eval_grad_f = eval_grad_f, + x0 = x0, + fun = fun, + constants = constants, + direction = direction) + + if (res$objective < y) { + y = res$objective + x = res$solution + } + + n = n + res$iterations + + self$state = c(self$state, set_names(list(list(model = res, start = x0)), paste0("iteration_", i))) + + if (restart_strategy == "none") break + } + as.data.table(as.list(set_names(c(x, y * direction), c(self$acq_function$domain$ids(), self$acq_function$codomain$ids())))) + }, + + #' @description + #' Reset the acquisition function optimizer. + #' + #' Currently not used. + reset = function() { + + } + ), + + active = list( + #' @template field_print_id + print_id = function(rhs) { + if (missing(rhs)) { + paste0("(", class(self$optimizer)[1L], " | ", class(self$terminator)[1L], ")") + } else { + stop("$print_id is read-only.") + } + }, + + #' @field param_set ([paradox::ParamSet])\cr + #' Set of hyperparameters. + param_set = function(rhs) { + if (!missing(rhs) && !identical(rhs, private$.param_set)) { + stop("$param_set is read-only.") + } + private$.param_set + } + ), + + private = list( + .param_set = NULL, + + deep_clone = function(name, value) { + switch(name, + optimizer = value$clone(deep = TRUE), + terminator = value$clone(deep = TRUE), + acq_function = if (!is.null(value)) value$clone(deep = TRUE) else NULL, + .param_set = value$clone(deep = TRUE), + value + ) + } + ) +) + diff --git a/R/AcqOptimizerLocalSearch.R b/R/AcqOptimizerLocalSearch.R new file mode 100644 index 00000000..c6610fd6 --- /dev/null +++ b/R/AcqOptimizerLocalSearch.R @@ -0,0 +1,92 @@ +#' @title Local Search Acquisition Function Optimizer +#' +#' @export +AcqOptimizerLocalSearch = R6Class("AcqOptimizerLocalSearch", + inherit = AcqOptimizer, + public = list( + + #' @field state (`list()`)\cr + #' List of [cmaes::cma_es()] results. + state = NULL, + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + #' + #' @param acq_function (`NULL` | [AcqFunction]). + initialize = function(acq_function = NULL) { + self$acq_function = assert_r6(acq_function, "AcqFunction", null.ok = TRUE) + param_set = ps( + n_searches = p_int(lower = 1L, default = 10L), + n_steps = p_int(lower = 0L, default = 5L), + n_neighs = p_int(lower = 1L, default = 10L), + mut_sd = p_dbl(lower = 0, default = 0.1), + stagnate_max = p_int(lower = 1L, default = 10L) + ) + private$.param_set = param_set + }, + + #' @description + #' Optimize the acquisition function. + #' + #' @return [data.table::data.table()] with 1 row per candidate. + optimize = function() { + pv = self$param_set$get_values() + control = invoke(bbotk::local_search_control, minimize = self$acq_function$codomain$direction == 1L, .args = pv) + + wrapper = function(xdt) { + mlr3misc::invoke(self$acq_function$fun, xdt = xdt, .args = self$acq_function$constants$values)[[1]] + } + + res = invoke(bbotk::local_search, + objective = wrapper, + search_space = self$acq_function$domain, + control = control + ) + + as.data.table(as.list(set_names(c(res$x, res$y), c(self$acq_function$domain$ids(), self$acq_function$codomain$ids())))) + }, + + #' @description + #' Reset the acquisition function optimizer. + #' + #' Currently not used. + reset = function() { + + } + ), + + active = list( + #' @template field_print_id + print_id = function(rhs) { + if (missing(rhs)) { + paste0("(", class(self$optimizer)[1L], " | ", class(self$terminator)[1L], ")") + } else { + stop("$print_id is read-only.") + } + }, + + #' @field param_set ([paradox::ParamSet])\cr + #' Set of hyperparameters. + param_set = function(rhs) { + if (!missing(rhs) && !identical(rhs, private$.param_set)) { + stop("$param_set is read-only.") + } + private$.param_set + } + ), + + private = list( + .param_set = NULL, + + deep_clone = function(name, value) { + switch(name, + optimizer = value$clone(deep = TRUE), + terminator = value$clone(deep = TRUE), + acq_function = if (!is.null(value)) value$clone(deep = TRUE) else NULL, + .param_set = value$clone(deep = TRUE), + value + ) + } + ) +) + diff --git a/R/AcqOptimzerRandomSearch.R b/R/AcqOptimzerRandomSearch.R new file mode 100644 index 00000000..f96f49b2 --- /dev/null +++ b/R/AcqOptimzerRandomSearch.R @@ -0,0 +1,118 @@ +#' @title Random Search Acquisition Function Optimizer +#' +#' @export +AcqOptimizerRandomSearch = R6Class("AcqOptimizerRandomSearch", + inherit = AcqOptimizer, + public = list( + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + #' + #' @param acq_function (`NULL` | [AcqFunction]). + initialize = function(acq_function = NULL) { + self$acq_function = assert_r6(acq_function, "AcqFunction", null.ok = TRUE) + param_set = ps( + max_fevals = p_int(lower = 1, init = 1000L) + # maxeval = p_int(lower = 1, init = 1000L, special_vals = list(-1)), + # fnscale = p_dbl(default = 1), + # #maxit = p_int(lower = 1L), + # stopfitness = p_dbl(default = -Inf), + # keep.best = p_lgl(default = TRUE), + # sigma = p_uty(default = 0.5), + # mu = p_int(lower = 1L), + # lambda = p_int(lower = 1L), + # weights = p_uty(), + # damps = p_dbl(), + # cs = p_dbl(), + # ccum = p_dbl(), + # ccov.1 = p_dbl(lower = 0), + # ccov.mu = p_dbl(lower = 0), + # diag.sigma = p_lgl(default = FALSE), + # diag.eigen = p_lgl(default = FALSE), + # diag.pop = p_lgl(default = FALSE), + # diag.value = p_lgl(default = FALSE), + # stop.tolx = p_dbl(), # undocumented stop criterion + # restart_strategy = p_fct(levels = c("none", "ipop"), init = "none"), + # n_restarts = p_int(lower = 0L, init = 0L), + # population_multiplier = p_int(lower = 1, init = 2L) + # start_values = p_fct(default = "random", levels = c("random", "center", "custom")), + # start = p_uty(default = NULL, depends = start_values == "custom"), + # n_candidates = p_int(lower = 1, default = 1L), + # logging_level = p_fct(levels = c("fatal", "error", "warn", "info", "debug", "trace"), default = "warn"), + # warmstart = p_lgl(default = FALSE), + # warmstart_size = p_int(lower = 1L, special_vals = list("all")), + # skip_already_evaluated = p_lgl(default = TRUE), + # catch_errors = p_lgl(default = TRUE) + ) + # ps$values = list(n_candidates = 1, logging_level = "warn", warmstart = FALSE, skip_already_evaluated = TRUE, catch_errors = TRUE) + # ps$add_dep("warmstart_size", on = "warmstart", cond = CondEqual$new(TRUE)) + private$.param_set = param_set + }, + + #' @description + #' Optimize the acquisition function. + #' + #' @return [data.table::data.table()] with 1 row per candidate. + optimize = function() { + pv = self$param_set$values + + fun = get_private(self$acq_function)$.fun + constants = self$acq_function$constants$values + direction = self$acq_function$codomain$direction + + xdt = generate_design_random(self$acq_function$domain, n = pv$max_fevals)$data + + ys = mlr3misc::invoke(fun, xdt = xdt, .args = constants)[[1]] + + id = if (direction == 1) which.min(ys) else which.max(ys) + x = xdt[id, ] + y = ys[id] + + set(x, j = self$acq_function$codomain$ids(), value = y) + x + }, + + #' @description + #' Reset the acquisition function optimizer. + #' + #' Currently not used. + reset = function() { + + } + ), + + active = list( + #' @template field_print_id + print_id = function(rhs) { + if (missing(rhs)) { + paste0("(", class(self$optimizer)[1L], " | ", class(self$terminator)[1L], ")") + } else { + stop("$print_id is read-only.") + } + }, + + #' @field param_set ([paradox::ParamSet])\cr + #' Set of hyperparameters. + param_set = function(rhs) { + if (!missing(rhs) && !identical(rhs, private$.param_set)) { + stop("$param_set is read-only.") + } + private$.param_set + } + ), + + private = list( + .param_set = NULL, + + deep_clone = function(name, value) { + switch(name, + optimizer = value$clone(deep = TRUE), + terminator = value$clone(deep = TRUE), + acq_function = if (!is.null(value)) value$clone(deep = TRUE) else NULL, + .param_set = value$clone(deep = TRUE), + value + ) + } + ) +) + diff --git a/R/LearnerRegrRangerMbo.R b/R/LearnerRegrRangerMbo.R new file mode 100644 index 00000000..a3943365 --- /dev/null +++ b/R/LearnerRegrRangerMbo.R @@ -0,0 +1,196 @@ +#' @title Custom Ranger Regression Learner for Model Based Optimization +#' +#' @name mlr_learners_regr.ranger +#' +#' @description +#' Random regression forest. +#' Calls [ranger::ranger()] from package \CRANpkg{ranger}. +#' +#' @export +LearnerRegrRangerMbo = R6Class("LearnerRegrRangerMbo", + inherit = LearnerRegr, + public = list( + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + initialize = function() { + ps = ps( + alpha = p_dbl(default = 0.5, tags = "train", depends = quote(splitrule == "maxstat")), + always.split.variables = p_uty(tags = "train"), + holdout = p_lgl(default = FALSE, tags = "train"), + importance = p_fct(c("none", "impurity", "impurity_corrected", "permutation"), tags = "train"), + keep.inbag = p_lgl(default = FALSE, tags = "train"), + max.depth = p_int(default = NULL, lower = 0L, special_vals = list(NULL), tags = "train"), + min.bucket = p_int(1L, default = 1L, tags = "train"), + min.node.size = p_int(1L, default = 5L, special_vals = list(NULL), tags = "train"), + minprop = p_dbl(default = 0.1, tags = "train", depends = quote(splitrule == "maxstat")), + mtry = p_int(lower = 1L, special_vals = list(NULL), tags = "train"), + mtry.ratio = p_dbl(lower = 0, upper = 1, tags = "train"), + node.stats = p_lgl(default = FALSE, tags = "train"), + num.random.splits = p_int(1L, default = 1L, tags = "train", depends = quote(splitrule == "extratrees")), + num.threads = p_int(1L, default = 1L, tags = c("train", "predict", "threads")), + num.trees = p_int(1L, default = 500L, tags = c("train", "predict", "hotstart")), + oob.error = p_lgl(default = TRUE, tags = "train"), + quantreg = p_lgl(default = FALSE, tags = "train"), + regularization.factor = p_uty(default = 1, tags = "train"), + regularization.usedepth = p_lgl(default = FALSE, tags = "train"), + replace = p_lgl(default = TRUE, tags = "train"), + respect.unordered.factors = p_fct(c("ignore", "order", "partition"), default = "ignore", tags = "train"), + sample.fraction = p_dbl(0L, 1L, tags = "train"), + save.memory = p_lgl(default = FALSE, tags = "train"), + scale.permutation.importance = p_lgl(default = FALSE, tags = "train", depends = quote(importance == "permutation")), + se.method = p_fct(c("jack", "infjack", "simple", "law_of_total_variance"), default = "infjack", tags = c("train", "predict")), # FIXME: only works if predict_type == "se" + seed = p_int(default = NULL, special_vals = list(NULL), tags = c("train", "predict")), + split.select.weights = p_uty(default = NULL, tags = "train"), + splitrule = p_fct(c("variance", "extratrees", "maxstat"), default = "variance", tags = "train"), + verbose = p_lgl(default = TRUE, tags = c("train", "predict")), + write.forest = p_lgl(default = TRUE, tags = "train") + ) + + ps$values = list(num.threads = 1L) + + super$initialize( + id = "regr.ranger_mbo", + param_set = ps, + predict_types = c("response", "se"), + feature_types = c("logical", "integer", "numeric", "character", "factor", "ordered"), + properties = c("weights", "importance", "oob_error", "hotstart_backward"), + packages = c("mlr3learners", "ranger"), + label = "Custom MBO Random Forest", + man = "mlr3mbo::mlr_learners_regr.ranger_mbo" + ) + }, + + #' @description + #' The importance scores are extracted from the model slot `variable.importance`. + #' Parameter `importance.mode` must be set to `"impurity"`, `"impurity_corrected"`, or + #' `"permutation"` + #' + #' @return Named `numeric()`. + importance = function() { + if (is.null(self$model)) { + stopf("No model stored") + } + if (self$model$importance.mode == "none") { + stopf("No importance stored") + } + + sort(self$model$variable.importance, decreasing = TRUE) + }, + + #' @description + #' The out-of-bag error, extracted from model slot `prediction.error`. + #' + #' @return `numeric(1)`. + oob_error = function() { + if (is.null(self$model)) { + stopf("No model stored") + } + self$model$prediction.error + } + ), + private = list( + .train = function(task) { + pv = self$param_set$get_values(tags = "train") + pv = mlr3learners:::convert_ratio(pv, "mtry", "mtry.ratio", length(task$feature_names)) + if ("se.method" %in% names(pv)) { + if (self$predict_type != "se") { + stop('$predict_type must be "se" if "se.method" is set to "simple" or "law_of_total_variance".') + } + pv[["se.method"]] = NULL + } + + if (self$predict_type == "se") { + pv$keep.inbag = TRUE # nolint + } + + model = mlr3misc::invoke(ranger::ranger, + dependent.variable.name = task$target_names, + data = task$data(), + case.weights = task$weights$weight, + .args = pv + ) + + if (isTRUE(self$param_set$get_values()[["se.method"]] %in% c("simple", "law_of_total_variance"))) { + data = mlr3learners:::ordered_features(task, self) + prediction_nodes = mlr3misc::invoke(predict, model, data = data, type = "terminalNodes", .args = pv[setdiff(names(pv), "se.method")], predict.all = TRUE) + y = task$data(cols = task$target_names)[[1L]] + observation_node_table = prediction_nodes$predictions + n_trees = NCOL(observation_node_table) + unique_nodes_per_tree = apply(observation_node_table, MARGIN = 2L, FUN = unique, simplify = FALSE) + mu_sigma2_per_node_per_tree = lapply(seq_len(n_trees), function(tree) { + nodes = unique_nodes_per_tree[[tree]] + setNames(lapply(nodes, function(node) { + y_tmp = y[observation_node_table[, tree] == node] + c(mu = mean(y_tmp), sigma2 = if (length(y_tmp) > 1L) var(y_tmp) else 0) + }), nm = nodes) + }) + list(model = model, mu_sigma2_per_node_per_tree = mu_sigma2_per_node_per_tree) + } else { + list(model = model) + } + }, + .predict = function(task) { + pv = self$param_set$get_values(tags = "predict") + newdata = mlr3learners:::ordered_features(task, self) + + if (isTRUE(pv$se.method == "law_of_total_variance")) { + prediction_nodes = mlr3misc::invoke(predict, self$model$model, data = newdata, type = "terminalNodes", .args = pv[setdiff(names(pv), "se.method")], predict.all = TRUE) + n_observations = NROW(prediction_nodes$predictions) + n_trees = length(self$model$mu_sigma2_per_node_per_tree) + response = numeric(n_observations) + se = numeric(n_observations) + for (i in seq_len(n_observations)) { + mu_sigma2_per_tree = lapply(seq_len(n_trees), function(tree) { + self$model$mu_sigma2_per_node_per_tree[[tree]][[as.character(prediction_nodes$predictions[i, tree])]] + }) + mus = sapply(mu_sigma2_per_tree, "[[", 1) + sigmas2 = sapply(mu_sigma2_per_tree, "[[", 2) + response[i] = mean(mus) + # law of total variance assuming a mixture of normal distributions for each tree + se[i] = sqrt(mean((mus ^ 2) + sigmas2) - (response[i] ^ 2)) + } + se[se < .Machine$double.eps | is.na(se)] = 1e-8 + list(response = response, se = se) + } else if (isTRUE(pv$se.method == "simple")) { + prediction_nodes = mlr3misc::invoke(predict, self$model$model, data = newdata, type = "terminalNodes", .args = pv[setdiff(names(pv), "se.method")], predict.all = TRUE) + n_observations = NROW(prediction_nodes$predictions) + n_trees = length(self$model$mu_sigma2_per_node_per_tree) + response = numeric(n_observations) + se = numeric(n_observations) + for (i in seq_len(n_observations)) { + mu_sigma2_per_tree = lapply(seq_len(n_trees), function(tree) { + self$model$mu_sigma2_per_node_per_tree[[tree]][[as.character(prediction_nodes$predictions[i, tree])]] + }) + mus = sapply(mu_sigma2_per_tree, "[[", 1) + response[i] = mean(mus) + se[i] = sqrt(var(mus)) + } + list(response = response, se = se) + } else { + prediction = mlr3misc::invoke(predict, self$model$model, data = newdata, type = self$predict_type, .args = pv) + list(response = prediction$predictions, se = prediction$se) + } + }, + + .hotstart = function(task) { + model = self$model + model$num.trees = self$param_set$values$num.trees + model + } + ) +) + +#' @export +default_values.LearnerRegrRangerMbo = function(x, search_space, task, ...) { # nolint + special_defaults = list( + mtry = floor(sqrt(length(task$feature_names))), + mtry.ratio = floor(sqrt(length(task$feature_names))) / length(task$feature_names), + sample.fraction = 1 + ) + defaults = insert_named(default_values(x$param_set), special_defaults) + defaults[search_space$ids()] +} + +#' @include aaa.R +learners[["regr.ranger_mbo"]] = LearnerRegrRangerMbo diff --git a/R/SurrogateGP.R b/R/SurrogateGP.R new file mode 100644 index 00000000..5303f6eb --- /dev/null +++ b/R/SurrogateGP.R @@ -0,0 +1,293 @@ +#' @title Surrogate Model Containing a Gaussian Process +#' +#' @description +#' Surrogate model containing a single Gaussian Process via [DiceKriging::km()] from package \CRANpkg{DiceKriging}. +#' Update and predict methods are inspired from [mlr3learners::LearnerRegrKM] from package \CRANpkg{mlr3learners}. +#' +#' Compared to using [mlr3learners::LearnerRegrKM] within a [SurrogateLearner] the update and predict methods of this class are much more efficient +#' as they skip many assertions and checks naturally arising when using a [SurrogateLearner] wrapping a [mlr3learners::LearnerRegrKM]. +#' +#' @section Parameters: +#' \describe{ +#' \item{`catch_errors`}{`logical(1)`\cr +#' Should errors during updating the surrogate be caught and propagated to the `loop_function` which can then handle +#' the failed acquisition function optimization (as a result of the failed surrogate) appropriately by, e.g., proposing a randomly sampled point for evaluation? +#' Default is `TRUE`. +#' } +#' \item{`impute_method`}{`character(1)`\cr +#' Method to impute missing values in the case of updating on an asynchronous [bbotk::ArchiveAsync] with pending evaluations. +#' Can be `"mean"` to use mean imputation or `"random"` to sample values uniformly at random between the empirical minimum and maximum. +#' Default is `"random"`. +#' } +#' } +#' For a description of all other parameters related to [DiceKriging::km()] directly, see the documentation of [DiceKriging::km()]. +#' * The predict type hyperparameter "type" defaults to "SK" (simple kriging). +#' * The additional hyperparameter `nugget.stability` is used to overwrite the +#' hyperparameter `nugget` with `nugget.stability * var(y)` before training to +#' improve the numerical stability. We recommend a value of `1e-8`. +#' * The additional hyperparameter `jitter` can be set to add +#' `N(0, [jitter])`-distributed noise to the data before prediction to avoid +#' perfect interpolation. We recommend a value of `1e-12`. +#' +#' @export +#' @examples +#' if (requireNamespace("DiceKriging") & +#' requireNamespace("rgenoud")) { +#' library(bbotk) +#' library(paradox) +#' +#' fun = function(xs) { +#' list(y = xs$x ^ 2) +#' } +#' domain = ps(x = p_dbl(lower = -10, upper = 10)) +#' codomain = ps(y = p_dbl(tags = "minimize")) +#' objective = ObjectiveRFun$new(fun = fun, domain = domain, codomain = codomain) +#' +#' instance = OptimInstanceBatchSingleCrit$new( +#' objective = objective, +#' terminator = trm("evals", n_evals = 5)) +#' +#' xdt = generate_design_random(instance$search_space, n = 4)$data +#' +#' instance$eval_batch(xdt) +#' +#' surrogate = SurrogateGP$new(archive = instance$archive) +#' surrogate$param_set$set_values( +#' covtype = "matern5_2", +#' optim.method = "gen", +#' control = list(trace = FALSE), +#' nugget.stability = 10^-8 +#' ) +#' +#' surrogate$update() +#' +#' surrogate$learner$model +#' } +SurrogateGP = R6Class("SurrogateGP", + inherit = Surrogate, + + public = list( + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + #' + #' @template param_archive_surrogate + #' @template param_col_y_surrogate + #' @template param_cols_x_surrogate + initialize = function(archive = NULL, cols_x = NULL, col_y = NULL) { + assert_r6(archive, classes = "Archive", null.ok = TRUE) + + assert_character(cols_x, min.len = 1L, null.ok = TRUE) + assert_string(col_y, null.ok = TRUE) + + # https://github.com/mlr-org/mlr3learners/blob/51c15755438078fc99b11a9ca0147c7f2dbb96d8/R/LearnerRegrKM.R#L34 + ps = ps( + bias.correct = p_lgl(default = FALSE, tags = "predict"), + checkNames = p_lgl(default = TRUE, tags = "predict"), + coef.cov = p_uty(default = NULL, tags = "train"), + coef.trend = p_uty(default = NULL, tags = "train"), + coef.var = p_uty(default = NULL, tags = "train"), + control = p_uty(default = NULL, tags = "train"), + # cov.compute = p_lgl(default = TRUE, tags = "predict"), + covtype = p_fct(c("gauss", "matern5_2", "matern3_2", "exp", "powexp"), default = "matern5_2", tags = "train"), + estim.method = p_fct(c("MLE", "LOO"), default = "MLE", tags = "train"), + gr = p_lgl(default = TRUE, tags = "train"), + iso = p_lgl(default = FALSE, tags = "train"), + jitter = p_dbl(0, default = 0, tags = "predict"), + kernel = p_uty(default = NULL, tags = "train"), + knots = p_uty(default = NULL, tags = "train", depends = quote(scaling == TRUE)), + light.return = p_lgl(default = FALSE, tags = "predict"), + lower = p_uty(default = NULL, tags = "train"), + multistart = p_int(default = 1, tags = "train", depends = quote(optim.method == "BFGS")), + noise.var = p_uty(default = NULL, tags = "train"), + nugget = p_dbl(tags = "train"), + nugget.estim = p_lgl(default = FALSE, tags = "train"), + nugget.stability = p_dbl(0, default = 0, tags = "train"), + optim.method = p_fct(c("BFGS", "gen"), default = "BFGS", tags = "train"), + parinit = p_uty(default = NULL, tags = "train"), + penalty = p_uty(default = NULL, tags = "train"), + scaling = p_lgl(default = FALSE, tags = "train"), + # se.compute = p_lgl(default = TRUE, tags = "predict"), + type = p_fct(c("SK", "UK"), default = "SK", tags = "predict"), + upper = p_uty(default = NULL, tags = "train"), + catch_errors = p_lgl(tags = "required"), + impute_method = p_fct(c("mean", "random"), tags = "required") + ) + ps$values = list(catch_errors = TRUE, impute_method = "random") + + super$initialize(learner = list(), archive = archive, cols_x = cols_x, cols_y = col_y, param_set = ps) + }, + + #' @description + #' Predict mean response and standard error. + #' + #' @param xdt ([data.table::data.table()])\cr + #' New data. One row per observation. + #' + #' @return [data.table::data.table()] with the columns `mean` and `se`. + predict = function(xdt) { + assert_xdt(xdt) + newdata = as_numeric_matrix(fix_xdt_missing(xdt, cols_x = self$cols_x, archive = self$archive)) + pv = self$param_set$get_values(tags = "predict") + + jitter = pv$jitter + if (!is.null(jitter) && jitter > 0) { + newdata = newdata + stats::rnorm(length(newdata), mean = 0, sd = jitter) + } + + p = invoke(DiceKriging::predict.km, + private$.model, + newdata = newdata, + type = if (is.null(pv$type)) "SK" else pv$type, + se.compute = TRUE, + .args = remove_named(pv, "jitter") + ) + + data.table(mean = p$mean, se = p$sd) + } + ), + + active = list( + + #' @template field_print_id + print_id = function(rhs) { + if (missing(rhs)) { + "DiceKriging::km" + } else { + stop("$print_id is read-only.") + } + }, + + #' @template field_n_learner_surrogate + n_learner = function() { + 1L + }, + + + #' @template field_packages_surrogate + packages = function(rhs) { + if (missing(rhs)) { + "DiceKriging" + } else { + stop("$packages is read-only.") + } + }, + + #' @template field_feature_types_surrogate + feature_types = function(rhs) { + if (missing(rhs)) { + c("logical", "integer", "numeric") + } else { + stop("$feature_types is read-only.") + } + }, + + #' @template field_properties_surrogate + properties = function(rhs) { + if (missing(rhs)) { + character(0L) + } else { + stop("$properties is read-only.") + } + }, + + #' @template field_predict_type_surrogate + predict_type = function(rhs) { + if (missing(rhs)) { + "se" + } else { + stop("$predict_type is read-only.") + } + } + ), + + private = list( + + .model = NULL, + + # Train learner with new data. + .update = function() { + pv = self$param_set$get_values(tags = "train") + design = as_numeric_matrix(self$archive$data[, self$cols_x, with = FALSE]) + response = self$archive$data[[self$cols_y]] + + if (!is.null(pv$optim.method) && pv$optim.method == "gen" && !requireNamespace("rgenoud", quietly = TRUE)) { + stopf("The 'rgenoud' package is required for optimization method 'gen'.") + } + + ns = pv$nugget.stability + if (!is.null(ns)) { + pv$nugget = if (ns == 0) 0 else ns * stats::var(response) + } + + private$.model = invoke(DiceKriging::km, + response = response, + design = design, + control = pv$control, + .args = remove_named(pv, c("control", "nugget.stability")) + ) + self$learner$model = private$.model + invisible(NULL) + }, + + # Train learner with new data. + # Operates on an asynchronous archive and performs imputation as needed. + .update_async = function() { + xydt = self$archive$rush$fetch_tasks_with_state(states = c("queued", "running", "finished"))[, c(self$cols_x, self$cols_y, "state"), with = FALSE] + + if (self$param_set$values$impute_method == "mean") { + mean_y = mean(xydt[[self$cols_y]], na.rm = TRUE) + xydt[c("queued", "running"), (self$cols_y) := mean_y, on = "state"] + } else if (self$param_set$values$impute_method == "random") { + min_y = min(xydt[[self$cols_y]], na.rm = TRUE) + max_y = max(xydt[[self$cols_y]], na.rm = TRUE) + xydt[c("queued", "running"), (self$cols_y) := runif(.N, min = min_y, max = max_y), on = "state"] + } + set(xydt, j = "state", value = NULL) + + pv = self$param_set$get_values(tags = "train") + design = as_numeric_matrix(xydt[, self$cols_x, with = FALSE]) + response = xydt[[self$cols_y]] + + if (!is.null(pv$optim.method) && pv$optim.method == "gen" && !requireNamespace("rgenoud", quietly = TRUE)) { + stopf("The 'rgenoud' package is required for optimization method 'gen'.") + } + + ns = pv$nugget.stability + if (!is.null(ns)) { + pv$nugget = if (ns == 0) 0 else ns * stats::var(response) + } + + private$.model = invoke(DiceKriging::km, + response = response, + design = design, + control = pv$control, + .args = remove_named(pv, c("control", "nugget.stability")) + ) + self$learner$model = private$.model + invisible(NULL) + }, + + .reset = function() { + self$learner = list() + private$.model = NULL + }, + + deep_clone = function(name, value) { + switch(name, + .param_set = value$clone(deep = TRUE), + .archive = value$clone(deep = TRUE), + value + ) + } + ) +) + +# https://github.com/mlr-org/mlr3learners/blob/51c15755438078fc99b11a9ca0147c7f2dbb96d8/R/helpers.R#L16C1-L16C18 +as_numeric_matrix = function(x) { + x = as.matrix(x) + if (is.logical(x)) { + storage.mode(x) = "double" + } + x +} diff --git a/R/SurrogateLearner.R b/R/SurrogateLearner.R index 3a23dab0..a79f179c 100644 --- a/R/SurrogateLearner.R +++ b/R/SurrogateLearner.R @@ -107,30 +107,41 @@ SurrogateLearner = R6Class("SurrogateLearner", #' #' @return [data.table::data.table()] with the columns `mean` and `se`. predict = function(xdt) { - assert_xdt(xdt) - xdt = fix_xdt_missing(copy(xdt), cols_x = self$cols_x, archive = self$archive) + # assert_xdt(xdt) + + # xdt = fix_xdt_missing(copy(xdt), cols_x = self$cols_x, archive = self$archive) + if (!is.null(self$input_trafo)) { xdt = self$input_trafo$transform(xdt) } - # speeding up some checks by constructing the predict task directly instead of relying on predict_newdata - task = self$learner$state$train_task$clone() - set(xdt, j = task$target_names, value = NA_real_) # tasks only have features and the target but we have to set the target to NA - newdata = as_data_backend(xdt) - task$backend = newdata - task$row_roles$use = task$backend$rownames - pred = self$learner$predict(task) + if (!inherits(self$learner, "GraphLearner")) { + pred = self$learner$predict_newdata_fast(xdt) + pred = set_names(pred, c("mean", "se")) + } else { + # speeding up some checks by constructing the predict task directly instead of relying on predict_newdata + task = self$learner$state$train_task$clone() + set(xdt, j = task$target_names, value = NA_real_) # tasks only have features and the target but we have to set the target to NA + newdata = as_data_backend(xdt) + task$backend = newdata + task$row_roles$use = task$backend$rownames + pred = self$learner$predict(task) + pred = if (self$learner$predict_type == "se") { + list(mean = pred$response, se = pred$se) + } else { + list(mean = pred$response) + } + } # slow #pred = self$learner$predict_newdata(newdata = xdt) - pred = if (self$learner$predict_type == "se") { - data.table(mean = pred$response, se = pred$se) - } else { - data.table(mean = pred$response) - } + + + + if (!is.null(self$output_trafo) && self$output_trafo$invert_posterior) { - pred = self$output_trafo$inverse_transform_posterior(pred) + pred = self$output_trafo$inverse_transform_posterior(as.data.table(pred)) } pred } diff --git a/R/SurrogateRF.R b/R/SurrogateRF.R new file mode 100644 index 00000000..f74c23ca --- /dev/null +++ b/R/SurrogateRF.R @@ -0,0 +1,307 @@ +#' @title Surrogate Model Containing a Random Forest +#' +#' @description +#' Surrogate model containing a single Random Forest via [ranger::ranger()] from package \CRANpkg{ranger}. +#' Update and predict methods are inspired from [mlr3learners::LearnerRegrRanger] from package \CRANpkg{mlr3learners}. +#' +#' Compared to using [mlr3learners::LearnerRegrRanger] within a [SurrogateLearner] the update and predict methods of this class are much more efficient +#' as they skip many assertions and checks naturally arising when using a [SurrogateLearner] wrapping a [mlr3learners::LearnerRegrRanger]. +#' +#' @section Parameters: +#' \describe{ +#' \item{`catch_errors`}{`logical(1)`\cr +#' Should errors during updating the surrogate be caught and propagated to the `loop_function` which can then handle +#' the failed acquisition function optimization (as a result of the failed surrogate) appropriately by, e.g., proposing a randomly sampled point for evaluation? +#' Default is `TRUE`. +#' } +#' \item{`impute_method`}{`character(1)`\cr +#' Method to impute missing values in the case of updating on an asynchronous [bbotk::ArchiveAsync] with pending evaluations. +#' Can be `"mean"` to use mean imputation or `"random"` to sample values uniformly at random between the empirical minimum and maximum. +#' Default is `"random"`. +#' } +#' } +#' For a description of all other parameters related to [DiceKriging::km()] directly, see the documentation of [DiceKriging::km()]. +#' * The predict type hyperparameter "type" defaults to "SK" (simple kriging). +#' * The additional hyperparameter `nugget.stability` is used to overwrite the +#' hyperparameter `nugget` with `nugget.stability * var(y)` before training to +#' improve the numerical stability. We recommend a value of `1e-8`. +#' * The additional hyperparameter `jitter` can be set to add +#' `N(0, [jitter])`-distributed noise to the data before prediction to avoid +#' perfect interpolation. We recommend a value of `1e-12`. +#' +#' @export +#' @examples +#' if (requireNamespace("DiceKriging") & +#' requireNamespace("rgenoud")) { +#' library(bbotk) +#' library(paradox) +#' +#' fun = function(xs) { +#' list(y = xs$x ^ 2) +#' } +#' domain = ps(x = p_dbl(lower = -10, upper = 10)) +#' codomain = ps(y = p_dbl(tags = "minimize")) +#' objective = ObjectiveRFun$new(fun = fun, domain = domain, codomain = codomain) +#' +#' instance = OptimInstanceBatchSingleCrit$new( +#' objective = objective, +#' terminator = trm("evals", n_evals = 5)) +#' +#' xdt = generate_design_random(instance$search_space, n = 4)$data +#' +#' instance$eval_batch(xdt) +#' +#' surrogate = SurrogateGP$new(archive = instance$archive) +#' surrogate$param_set$set_values( +#' covtype = "matern5_2", +#' optim.method = "gen", +#' control = list(trace = FALSE), +#' nugget.stability = 10^-8 +#' ) +#' +#' surrogate$update() +#' +#' surrogate$learner$model +#' } +SurrogateRF = R6Class("SurrogateRF", + inherit = Surrogate, + + public = list( + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + #' + #' @template param_archive_surrogate + #' @template param_col_y_surrogate + #' @template param_cols_x_surrogate + initialize = function(archive = NULL, cols_x = NULL, col_y = NULL) { + assert_r6(archive, classes = "Archive", null.ok = TRUE) + + assert_character(cols_x, min.len = 1L, null.ok = TRUE) + assert_string(col_y, null.ok = TRUE) + + # https://github.com/mlr-org/mlr3learners/blob/51c15755438078fc99b11a9ca0147c7f2dbb96d8/R/LearnerRegrKM.R#L34 + ps = ps( + always.split.variables = p_uty(tags = "train"), + holdout = p_lgl(default = FALSE, tags = "train"), + importance = p_fct(c("none", "impurity", "impurity_corrected", "permutation"), tags = "train"), + keep.inbag = p_lgl(default = FALSE, tags = "train"), + max.depth = p_int(default = NULL, lower = 1L, special_vals = list(NULL), tags = "train"), + min.bucket = p_int(1L, default = 1L, tags = "train"), + min.node.size = p_int(1L, default = 5L, special_vals = list(NULL), tags = "train"), + mtry = p_int(lower = 1L, special_vals = list(NULL), tags = "train"), + mtry.ratio = p_dbl(lower = 0, upper = 1, tags = "train"), + na.action = p_fct(c("na.learn", "na.omit", "na.fail"), default = "na.learn", tags = "train"), + node.stats = p_lgl(default = FALSE, tags = "train"), + num.random.splits = p_int(1L, default = 1L, tags = "train", depends = quote(splitrule == "extratrees")), + num.threads = p_int(1L, default = 1L, tags = c("train", "predict", "threads")), + num.trees = p_int(1L, default = 500L, tags = c("train", "predict", "hotstart")), + oob.error = p_lgl(default = TRUE, tags = "train"), + poisson.tau = p_dbl(default = 1, tags = "train", depends = quote(splitrule == "poisson")), + regularization.factor = p_uty(default = 1, tags = "train"), + regularization.usedepth = p_lgl(default = FALSE, tags = "train"), + replace = p_lgl(default = TRUE, tags = "train"), + respect.unordered.factors = p_fct(c("ignore", "order", "partition"), tags = "train"), + sample.fraction = p_dbl(0L, 1L, tags = "train"), + save.memory = p_lgl(default = FALSE, tags = "train"), + scale.permutation.importance = p_lgl(default = FALSE, tags = "train", depends = quote(importance == "permutation")), + se.method = p_fct(c("jack", "infjack", "simple", "law_of_total_variance"), default = "infjack", tags = "predict"), + seed = p_int(default = NULL, special_vals = list(NULL), tags = c("train", "predict")), + split.select.weights = p_uty(default = NULL, tags = "train"), + splitrule = p_fct(c("variance", "extratrees", "maxstat", "beta", "poisson"), default = "variance", tags = "train"), + verbose = p_lgl(default = TRUE, tags = c("train", "predict")), + write.forest = p_lgl(default = TRUE, tags = "train"), + catch_errors = p_lgl(tags = "required"), + impute_method = p_fct(c("mean", "random"), tags = "required") + ) + ps$values = list(catch_errors = TRUE, impute_method = "random") + + super$initialize(learner = list(), archive = archive, cols_x = cols_x, cols_y = col_y, param_set = ps) + }, + + #' @description + #' Predict mean response and standard error. + #' + #' @param xdt ([data.table::data.table()])\cr + #' New data. One row per observation. + #' + #' @return [data.table::data.table()] with the columns `mean` and `se`. + predict = function(xdt) { + assert_xdt(xdt) + newdata = as_numeric_matrix(fix_xdt_missing(xdt, cols_x = self$cols_x, archive = self$archive)) + pv = self$param_set$get_values(tags = "predict") + + p = invoke(predict, private$.model, + data = newdata, + type = self$predict_type, + quantiles = private$.quantiles, + .args = pv) + + list(mean = p$predictions, se = p$se) + } + ), + + active = list( + + #' @template field_print_id + print_id = function(rhs) { + if (missing(rhs)) { + "ranger::ranger" + } else { + stop("$print_id is read-only.") + } + }, + + #' @template field_n_learner_surrogate + n_learner = function() { + 1L + }, + + + #' @template field_packages_surrogate + packages = function(rhs) { + if (missing(rhs)) { + "DiceKriging" + } else { + stop("$packages is read-only.") + } + }, + + #' @template field_feature_types_surrogate + feature_types = function(rhs) { + if (missing(rhs)) { + c("logical", "integer", "numeric") + } else { + stop("$feature_types is read-only.") + } + }, + + #' @template field_properties_surrogate + properties = function(rhs) { + if (missing(rhs)) { + character(0L) + } else { + stop("$properties is read-only.") + } + }, + + #' @template field_predict_type_surrogate + predict_type = function(rhs) { + if (missing(rhs)) { + "se" + } else { + stop("$predict_type is read-only.") + } + } + ), + + private = list( + + .model = NULL, + + # Train learner with new data. + .update = function() { + pv = self$param_set$get_values(tags = "train") + pv = convert_ratio(pv, "mtry", "mtry.ratio", length(self$cols_x)) + design = as_numeric_matrix(self$archive$data[, c(self$cols_x, self$cols_y), with = FALSE]) + + model = invoke(ranger::ranger, + dependent.variable.name = self$cols_y, + data = design, + .args = pv + ) + + private$.model = if (isTRUE(self$param_set$values$se.method %in% c("simple", "law_of_total_variance"))) { + prediction_nodes = mlr3misc::invoke(predict, model, data = design, type = "terminalNodes", .args = pv[setdiff(names(pv), "se.method")], predict.all = TRUE) + storage.mode(prediction_nodes$predictions) = "integer" + mu_sigma = .Call("c_ranger_mu_sigma", model, prediction_nodes$predictions, self$archive$data[[self$cols_y]]) + list(model = model, mu_sigma = mu_sigma) + } else { + list(model = model) + } + + self$learner$model = private$.model + invisible(NULL) + }, + + # Train learner with new data. + # Operates on an asynchronous archive and performs imputation as needed. + .update_async = function() { + xydt = self$archive$rush$fetch_tasks_with_state(states = c("queued", "running", "finished"))[, c(self$cols_x, self$cols_y, "state"), with = FALSE] + + if (self$param_set$values$impute_method == "mean") { + mean_y = mean(xydt[[self$cols_y]], na.rm = TRUE) + xydt[c("queued", "running"), (self$cols_y) := mean_y, on = "state"] + } else if (self$param_set$values$impute_method == "random") { + min_y = min(xydt[[self$cols_y]], na.rm = TRUE) + max_y = max(xydt[[self$cols_y]], na.rm = TRUE) + xydt[c("queued", "running"), (self$cols_y) := runif(.N, min = min_y, max = max_y), on = "state"] + } + set(xydt, j = "state", value = NULL) + + pv = self$param_set$get_values(tags = "train") + pv = convert_ratio(pv, "mtry", "mtry.ratio", length(self$cols_x)) + design = as_numeric_matrix(xydt[, c(self$cols_x, self$cols_y), with = FALSE]) + + model = invoke(ranger::ranger, + dependent.variable.name = self$cols_y, + data = design, + .args = pv + ) + + private$.model = if (isTRUE(self$param_set$values$se.method %in% c("simple", "law_of_total_variance"))) { + prediction_nodes = mlr3misc::invoke(predict, model, data = design, type = "terminalNodes", .args = pv[setdiff(names(pv), "se.method")], predict.all = TRUE) + storage.mode(prediction_nodes$predictions) = "integer" + mu_sigma = .Call("c_ranger_mu_sigma", model, prediction_nodes$predictions, xydt[[self$cols_y]]) + list(model = model, mu_sigma = mu_sigma) + } else { + list(model = model) + } + + self$learner$model = private$.model + invisible(NULL) + }, + + .reset = function() { + self$learner = list() + private$.model = NULL + }, + + deep_clone = function(name, value) { + switch(name, + .param_set = value$clone(deep = TRUE), + .archive = value$clone(deep = TRUE), + value + ) + } + ) +) + +# https://github.com/mlr-org/mlr3learners/blob/51c15755438078fc99b11a9ca0147c7f2dbb96d8/R/helpers.R#L16C1-L16C18 +as_numeric_matrix = function(x) { + x = as.matrix(x) + if (is.logical(x)) { + storage.mode(x) = "double" + } + x +} + +convert_ratio = function(pv, target, ratio, n) { + switch(to_decimal(c(target, ratio) %in% names(pv)) + 1L, + # !mtry && !mtry.ratio + pv, + + # !mtry && mtry.ratio + { + pv[[target]] = max(ceiling(pv[[ratio]] * n), 1) + remove_named(pv, ratio) + }, + + + # mtry && !mtry.ratio + pv, + + # mtry && mtry.ratio + stopf("Hyperparameters '%s' and '%s' are mutually exclusive", target, ratio) + ) +} diff --git a/R/aaa.R b/R/aaa.R index ed02c255..8a83a3b6 100644 --- a/R/aaa.R +++ b/R/aaa.R @@ -1,2 +1,3 @@ optimizers = list() tuners = list() +learners = list() diff --git a/R/bayesopt_ego.R b/R/bayesopt_ego.R index 5dff5359..8a7b31a7 100644 --- a/R/bayesopt_ego.R +++ b/R/bayesopt_ego.R @@ -145,14 +145,22 @@ bayesopt_ego = function( if (isTRUE((instance$archive$n_evals - init_design_size + 1L) %% random_interleave_iter == 0)) { stop(set_class(list(message = "Random interleaving", call = NULL), classes = c("random_interleave", "mbo_error", "error", "condition"))) } - acq_function$surrogate$update() + + runtime_train_surrogate = system.time({ + acq_function$surrogate$update() + }) acq_function$update() - acq_optimizer$optimize() + runtime_acq_optimizer = system.time({ + res = acq_optimizer$optimize() + }) + res }, mbo_error = function(mbo_error_condition) { lg$info(paste0(class(mbo_error_condition), collapse = " / ")) lg$info("Proposing a randomly sampled point") generate_design_random(search_space, n = 1L)$data }) + set(xdt, j = "runtime_train_surrogate", value = runtime_train_surrogate["elapsed"]) + set(xdt, j = "runtime_acq_optimizer", value = runtime_acq_optimizer["elapsed"]) instance$eval_batch(xdt) if (instance$is_terminated) break diff --git a/R/zzz.R b/R/zzz.R index d561fff0..02983c42 100644 --- a/R/zzz.R +++ b/R/zzz.R @@ -19,6 +19,12 @@ register_bbotk = function() { iwalk(optimizers, function(obj, nm) x$add(nm, obj)) } # nocov end +register_mlr3 = function() { + # nocov start + x = utils::getFromNamespace("mlr_learners", ns = "mlr3") + iwalk(learners, function(obj, nm) x$add(nm, obj)) +} # nocov end + register_mlr3tuning = function() { # nocov start x = utils::getFromNamespace("mlr_tuners", ns = "mlr3tuning") @@ -29,6 +35,7 @@ register_mlr3tuning = function() { # nocov start register_namespace_callback(pkgname, "bbotk", register_bbotk) register_namespace_callback(pkgname, "mlr3tuning", register_mlr3tuning) + register_namespace_callback(pkgname, "mlr3", register_mlr3) #https://github.com/mlr-org/bbotk/blob/ae6cac60f71b3c44ce1bb29669f5d06cddeb95d4/R/zzz.R#L20 lg = lgr::get_logger("mlr3/bbotk") @@ -41,6 +48,7 @@ register_mlr3tuning = function() { .onUnload = function(libpaths) { # nolint # nocov start + walk(names(learners), function(id) mlr3::mlr_learners$remove(id)) walk(names(optimizers), function(id) bbotk::mlr_optimizers$remove(id)) walk(names(tuners), function(id) mlr3tuning::mlr_tuners$remove(id)) } # nocov end diff --git a/attic/AcqOptimizerCmaes.R b/attic/AcqOptimizerCmaes.R new file mode 100644 index 00000000..8c2b193e --- /dev/null +++ b/attic/AcqOptimizerCmaes.R @@ -0,0 +1,173 @@ +#' @title CMA-ES Acquisition Function Optimizer +#' +#' @description +#' IPOP CMA-ES runs for `n_iterations` iterations. +#' The population size is increased by `population_multiplier` after each iteration. +#' In the first iteration, the start point is the best point in the archive. +#' In the subsequent iterations, the start point is a random point in the search space. +#' +#' @export +AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", + inherit = AcqOptimizer, + public = list( + + #' @field state (`list()`)\cr + #' List of [cmaes::cma_es()] results. + state = NULL, + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + #' + #' @param acq_function (`NULL` | [AcqFunction]). + initialize = function(acq_function = NULL) { + self$acq_function = assert_r6(acq_function, "AcqFunction", null.ok = TRUE) + param_set = ps( + maxeval = p_int(lower = 1, init = 1000L, special_vals = list(-1)), + fnscale = p_dbl(default = 1), + #maxit = p_int(lower = 1L), + stopfitness = p_dbl(default = -Inf), + keep.best = p_lgl(default = TRUE), + sigma = p_uty(default = 0.5), + mu = p_int(lower = 1L), + lambda = p_int(lower = 1L), + weights = p_uty(), + damps = p_dbl(), + cs = p_dbl(), + ccum = p_dbl(), + ccov.1 = p_dbl(lower = 0), + ccov.mu = p_dbl(lower = 0), + diag.sigma = p_lgl(default = FALSE), + diag.eigen = p_lgl(default = FALSE), + diag.pop = p_lgl(default = FALSE), + diag.value = p_lgl(default = FALSE), + stop.tolx = p_dbl(), # undocumented stop criterion + restart_strategy = p_fct(levels = c("none", "ipop"), init = "none"), + n_restarts = p_int(lower = 0L, init = 0L), + population_multiplier = p_int(lower = 1, init = 2L) + # start_values = p_fct(default = "random", levels = c("random", "center", "custom")), + # start = p_uty(default = NULL, depends = start_values == "custom"), + # n_candidates = p_int(lower = 1, default = 1L), + # logging_level = p_fct(levels = c("fatal", "error", "warn", "info", "debug", "trace"), default = "warn"), + # warmstart = p_lgl(default = FALSE), + # warmstart_size = p_int(lower = 1L, special_vals = list("all")), + # skip_already_evaluated = p_lgl(default = TRUE), + # catch_errors = p_lgl(default = TRUE) + ) + # ps$values = list(n_candidates = 1, logging_level = "warn", warmstart = FALSE, skip_already_evaluated = TRUE, catch_errors = TRUE) + # ps$add_dep("warmstart_size", on = "warmstart", cond = CondEqual$new(TRUE)) + private$.param_set = param_set + }, + + #' @description + #' Optimize the acquisition function. + #' + #' @return [data.table::data.table()] with 1 row per candidate. + optimize = function() { + pv = self$param_set$values + min_iterations = if (pv$restart_strategy == "ipop") pv$n_restarts + 1L else 1L + pv$n_restarts = NULL + pv$restart_strategy = NULL + pv$vectorized = TRUE + + # set package defaults if not set by user + # restarts needs lambda and mu to be set + if (is.null(pv$lambda)) pv$lambda = 4 + floor(3 * log(length(par))) + if (is.null(pv$mu)) pv$mu = floor(pv$lambda / 2) + + wrapper = function(xmat, fun, constants, direction) { + xdt = as.data.table(t(xmat)) + res = mlr3misc::invoke(fun, xdt = xdt, .args = constants)[[1]] + res * direction + } + + fun = get_private(self$acq_function)$.fun + constants = self$acq_function$constants$values + direction = self$acq_function$codomain$direction + + y = Inf + n = 0L + i = 0L + maxeval_i = floor(pv$maxeval / min_iterations) + while (n < pv$maxeval) { + i = i + 1L + + par = if (i == 1L) { + set_names(as.numeric(self$acq_function$archive$best()[, self$acq_function$domain$ids(), with = FALSE]), self$acq_function$domain$ids()) + } else { + # random restart + set_names(as.numeric(generate_design_random(self$acq_function$domain, n = 1)$data), self$acq_function$domain$ids()) + } + + res = invoke(cmaes::cma_es, + par = par, + fn = wrapper, + lower = self$acq_function$domain$lower, + upper = self$acq_function$domain$upper, + control = pv, # ceiling(min(maxeval_i / pv$lambda, (pv$maxeval - n) / pv$lambda))) + fun = fun, + constants = constants, + direction = direction) + + browser() + + # set best + if (res$value < y) { + y = res$value + x = set_names(res$par, self$acq_function$domain$ids()) + } + + pv$mu = pv$mu * pv$population_multiplier + pv$lambda = pv$lambda * pv$population_multiplier + + n = n + res$counts[1] + + self$state = c(self$state, set_names(list(res), paste0("iteration_", i))) + } + + as.data.table(as.list(set_names(c(x, y * direction), c(self$acq_function$domain$ids(), self$acq_function$codomain$ids())))) + }, + + #' @description + #' Reset the acquisition function optimizer. + #' + #' Currently not used. + reset = function() { + + } + ), + + active = list( + #' @template field_print_id + print_id = function(rhs) { + if (missing(rhs)) { + paste0("(", class(self$optimizer)[1L], " | ", class(self$terminator)[1L], ")") + } else { + stop("$print_id is read-only.") + } + }, + + #' @field param_set ([paradox::ParamSet])\cr + #' Set of hyperparameters. + param_set = function(rhs) { + if (!missing(rhs) && !identical(rhs, private$.param_set)) { + stop("$param_set is read-only.") + } + private$.param_set + } + ), + + private = list( + .param_set = NULL, + + deep_clone = function(name, value) { + switch(name, + optimizer = value$clone(deep = TRUE), + terminator = value$clone(deep = TRUE), + acq_function = if (!is.null(value)) value$clone(deep = TRUE) else NULL, + .param_set = value$clone(deep = TRUE), + value + ) + } + ) +) + diff --git a/attic/LearnerRegrRangerCustom.R b/attic/LearnerRegrRangerCustom.R new file mode 100644 index 00000000..4fc8e4fd --- /dev/null +++ b/attic/LearnerRegrRangerCustom.R @@ -0,0 +1,212 @@ +#' @title Custom Ranger Regression Learner +#' +#' @name mlr_learners_regr.ranger_custom +#' +#' @description +#' Custom random regression forest. +#' Calls [ranger::ranger()] from package \CRANpkg{ranger}. +#' This custom random regression forest tries to mimic the behaviour of a Gaussian process and therefore may be more suitable for Bayesian optimization compared to a normal random regression forest. +#' +#' The `"extratrees"` splitting rule is used with `num.random.splits = 1`. +#' This results in a rather smooth prediction function. +#' No bootstrap or subsampling is used but rather all training data (`replace = FALSE` and `sample.fraction = 1`). +#' This may seem counter-intuitive at first but ensures that training data is almost perfectly interpolated (given `min.node.size = 1`). +#' Per default, `1000` trees are used. +#' # FIXME: +#' Classical standard error estimation is no longer possible, but the `"extratrees"` splitting rule allows for estimating the variance based on the variance of the trees (simply taking the variance over the predictions of the trees). +#' This results in low variance close to training data points. +#' If `se.simple.spatial = TRUE` this point-wise variance is upwardly biased by adding the pointwise variance again scaled by the minimum Gower distance of the point to the training data (recall that a Gower distance of 0 means an identical point is present in the training data, whereas a distance of 1 means total dissimilarity to the training data). +#' This ensures that the variance is larger in regions dissimilar to the training data. +#' Summarizing, the standard error for a point is calculated as follows: +#' \deqn{\hat{\mathrm{SE}}(x_{i}) = \sqrt{(\hat{\mathrm{VAR}}(x_{i}) + \epsilon) + (\hat{\mathrm{VAR}}(x_{i}) + \epsilon) * \mathrm{GD}_{i})}} +#' Here, \eqn{\epsilon} is given by `se.simple.spatial.nugget` which can be set larger than 0 in the case of noisy function observations. +#' +#' @templateVar id regr.ranger_custom +#' @template learner +#' +#' @export +#' @template seealso_learner +#' @examples +#' library(data.table) +#' library(mlr3) +#' x = seq(-5, 5, length.out = 1001) +#' dat = data.table(x = c(-5, -2.5, -1, 0, 1, 2.5, 5), y = c(0, -0.1, 0.3, -1, 0.3, -0.1, 0)) +#' task = TaskRegr$new("example", backend = dat, target = "y") +#' learner = lrn("regr.ranger_custom") +#' learner$predict_type = "se" +#' learner$train(task) +#' predictions = learner$predict_newdata(data.table(x = x)) +#' if (requireNamespace("ggplot2")) { +#' library(ggplot2) +#' ggplot(aes(x = x, y = response), data = cbind(x, as.data.table(predictions))) + +#' geom_line() + +#' geom_ribbon(aes(min = response - se, max = response + se), alpha = 0.1) + +#' geom_point(aes(x = x, y = y), data = task$data()) + +#' labs(x = "x", y = "Prediction") + +#' theme_minimal() +#' } +LearnerRegrRangerCustom = R6Class("LearnerRegrRangerCustom", + inherit = mlr3::LearnerRegr, + + public = list( + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + initialize = function() { + ps = ps( + always.split.variables = p_uty(tags = "train"), + holdout = p_lgl(default = FALSE, tags = "train"), + importance = p_fct(levels = c("none", "impurity", "impurity_corrected", "permutation"), tags = "train"), + keep.inbag = p_lgl(default = FALSE, tags = "train"), + local.importance = p_lgl(default = FALSE, tags = "train"), + max.depth = p_int(default = NULL, lower = 0L, special_vals = list(NULL), tags = "train"), + mtry = p_int(lower = 1L, special_vals = list(NULL), tags = "train"), + mtry.ratio = p_dbl(lower = 0, upper = 1, tags = "train"), + num.threads = p_int(lower = 1L, default = 1L, tags = c("train", "predict", "threads")), # changed + num.trees = p_int(lower = 1L, default = 1000L, tags = c("train", "predict", "hotstart")), # changed + oob.error = p_lgl(default = FALSE, tags = "train"), # changed + regularization.factor = p_uty(default = 1, tags = "train"), + regularization.usedepth = p_lgl(default = FALSE, tags = "train"), + respect.unordered.factors = p_fct(levels = c("ignore", "order", "partition"), default = "ignore", tags = "train"), + save.memory = p_lgl(default = FALSE, tags = "train"), + scale.permutation.importance = p_lgl(default = FALSE, tags = "train"), + se.simple.spatial = p_lgl(default = TRUE, tags = "predict"), + se.simple.spatial.nugget = p_dbl(lower = 0, default = 0, tags = "predict"), + seed = p_int(default = NULL, special_vals = list(NULL), tags = c("train", "predict")), + split.select.weights = p_uty(default = NULL, tags = "train"), + verbose = p_lgl(default = TRUE, tags = c("train", "predict")), + write.forest = p_lgl(default = TRUE, tags = "train") + ) + ps$values = list(num.trees = 1000L, oob.error = FALSE, num.threads = 1L) + + # deps + ps$add_dep("se.simple.spatial.nugget", "se.simple.spatial", CondEqual$new(TRUE)) + + super$initialize( + id = "regr.ranger_custom", + param_set = ps, + predict_types = c("response", "se"), + feature_types = c("logical", "integer", "numeric", "character", "factor", "ordered"), + properties = c("weights", "importance", "oob_error", "hotstart_backward"), + packages = c("mlr3mbo", "ranger"), + man = "mlr3learners::mlr_learners_regr.ranger_custom" + ) + }, + + #' @description + #' The importance scores are extracted from the model slot `variable.importance`. + #' Parameter `importance.mode` must be set to `"impurity"`, `"impurity_corrected"`, or + #' `"permutation"` + #' + #' @return Named `numeric()`. + importance = function() { + if (is.null(self$model)) { + stopf("No model stored") + } + if (self$model$importance.mode == "none") { + stopf("No importance stored") + } + + sort(self$model$variable.importance, decreasing = TRUE) + }, + + #' @description + #' The out-of-bag error, extracted from model slot `prediction.error`. + #' + #' @return `numeric(1)`. + oob_error = function() { + if (is.null(self$model)) { + stopf("No model stored") + } + self$model$prediction.error + } + ), + + private = list( + .train = function(task) { + pv = self$param_set$get_values(tags = "train") + pv = convert_ratio_ranger(pv, "mtry", "mtry.ratio", length(task$feature_names)) + pv = insert_named(list(min.node.size = 1L, replace = FALSE, sample.fraction = 1L, splitrule = "extratrees", num.random.splits = 1L), pv) + + private$.train_task = task$clone(deep = TRUE) # required for se.method == "simple" and se.simple.spatial == TRUE + + invoke(ranger::ranger, dependent.variable.name = task$target_names, data = task$data(), case.weights = task$weights$weight, .args = pv) + }, + + .predict = function(task) { + pv = self$param_set$get_values(tags = "predict") + pv = insert_named(list(se.simple.spatial = TRUE, se.simple.spatial.nugget = 0), pv) + newdata = ordered_features(task, self) + traindata = ordered_features(private$.train_task, self) + data = rbind(traindata, newdata) + n_traindata = nrow(traindata) + n_newdata = nrow(newdata) + traindata_ids = seq_len(n_traindata) + newdata_ids = (n_traindata + 1L):nrow(data) + + prediction = invoke(predict, self$model, data = data, type = "response", predict.all = TRUE, .args = pv) + response = apply(prediction$predictions, MARGIN = 1L, FUN = mean) + + variance = numeric(n_newdata) + for (i in newdata_ids) { + variance[i - n_traindata] = var(prediction$predictions[i, ]) + var(response[c(traindata_ids, i)]) + } + if (pv$se.simple.spatial) { + gw_dists = get_gower_dist(fct_to_char(newdata), fct_to_char(traindata)) # 0 if identical, 1 if maximally dissimilar + min_gw_dists = apply(gw_dists, MARGIN = 1L, FUN = min) # get the minium for each new point to the points used for training + variance = (variance * pv$se.simple.spatial.nugget) + (variance * min_gw_dists) + } + se = sqrt(variance) + list(response = response[newdata_ids], se = if (self$predict_type == "se") se else NULL) + }, + + .hotstart = function(task) { + model = self$model + model$num.trees = self$param_set$values$num.trees + model + }, + + .train_task = NULL + ) +) + +#' @export +default_values.LearnerRegrRangerCustom = function(x, search_space, task, ...) { # nolint + special_defaults = list( + mtry = floor(sqrt(length(task$feature_names))), + mtry.ratio = floor(sqrt(length(task$feature_names))) / length(task$feature_names), + num.trees = 1000L, + min.node.size = 1L, + replace = FALSE, + sample.fraction = 1, + splitrule = "extratrees", + num.random.splits = 1L + ) + defaults = insert_named(default_values(x$param_set), special_defaults) + defaults[search_space$ids()] +} + +convert_ratio_ranger = function(pv, target, ratio, n) { + switch(to_decimal(c(target, ratio) %in% names(pv)) + 1L, + # !mtry && !mtry.ratio + pv, + + # !mtry && mtry.ratio + { + pv[[target]] = max(ceiling(pv[[ratio]] * n), 1) + remove_named(pv, ratio) + }, + + # mtry && !mtry.ratio + pv, + + # mtry && mtry.ratio + stopf("Hyperparameters '%s' and '%s' are mutually exclusive", target, ratio) + ) +} + +ordered_features = function(task, learner) { + cols = names(learner$state$data_prototype) + task$data(cols = intersect(cols, task$feature_names)) +} + diff --git a/attic/so_config/AcqFunctionLogEI.R b/attic/so_config/AcqFunctionLogEI.R new file mode 100644 index 00000000..3f2d2035 --- /dev/null +++ b/attic/so_config/AcqFunctionLogEI.R @@ -0,0 +1,64 @@ +#' @title Acquisition Function Log Expected Improvement +#' +#' @include AcqFunction.R +#' @name mlr_acqfunctions_log_ei +#' +#' @templateVar id log_ei +#' @template section_dictionary_acqfunctions +#' +#' @description +#' Expected Improvement assuming that the target variable is to be minimized and has been modeled on log scale. +#' +#' @family Acquisition Function +#' @export +#' @examples +AcqFunctionLogEI = R6Class("AcqFunctionLogEI", + inherit = AcqFunction, + + public = list( + + #' @field y_best (`numeric(1)`)\cr + #' Best objective function value observed so far. + #' On log scale. + y_best = NULL, + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + #' + #' @param surrogate (`NULL` | [SurrogateLearner]). + initialize = function(surrogate = NULL) { + assert_r6(surrogate, "SurrogateLearner", null.ok = TRUE) + super$initialize("acq_log_ei", surrogate = surrogate, direction = "maximize", label = "Log Expected Improvement", man = "mlr3mbo::mlr_acqfunctions_log_ei") + }, + + #' @description + #' Updates acquisition function and sets `y_best`. + update = function() { + if (self$surrogate_max_to_min != 1L) { + stop("Log EI assumes minimization of the log transformed target value.") + } + self$y_best = min(self$surrogate_max_to_min * self$archive$data[[self$surrogate$y_cols]]) + } + ), + + private = list( + .fun = function(xdt) { + if (is.null(self$y_best)) { + stop("$y_best is not set. Missed to call $update()?") + } + if (self$surrogate_max_to_min != 1L) { + stop("Log EI assumes minimization of the log transformed target value.") + } + p = self$surrogate$predict(xdt) + mu = p$mean + se = p$se + d_norm = (self$y_best - mu) / se + log_ei = (exp(self$y_best) * pnorm(d_norm)) - (exp((0.5 * se ^ 2) + mu) * pnorm(d_norm - se)) + log_ei = ifelse(se < 1e-20, 0, log_ei) + data.table(acq_log_ei = log_ei) + } + ) +) + +mlr_acqfunctions$add("log_ei", AcqFunctionLogEI) + diff --git a/attic/so_config/LearnerRegrGowerNNEnsemble.R b/attic/so_config/LearnerRegrGowerNNEnsemble.R new file mode 100644 index 00000000..f229cb73 --- /dev/null +++ b/attic/so_config/LearnerRegrGowerNNEnsemble.R @@ -0,0 +1,119 @@ +#' @title Gower Distance Nearest Neighbor Regression Ensemble +#' +#' @name mlr_learners_regr.gower_nn_ensemble +#' +#' @description +#' FIXME: +#' +#' @templateVar id regr.gower_nn_ensemble +#' @template learner +#' +#' @export +#' @template seealso_learner +#' @template example +LearnerRegrGowerNNEnsemble = R6Class("LearnerRegrGowerNNEnsemble", + inherit = LearnerRegr, + + public = list( + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + initialize = function() { + ps = ps( + ks = p_uty(tags = "train"), + sample.fraction = p_dbl(lower = 0, upper = 1, tags = "train"), + replace = p_lgl(tags = "train"), + mtry.ratio = p_dbl(lower = 0, upper = 1, tags = "train"), + nthread = p_int(lower = 1, tags = "predict") + ) + + ps$values$ks = c(1L, 3L, 5L, 7L, 10L) + ps$values$sample.fraction = 1 + ps$values$replace = TRUE + ps$values$nthread = 1L + + super$initialize( + id = "regr.gower_nn_ensemble", + param_set = ps, + predict_types = c("response", "se"), + feature_types = c("logical", "integer", "numeric", "character", "factor", "ordered"), + properties = character(0), + packages = c("mlr3mbo", "gower"), + man = "mlr3mbo::mlr_learners_regr.gower_nn_ensemble" + ) + } + ), + + private = list( + .train = function(task) { + pv = self$param_set$get_values(tags = "train") + if(is.null(pv$mtry.ratio)) { + pv = insert_named(pv, list(mtry.ratio = floor(sqrt(length(task$feature_names))) / length(task$feature_names))) + } + if (any(pv$ks >= task$nrow)) { + index = which(pv$ks >= task$nrow) + stopf("Parameter k = %i must be smaller than the number of observations n = %i", pv$k[index], task$nrow) + } + + list( + target_name = task$target_names, + feature_names = task$feature_names, + data = task$data(), + pv = pv + ) + + }, + + .predict = function(task) { + model = self$model + newdata = mlr3learners:::ordered_features(task, self) + pv = insert_named(model$pv, self$param_set$get_values(tags = "predict")) + + data = self$model$data + X = data[, self$model$feature_names, with = FALSE] + y = data[, self$model$target_name, with = FALSE] + + stopifnot(all(colnames(X) == colnames(newdata))) + + mtry = as.integer(round(pv$mtry.ratio * ncol(X))) + + # FIXME: what about ordered here? + for (feature in colnames(X)) { + if ("factor" %in% class(X[[feature]])) { + factor_levels = union(levels(X[[feature]]), levels(newdata[[feature]])) + levels(X[[feature]]) = factor_levels + levels(newdata[[feature]]) = factor_levels + } + } + + n = as.integer(max(ceiling(pv$sample.fraction * nrow(X)), nrow(X))) + + results = map_dtr(pv$ks, function(k) { + ids = sample(seq_len(nrow(X)), size = n, replace = pv$replace) + features = sample(colnames(X), size = mtry, replace = FALSE) + top_n = invoke(gower::gower_topn, x = newdata[, features, with = FALSE], y = X[ids, features, with = FALSE], n = k, nthread = pv$nthread)$index + tmp = map_dtc(seq_len(k), function(i) { + setNames(y[ids, ][top_n[i, ], ], nm = paste0(self$model$target_name, "_", i)) + }) + mu = rowMeans(tmp) + sigma_2 = apply(tmp, MARGIN = 1L, FUN = var) + sigma_2[is.na(sigma_2)] = 0 + result = data.table(mu = mu, sigma_2 = sigma_2) + result[, k := k] + result[, ID := seq_len(.N)] + result + }) + + response = results[, .(response = mean(mu)), by = .(ID)] + results[, first_term := 0.001 + sigma_2 + mu ^ 2] + variance = results[, .(variance = mean(first_term)), by = .(ID)] + setorderv(response, col = "ID") + setorderv(variance, col = "ID") + stopifnot(all(response[["ID"]] == variance[["ID"]])) + variance$variance = variance$variance - response$response ^ 2 + + list(response = response$response, se = sqrt(variance$variance)) + } + ) +) + diff --git a/attic/so_config/LearnerRegrRangerCustom.R b/attic/so_config/LearnerRegrRangerCustom.R new file mode 100644 index 00000000..1a394684 --- /dev/null +++ b/attic/so_config/LearnerRegrRangerCustom.R @@ -0,0 +1,159 @@ +#' @title Ranger Regression Learner Custom +#' +#' @name mlr_learners_regr.ranger_custom +#' +#' @description +#' Random regression forest. +#' Calls [ranger::ranger()] from package \CRANpkg{ranger}. +#' +#' @inheritSection mlr_learners_classif.ranger Custom mlr3 parameters +#' @inheritSection mlr_learners_classif.ranger Initial parameter values +#' +#' @templateVar id regr.ranger_custom +#' @template learner +#' +#' @references +#' `r format_bib("wright_2017", "breiman_2001")` +#' +#' @export +#' @template seealso_learner +#' @template example +LearnerRegrRangerCustom = R6Class("LearnerRegrRangerCustom", + inherit = LearnerRegr, + + public = list( + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + initialize = function() { + ps = ps( + alpha = p_dbl(default = 0.5, tags = "train"), + always.split.variables = p_uty(tags = "train"), + holdout = p_lgl(default = FALSE, tags = "train"), + importance = p_fct(c("none", "impurity", "impurity_corrected", "permutation"), tags = "train"), + keep.inbag = p_lgl(default = FALSE, tags = "train"), + max.depth = p_int(default = NULL, lower = 0L, special_vals = list(NULL), tags = "train"), + min.node.size = p_int(1L, default = 5L, special_vals = list(NULL), tags = "train"), + min.prop = p_dbl(default = 0.1, tags = "train"), + minprop = p_dbl(default = 0.1, tags = "train"), + mtry = p_int(lower = 1L, special_vals = list(NULL), tags = "train"), + mtry.ratio = p_dbl(lower = 0, upper = 1, tags = "train"), + num.random.splits = p_int(1L, default = 1L, tags = "train"), + num.threads = p_int(1L, default = 1L, tags = c("train", "predict", "threads")), + num.trees = p_int(1L, default = 500L, tags = c("train", "predict", "hotstart")), + oob.error = p_lgl(default = TRUE, tags = "train"), + quantreg = p_lgl(default = FALSE, tags = "train"), + regularization.factor = p_uty(default = 1, tags = "train"), + regularization.usedepth = p_lgl(default = FALSE, tags = "train"), + replace = p_lgl(default = TRUE, tags = "train"), + respect.unordered.factors = p_fct(c("ignore", "order", "partition"), default = "ignore", tags = "train"), + sample.fraction = p_dbl(0L, 1L, tags = "train"), + save.memory = p_lgl(default = FALSE, tags = "train"), + scale.permutation.importance = p_lgl(default = FALSE, tags = "train"), + se.method = p_fct(c("jack", "infjack", "simple"), default = "infjack", tags = "predict"), # FIXME: only works if predict_type == "se". How to set dependency? + seed = p_int(default = NULL, special_vals = list(NULL), tags = c("train", "predict")), + split.select.weights = p_uty(default = NULL, tags = "train"), + splitrule = p_fct(c("variance", "extratrees", "maxstat"), default = "variance", tags = "train"), + verbose = p_lgl(default = TRUE, tags = c("train", "predict")), + write.forest = p_lgl(default = TRUE, tags = "train") + ) + + ps$values = list(num.threads = 1L) + + # deps + ps$add_dep("num.random.splits", "splitrule", CondEqual$new("extratrees")) + ps$add_dep("alpha", "splitrule", CondEqual$new("maxstat")) + ps$add_dep("minprop", "splitrule", CondEqual$new("maxstat")) + ps$add_dep("scale.permutation.importance", "importance", CondEqual$new("permutation")) + + + super$initialize( + id = "regr.ranger_custom", + param_set = ps, + predict_types = c("response", "se"), + feature_types = c("logical", "integer", "numeric", "character", "factor", "ordered"), + properties = c("weights", "importance", "oob_error", "hotstart_backward"), + packages = c("mlr3learners", "ranger"), + man = "mlr3learners::mlr_learners_regr.ranger_custom" + ) + }, + + #' @description + #' The importance scores are extracted from the model slot `variable.importance`. + #' Parameter `importance.mode` must be set to `"impurity"`, `"impurity_corrected"`, or + #' `"permutation"` + #' + #' @return Named `numeric()`. + importance = function() { + if (is.null(self$model)) { + stopf("No model stored") + } + if (self$model$importance.mode == "none") { + stopf("No importance stored") + } + + sort(self$model$variable.importance, decreasing = TRUE) + }, + + #' @description + #' The out-of-bag error, extracted from model slot `prediction.error`. + #' + #' @return `numeric(1)`. + oob_error = function() { + if (is.null(self$model)) { + stopf("No model stored") + } + self$model$prediction.error + } + ), + + private = list( + .train = function(task) { + pv = self$param_set$get_values(tags = "train") + pv = mlr3learners:::convert_ratio(pv, "mtry", "mtry.ratio", length(task$feature_names)) + + if (self$predict_type == "se") { + pv$keep.inbag = TRUE # nolint + } + + mlr3misc::invoke(ranger::ranger, + dependent.variable.name = task$target_names, + data = task$data(), + case.weights = task$weights$weight, + .args = pv + ) + }, + + .predict = function(task) { + pv = self$param_set$get_values(tags = "predict") + newdata = mlr3learners:::ordered_features(task, self) + + if (isTRUE(pv$se.method == "simple")) { + prediction = mlr3misc::invoke(predict, self$model, data = newdata, type = "response", .args = pv[setdiff(names(pv), "se.method")], predict.all = TRUE) + response = rowMeans(prediction$predictions) + variance = rowMeans(0.01 + prediction$predictions ^ 2) - (response ^ 2) # law of total variance assuming sigma_b(x) is 0 due to min.node.size = 1 and always splitting; then set 0.01 as lower bound for sigma_b(x) + list(response = response, se = sqrt(variance)) + } else { + prediction = mlr3misc::invoke(predict, self$model, data = newdata, type = self$predict_type, .args = pv) + list(response = prediction$predictions, se = prediction$se) + } + }, + + .hotstart = function(task) { + model = self$model + model$num.trees = self$param_set$values$num.trees + model + } + ) +) + +#' @export +default_values.LearnerRegrRangerCustom = function(x, search_space, task, ...) { # nolint + special_defaults = list( + mtry = floor(sqrt(length(task$feature_names))), + mtry.ratio = floor(sqrt(length(task$feature_names))) / length(task$feature_names), + sample.fraction = 1 + ) + defaults = insert_named(default_values(x$param_set), special_defaults) + defaults[search_space$ids()] +} diff --git a/attic/so_config/OptimizerChain.R b/attic/so_config/OptimizerChain.R new file mode 100644 index 00000000..4f04a373 --- /dev/null +++ b/attic/so_config/OptimizerChain.R @@ -0,0 +1,83 @@ +OptimizerChain = R6Class("OptimizerChain", inherit = bbotk::Optimizer, + public = list( + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + #' + #' @param optimizers (list of [Optimizer]s). + #' @param terminators (list of [Terminator]s | NULL). + initialize = function(optimizers, terminators = rep(list(NULL), length(optimizers))) { + assert_list(optimizers, types = "Optimizer", any.missing = FALSE) + assert_list(terminators, types = c("Terminator", "NULL"), len = length(optimizers)) + + param_sets = vector(mode = "list", length = length(optimizers)) + ids_taken = character(0L) + # for each optimizer check whether the id of the param_set + # (decuded from the optimizer class) is already taken; + # if necessary postfix the id + for (i_opt in seq_along(optimizers)) { + opt = optimizers[[i_opt]] + ps = opt$param_set$clone(deep = TRUE) + ps$set_id = class(opt)[[1L]] + try_postfix = 0L + while (ps$set_id %in% ids_taken) { + try_postfix = try_postfix + 1L + ps$set_id = paste0(class(opt)[[1L]], "_", try_postfix) + } + ids_taken[[i_opt]] = ps$set_id + param_sets[[i_opt]] = ps + } + private$.ids = map_chr(param_sets, "set_id") + super$initialize( + param_set = ParamSetCollection$new(param_sets), + param_classes = Reduce(intersect, mlr3misc::map(optimizers, "param_classes")), + properties = Reduce(intersect, mlr3misc::map(optimizers, "properties")), + packages = unique(unlist(mlr3misc::map(optimizers, "packages"))) + ) + private$.optimizers = optimizers + private$.terminators = terminators + } + ), + + private = list( + .optimizers = NULL, + .terminators = NULL, + .ids = NULL, + + .optimize = function(inst) { + terminator = inst$terminator + on.exit({inst$terminator = terminator}) + inner_inst = inst$clone(deep = TRUE) + + for (i_opt in seq_along(private$.optimizers)) { + inner_term = private$.terminators[[i_opt]] + if (!is.null(inner_term)) { + inner_inst$terminator = TerminatorCombo$new(list(inner_term, terminator)) + } else { + inner_inst$terminator = terminator + } + opt = private$.optimizers[[i_opt]] + opt$param_set$values = self$param_set$.__enclos_env__$private$.sets[[i_opt]]$values + opt$optimize(inner_inst) + inner_inst$archive$data$batch_nr = max(inst$archive$data$batch_nr, 0L) + + inner_inst$archive$data$batch_nr + inner_inst$archive$data$optimizer = private$.ids[i_opt] + inst$archive$data = rbind(inst$archive$data, inner_inst$archive$data, fill = TRUE) + inner_inst$archive$data = data.table() + if (terminator$is_terminated(inst$archive)) { + break + } + } + }, + + deep_clone = function(name, value) { + switch( + name, + .optimizers = mlr3misc::map(value, .f = function(x) x$clone(deep = TRUE)), + .terminators = mlr3misc::map(value, .f = function(x) if (!is.null(x)) x$clone(deep = TRUE)), + value + ) + } + ) +) + diff --git a/attic/so_config/OptimizerCoordinateDescent.R b/attic/so_config/OptimizerCoordinateDescent.R new file mode 100644 index 00000000..5670d1b9 --- /dev/null +++ b/attic/so_config/OptimizerCoordinateDescent.R @@ -0,0 +1,121 @@ +OptimizerCoordinateDescent = R6Class("OptimizerCoordinateDescent", inherit = bbotk::Optimizer, + public = list( + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + initialize = function() { + param_set = ps( + n_coordinate_tryouts = p_int(lower = 1L, default = 10L, tags = "required"), + max_gen = p_int(lower = 1L, default = 10L, tags = "required"), + rds_name = p_uty(default = "cd_instance.rds", tags = "required") + ) + param_set$values = list(n_coordinate_tryouts = 10L, max_gen = 10L, rds_name = "cd_instance.rds") + + super$initialize( + id = "coordinate_descent", + param_set = param_set, + param_classes = c("ParamLgl", "ParamInt", "ParamDbl", "ParamFct"), + properties = c("single-crit", "dependencies"), + label = "Coordinate Descent", + man = "" + ) + } + ), + + private = list( + .optimize = function(inst) { + n_coordinate_tryouts = self$param_set$values$n_coordinate_tryouts + if (inst$archive$n_evals == 0L) { + xdt = generate_design_random(inst$search_space, n = 1L)$data + inst$eval_batch(xdt) + } + # check if .gen is already present, if yes continue from there + if (inst$archive$n_evals > 0L & ".gen" %in% colnames(inst$archive$data)) { + gen = max(inst$archive$data[[".gen"]], na.rm = TRUE) + } else { + gen = 0L + set(inst$archive$data, j = ".gen", value = gen) # 0 for first batch + } + + y_col = inst$archive$cols_y + y_col_orig = paste0(y_col, "_orig") + + inst$archive$data[.gen == gen, (y_col_orig) := get(y_col)] + + incumbent = inst$archive$best()[, inst$archive$cols_x, with = FALSE] + + repeat { + gen = gen + 1L + for (param_id in shuffle(inst$search_space$ids())) { + if (!is.na(incumbent[[param_id]])) { + xdt = get_xdt_coordinate(copy(incumbent), param = inst$search_space$params[[param_id]], n_coordinate_tryouts = n_coordinate_tryouts) + # previously inactive parameters can now be active and need a value + if (inst$search_space$has_deps & param_id %in% inst$search_space$deps$on) { + deps = inst$search_space$deps[on == param_id, ] + for (i in seq_len(nrow(deps))) { + to_replace = which(map_lgl(xdt[[param_id]], function(x) deps$cond[[i]]$test(x))) + if (test_r6(inst$search_space$params[[deps$id[[i]]]]$default, classes = "NoDefault")) { + set(xdt, i = to_replace, j = deps$id[[i]], value = sample_random(inst$search_space$params[[deps$id[[i]]]], n = length(to_replace))) # random + } else { + set(xdt, i = to_replace, j = deps$id[[i]], value = inst$search_space$params[[deps$id[[i]]]]$default) # default + } + } + } + xdt = Design$new(inst$search_space, data = xdt, remove_dupl = TRUE)$data # fixes potentially broken dependencies + set(xdt, j = ".param", value = param_id) + xdt = rbind(xdt, incumbent, fill = TRUE) # also reevaluate the incumbent to see how noisy evaluations are + set(xdt, j = ".gen", value = gen) + set(xdt, i = nrow(xdt), j = ".param", value = "incumbent") + set(xdt, j = "incumbent", value = list(inst$archive$best()[, c(inst$archive$cols_x, inst$archive$cols_y), with = FALSE])) + inst$eval_batch(xdt) + incumbent = inst$archive$best()[, inst$archive$cols_x, with = FALSE] + saveRDS(inst, file = self$param_set$values$rds_name) + } + } + + # after each gen, update target evaluations of incumbents that have been evaluated multiple times by their mean + tmp = copy(inst$archive$data) + hashes = as.numeric(as.factor(map_chr(seq_len(nrow(tmp)), function(i) paste0(tmp[i, inst$archive$cols_x, with = FALSE], collapse = "_")))) + tmp[, hash := hashes] + tmp[.gen == gen, (y_col_orig) := get(y_col)] + tmp[, n_incumbent_repls := .N, by = .(hash)] + tmp[, (y_col) := mean(get(y_col_orig)), by = .(hash)] + inst$archive$data = tmp + incumbent = inst$archive$best()[, inst$archive$cols_x, with = FALSE] + + if (gen >= self$param_set$values$max_gen) { + break + } + } + } + ) +) + +get_xdt_coordinate = function(incumbent, param, n_coordinate_tryouts) { + if (param$class == "ParamDbl") { + x = runif(n = n_coordinate_tryouts, min = param$lower, max = param$upper) + } else if (param$class == "ParamInt") { + levels = setdiff(seq(param$lower, param$upper, by = 1L), incumbent[[param$id]]) + n_coordinate_tryouts = min(n_coordinate_tryouts, length(levels)) + x = sample(levels, size = n_coordinate_tryouts, replace = FALSE) + } else { + n_coordinate_tryouts = min(n_coordinate_tryouts, param$nlevels - 1L) + x = sample(x = setdiff(param$levels, incumbent[[param$id]]), size = n_coordinate_tryouts, replace = FALSE) + } + xdt = incumbent[rep(1L, n_coordinate_tryouts), ] + set(xdt, j = param$id, value = x) + xdt +} + +sample_random = function(param, n) { + if (param$class == "ParamDbl") { + x = runif(n = n, min = param$lower, max = param$upper) + } else if (param$class == "ParamInt") { + levels = seq(param$lower, param$upper, by = 1L) + x = sample(levels, size = n, replace = TRUE) + } else { + x = sample(param$levels, size = n, replace = TRUE) + } + x +} + diff --git a/attic/so_config/analyze_cd.R b/attic/so_config/analyze_cd.R new file mode 100644 index 00000000..8f4ac2d9 --- /dev/null +++ b/attic/so_config/analyze_cd.R @@ -0,0 +1,42 @@ +library(data.table) +library(ggplot2) +library(pammtools) +library(mlr3misc) + +instance = readRDS("cd_instance.rds") +archive = instance$archive +data = instance$archive$data +cols = c(archive$cols_x, archive$cols_y, ".param") + +for (i in seq_along(idx)) { + print(data[idx[[i]], cols, with = FALSE]) + cat("\n") + cat("\n") +} + +png("hist1_k.png") +hist(data[1L, ]$raw_k[[1L]], main = "k, geom mean = 4.265765", xlab = "") +dev.off() + +tmp = data[1L, ]$raw_mean_best[[1L]] +tmp[tmp > 1] = tmp[tmp > 1] / 100 +png("hist1_target.png") +hist(tmp, main = "Mean target, mean = 0.9433329", xlab = "") +dev.off() + +tmp = data[4L, ]$raw_mean_best[[1L]] +tmp[tmp > 1] = tmp[tmp > 1] / 100 + +png("hist2_k.png") +hist(data[5L, ]$raw_k[[1L]], main = "k, geom mean = 4.894954", xlab = "") +dev.off() + +tmp = data[5L, ]$raw_mean_best[[1L]] +tmp[tmp > 1] = tmp[tmp > 1] / 100 +png("hist2_target.png") +hist(tmp, main = "Mean target, mean = 0.9452976", xlab = "") +dev.off() + + + + diff --git a/attic/so_config/analyze_yahpo.R b/attic/so_config/analyze_yahpo.R new file mode 100644 index 00000000..5b8db566 --- /dev/null +++ b/attic/so_config/analyze_yahpo.R @@ -0,0 +1,90 @@ +library(data.table) +library(ggplot2) +library(pammtools) +library(mlr3misc) + +dat = rbind(readRDS("results_yahpo.rds"), readRDS("results_yahpo_mbox.rds")) +dat[, cumbudget := cumsum(budget), by = .(method, scenario, instance, repl)] +dat[, cumbudget_scaled := cumbudget / max(cumbudget), by = .(method, scenario, instance, repl)] +dat[, normalized_regret := (target - min(target)) / (max(target) - min(target)), by = .(scenario, instance)] +dat[, incumbent := cummin(normalized_regret), by = .(method, scenario, instance, repl)] + +get_incumbent_cumbudget = function(incumbent, cumbudget_scaled) { + budgets = seq(0, 1, length.out = 101) + map_dbl(budgets, function(budget) { + ind = which(cumbudget_scaled <= budget) + if (length(ind) == 0L) { + max(incumbent) + } else { + min(incumbent[ind]) + } + }) +} + +dat_budget = dat[, .(incumbent_budget = get_incumbent_cumbudget(incumbent, cumbudget_scaled), cumbudget_scaled = seq(0, 1, length.out = 101)), by = .(method, scenario, instance, repl)] + +agg_budget = dat_budget[, .(mean = mean(incumbent_budget), se = sd(incumbent_budget) / sqrt(.N)), by = .(cumbudget_scaled, method, scenario, instance)] +#agg_budget[, method := factor(method, levels = c("random", "smac4hpo", "hb", "bohb", "dehb", "smac4mf", "optuna", "mlr3mbo", "mlr3mbo_default"), labels = c("Random", "SMAC", "HB", "BOHB", "DEHB", "SMAC-HB", "optuna", "mlr3mbo", "mlr3mbo_default"))] + +g = ggplot(aes(x = cumbudget_scaled, y = mean, colour = method, fill = method), data = agg_budget[cumbudget_scaled > 0.10]) + + scale_y_log10() + + geom_step() + + geom_stepribbon(aes(min = mean - se, max = mean + se), colour = NA, alpha = 0.3) + + labs(x = "Fraction of Budget Used", y = "Mean Normalized Regret", colour = "Optimizer", fill = "Optimizer") + + facet_wrap(~ scenario + instance, scales = "free", ncol = 5) + + theme_minimal() + + theme(legend.position = "bottom", legend.title = element_text(size = rel(0.75)), legend.text = element_text(size = rel(0.5))) +ggsave("anytime.png", plot = g, device = "png", width = 15, height = 10) + +overall_budget = agg_budget[, .(mean = mean(mean), se = sd(mean) / sqrt(.N)), by = .(method, cumbudget_scaled)] + +g = ggplot(aes(x = cumbudget_scaled, y = mean, colour = method, fill = method), data = overall_budget[cumbudget_scaled > 0.10]) + + scale_y_log10() + + geom_step() + + geom_stepribbon(aes(min = mean - se, max = mean + se), colour = NA, alpha = 0.1) + + labs(x = "Fraction of Budget Used", y = "Mean Normalized Regret", colour = "Optimizer", fill = "Optimizer") + + theme_minimal() + + theme(legend.position = "bottom", legend.title = element_text(size = rel(0.75)), legend.text = element_text(size = rel(0.75))) +ggsave("anytime_average.png", plot = g, device = "png", width = 6, height = 4) + +methods = unique(agg_budget$method) +ranks = map_dtr(unique(agg_budget$scenario), function(scenario_) { + map_dtr(unique(agg_budget$instance), function(instance_) { + map_dtr(unique(agg_budget$cumbudget_scaled), function(cumbudget_scaled_) { + res = agg_budget[scenario == scenario_ & instance == instance_ & cumbudget_scaled == cumbudget_scaled_] + if (nrow(res) == 0L) { + return(data.table()) + } + setorderv(res, "mean") + data.table(rank = match(methods, res$method), method = methods, scenario = scenario_, instance = instance_, cumbudget_scaled = cumbudget_scaled_) + }) + }) +}) + +ranks_overall = ranks[, .(mean = mean(rank), se = sd(rank) / sqrt(.N)), by = .(method, cumbudget_scaled)] + +g = ggplot(aes(x = cumbudget_scaled, y = mean, colour = method, fill = method), data = ranks_overall[cumbudget_scaled > 0.10]) + + geom_line() + + geom_ribbon(aes(min = mean - se, max = mean + se), colour = NA, alpha = 0.3) + + labs(x = "Fraction of Budget Used", y = "Mean Rank", colour = "Optimizer", fill = "Optimizer") + + theme_minimal() + + theme(legend.position = "bottom", legend.title = element_text(size = rel(0.75)), legend.text = element_text(size = rel(0.75))) +ggsave("anytime_average_rank.png", plot = g, device = "png", width = 6, height = 4) + +library(scmamp) +best_agg = agg_budget[cumbudget_scaled == 0.25] +best_agg[, problem := paste0(scenario, "_", instance)] +tmp = - as.matrix(dcast(best_agg, problem ~ method, value.var = "mean")[, -1]) +friedmanTest(tmp) +png("cd_025_mf.png", width = 6, height = 4, pointsize = 10, units = "in", res = 150) +plotCD(tmp, cex = 1) +dev.off() + +best_agg = agg_budget[cumbudget_scaled == 1] +best_agg[, problem := paste0(scenario, "_", instance)] +tmp = - as.matrix(dcast(best_agg, problem ~ method, value.var = "mean")[, -1]) +friedmanTest(tmp) +png("cd_1_mf.png", width = 6, height = 4, pointsize = 10, units = "in", res = 150) +plotCD(tmp, cex = 1) +dev.off() + diff --git a/attic/so_config/cd_example.R b/attic/so_config/cd_example.R new file mode 100644 index 00000000..6917bea5 --- /dev/null +++ b/attic/so_config/cd_example.R @@ -0,0 +1,32 @@ +library(bbotk) +library(paradox) +library(data.table) +library(R6) +library(checkmate) +library(mlr3misc) + +source("OptimizerCoordinateDescent.R") + +domain = ps(x1 = p_dbl(lower = -1, upper = 1), + x2 = p_fct(c("a", "b", "c")), + x3 = p_int(lower = 1, upper = 3, depends = x2 == "a"), + x4 = p_lgl(), + x6 = p_dbl(lower = 0, upper = 10, depends = (x2 == "b" && x4 == TRUE)), + x7 = p_fct(c("A", "B"), depends = x2 %in% c("a", "b"))) + +objective = ObjectiveRFunDt$new( + fun = function(xdt) { + x3 = xdt$x3 + x3[is.na(x3)] = 1 + x6 = xdt$x6 + x6[is.na(x6)] = 0 + y = x3 * xdt$x1 - x6 + data.table(y = y + rnorm(nrow(xdt))) + }, + domain = domain, + codomain = ps(y = p_dbl(tags = "minimize")) +) + +instance = OptimInstanceSingleCrit$new(objective = objective, terminator = trm("none")) +optimizer = OptimizerCoordinateDescent$new() +optimizer$optimize(instance) diff --git a/attic/so_config/coordinate_descent.R b/attic/so_config/coordinate_descent.R new file mode 100644 index 00000000..70519b40 --- /dev/null +++ b/attic/so_config/coordinate_descent.R @@ -0,0 +1,278 @@ +#!/usr/bin/env Rscript +# chmod ug+x +library(data.table) +library(mlr3) +library(mlr3learners) +library(mlr3pipelines) +library(mlr3misc) +library(mlr3mbo) # @so_config +library(bbotk) # @localsearch +library(paradox) +library(R6) +library(checkmate) + +# FIXME: error handling +# how to continue after one job is finished (i.e. from an intermediate gen) save seed? + +reticulate::use_condaenv("/home/lschnei8/.conda/envs/env", required = TRUE) +library(reticulate) +library(yahpogym) +library("future") +library("future.batchtools") +library("future.apply") + +source("OptimizerCoordinateDescent.R") +source("AcqFunctionLogEI.R") +source("LearnerRegrRangerCustom.R") +source("OptimizerChain.R") + +search_space = ps( + loop_function = p_fct(c("ego", "ego_log"), default = "ego"), + init = p_fct(c("random", "lhs", "sobol"), default = "random"), + init_size_fraction = p_fct(c("0.05", "0.10", "0.25"), default = "0.25"), + random_interleave = p_lgl(default = FALSE), + random_interleave_iter = p_fct(c("2", "5", "10"), depends = random_interleave == TRUE, default = "10"), + rf_type = p_fct(c("standard", "extratrees", "smaclike_boot", "smaclike_no_boot"), default = "standard"), + acqf = p_fct(c("EI", "CB", "PI", "Mean"), default = "EI"), + acqf_ei_log = p_lgl(depends = loop_function == "ego_log" && acqf == "EI", default = FALSE), + lambda = p_fct(c("1", "3", "10"), depends = acqf == "CB", default = "1"), + acqopt = p_fct(c("RS_1000", "RS", "FS", "LS"), default = "RS_1000") +) + +instances = setup = data.table(scenario = rep(c("lcbench", paste0("rbv2_", c("aknn", "glmnet", "ranger", "rpart", "super", "svm", "xgboost"))), each = 4L), + instance = c("167185", "167152", "168910", "189908", + "40499", "1476", "6", "12", + "40979", "1501", "40966", "1478", + "12", "458", "1510", "1515", + "1478", "40979", "12", "28", + "41164", "37", "1515", "1510", + "1478", "1501", "40499", "40979", + "40984", "40979", "40966", "28"), + target = rep(c("val_accuracy", "acc"), c(4L, 28L)), + budget = rep(c(126L, 118L, 90L, 134L, 110L, 267L, 118L, 170L), each = 4L)) +instances[, problem := paste0(scenario, "_", instance)] +setorderv(instances, col = "budget", order = -1L) + +fs_average = readRDS("/gscratch/lschnei8/results_yahpo_fs_average.rds") +fs_extrapolation = readRDS("/gscratch/lschnei8/fs_extrapolation.rds") + +get_k = function(best, scenario_, instance_, budget_) { + # assumes maximization + if (best > max(fs_average[scenario == scenario_ & instance == instance_][["mean_best"]])) { + extrapolate = TRUE + k = fs_extrapolation[scenario == scenario_ & instance == instance_][["model"]][[1L]](best) + } else { + extrapolate = FALSE + k = min(fs_average[scenario == scenario_ & instance == instance_ & mean_best >= best]$iter) # min k so that mean_best_fs[k] >= best_mbo[final] + } + k = k / budget_ # sample efficiency compared to fs + attr(k, "extrapolate") = extrapolate + k +} + +evaluate = function(xdt, instance) { + id = xdt$id + repl = xdt$repl + xdt$id = NULL + xdt$repl = NULL + + library(data.table) + library(mlr3) + library(mlr3learners) + library(mlr3pipelines) + library(mlr3misc) + library(mlr3mbo) # @so_config + library(bbotk) # @localsearch + library(paradox) + library(R6) + library(checkmate) + reticulate::use_condaenv("/home/lschnei8/.conda/envs/env", required = TRUE) + library(reticulate) + library(yahpogym) + + source("AcqFunctionLogEI.R") + source("LearnerRegrRangerCustom.R") + source("OptimizerChain.R") + + logger = lgr::get_logger("mlr3") + logger$set_threshold("warn") + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + + make_optim_instance = function(instance) { + benchmark = BenchmarkSet$new(instance$scenario, instance = instance$instance) + benchmark$subset_codomain(instance$target) + objective = benchmark$get_objective(instance$instance, multifidelity = FALSE, check_values = FALSE) + budget = instance$budget + optim_instance = OptimInstanceSingleCrit$new(objective, search_space = benchmark$get_search_space(drop_fidelity_params = TRUE), terminator = trm("evals", n_evals = budget), check_values = FALSE) + optim_instance + } + + optim_instance = make_optim_instance(instance) + + init_design_size = ceiling(as.numeric(xdt$init_size_fraction) * optim_instance$terminator$param_set$values$n_evals) + init_design = if (xdt$init == "random") { + generate_design_random(optim_instance$search_space, n = init_design_size)$data + } else if (xdt$init == "lhs") { + generate_design_lhs(optim_instance$search_space, n = init_design_size)$data + } else if (xdt$init == "sobol") { + generate_design_sobol(optim_instance$search_space, n = init_design_size)$data + } + + optim_instance$eval_batch(init_design) + + random_interleave_iter = if(xdt$random_interleave) as.numeric(xdt$random_interleave_iter) else 0L + + learner = LearnerRegrRangerCustom$new() + learner$predict_type = "se" + learner$param_set$values$keep.inbag = TRUE + + if (xdt$rf_type == "standard") { + learner$param_set$values$se.method = "jack" + learner$param_set$values$splitrule = "variance" + learner$param_set$values$num.trees = 1000L + } else if (xdt$rf_type == "extratrees") { + learner$param_set$values$se.method = "jack" + learner$param_set$values$splitrule = "extratrees" + learner$param_set$values$num.random.splits = 1L + learner$param_set$values$num.trees = 1000L + } else if (xdt$rf_type == "smaclike_boot") { + learner$param_set$values$se.method = "simple" + learner$param_set$values$splitrule = "extratrees" + learner$param_set$values$num.random.splits = 1L + learner$param_set$values$num.trees = 10L + learner$param_set$values$replace = TRUE + learner$param_set$values$sample.fraction = 1 + learner$param_set$values$min.node.size = 1 + learner$param_set$values$mtry.ratio = 1 + } else if (xdt$rf_type == "smaclike_no_boot") { + learner$param_set$values$se.method = "simple" + learner$param_set$values$splitrule = "extratrees" + learner$param_set$values$num.random.splits = 1L + learner$param_set$values$num.trees = 10L + learner$param_set$values$replace = FALSE + learner$param_set$values$sample.fraction = 1 + learner$param_set$values$min.node.size = 1 + learner$param_set$values$mtry.ratio = 1 + } + + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% + po("imputeoor", multiplier = 3, affect_columns = selector_type(c("integer", "numeric", "character", "factor", "ordered"))) %>>% + po("colapply", applicator = as.factor, affect_columns = selector_type("character")) %>>% + learner)) + surrogate$param_set$values$catch_errors = FALSE + + acq_optimizer = if (xdt$acqopt == "RS_1000") { + AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 1000L)) + } else if (xdt$acqopt == "RS") { + AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 20000L)) + } else if (xdt$acqopt == "FS") { + n_repeats = 2L + maxit = 9L + batch_size = ceiling((20000L / n_repeats) / (1 + maxit)) # 1000L + AcqOptimizer$new(opt("focus_search", n_points = batch_size, maxit = maxit), terminator = trm("evals", n_evals = 20000L)) + } else if (xdt$acqopt == "LS") { + optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10000L), trm("evals", n_evals = 10000L))) + acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20000L)) + acq_optimizer$param_set$values$warmstart = TRUE + acq_optimizer$param_set$values$warmstart_size = "all" + acq_optimizer + } + acq_optimizer$param_set$values$catch_errors = FALSE + + acq_function = if (xdt$acqf == "EI") { + if (isTRUE(xdt$acqf_ei_log)) { + AcqFunctionLogEI$new() + } else { + AcqFunctionEI$new() + } + } else if (xdt$acqf == "CB") { + AcqFunctionCB$new(lambda = as.numeric(xdt$lambda)) + } else if (xdt$acqf == "PI") { + AcqFunctionPI$new() + } else if (xdt$acqf == "Mean") { + AcqFunctionMean$new() + } + + if (xdt$loop_function == "ego") { + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + } else if (xdt$loop_function == "ego_log") { + bayesopt_ego_log(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + } + + best = optim_instance$archive$best()[[instance$target]] + + data.table(best = best, scenario = instance$scenario, instance = instance$instance, target = instance$target, id = id, repl = repl) +} + +objective = ObjectiveRFunDt$new( + fun = function(xdt) { + n_repl = 5L + xdt[, id := seq_len(.N)] + xdt_tmp = map_dtr(seq_len(nrow(instances)), function(i) copy(xdt)) + setorderv(xdt_tmp, col = "id") + xdt_tmp = map_dtr(seq_len(n_repl), function(i) { + tmp = copy(xdt_tmp) + tmp[, repl := i] + tmp + }) + instances_tmp = map_dtr(seq_len(nrow(xdt) * n_repl), function(i) copy(instances)) + current_seed = .Random.seed + res = tryCatch(future_mapply(FUN = evaluate, transpose_list(xdt_tmp), transpose_list(instances_tmp), SIMPLIFY = FALSE, future.seed = TRUE, future.scheduling = FALSE), error = function(ec) ec) + if (inherits(res, "error")) { + browser() # last manual fallback resort to get things working again + # cleanup future stuff + # reset the current seed and continue the eval + # .Random.seed = current_seed + # res = future_mapply(FUN = evaluate, transpose_list(xdt_tmp), transpose_list(instances_tmp), SIMPLIFY = FALSE, future.seed = TRUE, future.scheduling = FALSE) + } + res = rbindlist(res) + setorderv(res, col = "instance") + setorderv(res, col = "id") + setorderv(res, col = "repl") + res[, problem := paste0(scenario, "_", instance)] + + # average best over replications and determine ks + agg = res[, .(mean_best = mean(best), raw_best = list(best), n_na = sum(is.na(best)), n = .N), by = .(id, problem, scenario, instance)] + ks = map_dbl(seq_len(nrow(agg)), function(i) { + if (agg[i, ][["n"]] < n_repl) { + 0 + } else { + tryCatch( + get_k(agg[i, ][["mean_best"]], + scenario_ = agg[i, ][["scenario"]], + instance_ = agg[i, ][["instance"]], + budget_ = instances[problem == agg[i, ][["problem"]]][["budget"]]), + error = function(ec) 0 + ) + } + }) + agg[, k := ks] + + # average k over instances and determine mean_k + agg_k = agg[, .(mean_k = exp(mean(log(k))), raw_k = list(k), n_na = sum(is.na(k)), n = .N, raw_mean_best = list(mean_best)), by = .(id)] + agg_k[n < nrow(instances), mean_k := 0] + agg_k + }, + domain = search_space, + codomain = ps(mean_k = p_dbl(tags = "maximize")), + check_values = FALSE +) + +cd_instance = OptimInstanceSingleCrit$new( + objective = objective, + search_space = search_space, + terminator = trm("none") # OptimizerCoordinateDescent currently terminates on its own +) + +optimizer = OptimizerCoordinateDescent$new() +optimizer$param_set$values$max_gen = 5L + +init = data.table(loop_function = "ego", init = "random", init_size_fraction = "0.25", random_interleave = FALSE, random_interleave_iter = NA_character_, rf_type = "standard", acqf = "EI", acqf_ei_log = NA, lambda = NA_character_, acqopt = "RS_1000") +options(future.cache.path = "/home/lschnei8/mlr3mbo_config/future") +set.seed(2906, kind = "L'Ecuyer-CMRG") +# currently we evaluate at most 4 * 32 * 5 jobs in parallel so 650 workers is enough +plan("batchtools_slurm", template = "slurm_wyoming_cd.tmpl", resources = list(walltime = 3600L * 9L, ncpus = 1L, memory = 4000L), workers = 650L) +cd_instance$eval_batch(init) +optimizer$optimize(cd_instance) + diff --git a/attic/so_config/helpers.R b/attic/so_config/helpers.R new file mode 100644 index 00000000..a66f06b4 --- /dev/null +++ b/attic/so_config/helpers.R @@ -0,0 +1,9 @@ +make_optim_instance = function(instance) { + benchmark = BenchmarkSet$new(as.character(instance$scenario), instance = as.character(instance$instance)) + benchmark$subset_codomain(instance$target) + objective = benchmark$get_objective(as.character(instance$instance), multifidelity = FALSE, check_values = FALSE) + n_evals = as.integer(ceiling(instance$budget / instance$max_budget)) # full budget + optim_instance = OptimInstanceSingleCrit$new(objective, search_space = benchmark$get_search_space(drop_fidelity_params = TRUE), terminator = trm("evals", n_evals = n_evals), check_values = FALSE) + optim_instance +} + diff --git a/attic/so_config/noisyness_of_k.R b/attic/so_config/noisyness_of_k.R new file mode 100644 index 00000000..5c2373d9 --- /dev/null +++ b/attic/so_config/noisyness_of_k.R @@ -0,0 +1,45 @@ +library(data.table) +library(mlr3misc) + +fs_average = readRDS("results_yahpo_fs_average.rds") +fs = readRDS("results_yahpo_fs.rds")[scenario == "lcbench" & instance == 167152] + + +get_k = function(best, scenario_, instance_, budget_) { + # assumes maximization + if (best > max(fs_average[scenario == scenario_ & instance == instance_][["mean_best"]])) { + extrapolate = TRUE + k = fs_extrapolation[scenario == scenario_ & instance == instance_][["model"]][[1L]](best) + } else { + extrapolate = FALSE + k = min(fs_average[scenario == scenario_ & instance == instance_ & mean_best >= best]$iter) # min k so that mean_best_fs[k] >= best_mbo[final] + } + k = k / budget_ # sample efficiency compared to fs + attr(k, "extrapolate") = extrapolate + k +} + +k = get_k(97.543, scenario_ = "lcbench", instance_ = 167152, budget_ = 126) + +perfs = 97.543 + rnorm(100L, mean = 0, sd = 0.01) +ks = map_dbl(perfs, function(perf) get_k(perf, scenario_ = "lcbench", instance_ = 167152, budget_ = 126)) + +png("k_noise.png") +plot(perfs, ks, xlab = "Validation Accuracy", ylab = "k") +dev.off() + +ks_alt = map_dtr(perfs, function(perf) { + tmp = map_dbl(1:30, function(i) { + min(fs[scenario == "lcbench" & instance == 167152 & repl == i & best >= perf]$iter) / 126 + }) + data.table(mean = mean(tmp), geom = exp(mean(log(tmp)))) +}) + +png("k_noise_mean.png") +plot(perfs, ks_alt$mean, xlab = "Validation Accuracy", ylab = "k") +dev.off() + +png("k_noise_geom.png") +plot(perfs, ks_alt$geom, xlab = "Validation Accuracy", ylab = "k") +dev.off() + diff --git a/attic/so_config/run_yahpo.R b/attic/so_config/run_yahpo.R new file mode 100644 index 00000000..3dd60a4d --- /dev/null +++ b/attic/so_config/run_yahpo.R @@ -0,0 +1,390 @@ +library(batchtools) +library(data.table) +library(mlr3) +library(mlr3learners) +library(mlr3pipelines) +library(mlr3misc) +library(mlr3mbo) # @so_config +library(bbotk) # @localsearch +library(paradox) +library(R6) +library(checkmate) + +reticulate::use_condaenv("/home/lschnei8/.conda/envs/env", required = TRUE) +library(reticulate) +yahpo_gym = import("yahpo_gym") + +packages = c("data.table", "mlr3", "mlr3learners", "mlr3pipelines", "mlr3misc", "mlr3mbo", "bbotk", "paradox", "R6", "checkmate") + +#RhpcBLASctl::blas_set_num_threads(1L) +#RhpcBLASctl::omp_set_num_threads(1L) + +root = here::here() +experiments_dir = file.path(root) + +source_files = map_chr(c("helpers.R", "AcqFunctionLogEI.R", "LearnerRegrRangerCustom.R", "LearnerRegrGowerNNEnsemble.R", "OptimizerChain.R"), function(x) file.path(experiments_dir, x)) +for (sf in source_files) { + source(sf) +} + +reg = makeExperimentRegistry(file.dir = "/gscratch/lschnei8/registry_mlr3mbo_lfbox", packages = packages, source = source_files) +#reg = makeExperimentRegistry(file.dir = NA, conf.file = NA, packages = packages, source = source_files) # interactive session +saveRegistry(reg) + +mlr3mbo_wrapper = function(job, data, instance, ...) { + reticulate::use_condaenv("/home/lschnei8/.conda/envs/env", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + optim_instance = make_optim_instance(instance) + + xdt = data.table(loop_function = "ego_log", init = "sobol", init_size_fraction = "0.25", random_interleave = FALSE, random_interleave_iter = "NA_chr", rf_type = "smaclike_boot", acqf = "EI", acqf_ei_log = FALSE, lambda = "NA_chr", acqopt = "LS") + + init_design_size = ceiling(as.numeric(xdt$init_size_fraction) * optim_instance$terminator$param_set$values$n_evals) + init_design = if (xdt$init == "random") { + generate_design_random(optim_instance$search_space, n = init_design_size)$data + } else if (xdt$init == "lhs") { + generate_design_lhs(optim_instance$search_space, n = init_design_size)$data + } else if (xdt$init == "sobol") { + generate_design_sobol(optim_instance$search_space, n = init_design_size)$data + } + + optim_instance$eval_batch(init_design) + + random_interleave_iter = if(xdt$random_interleave) as.numeric(xdt$random_interleave_iter) else 0L + + learner = LearnerRegrRangerCustom$new() + learner$predict_type = "se" + learner$param_set$values$keep.inbag = TRUE + + if (xdt$rf_type == "standard") { + learner$param_set$values$se.method = "jack" + learner$param_set$values$splitrule = "variance" + learner$param_set$values$num.trees = 1000L + } else if (xdt$rf_type == "extratrees") { + learner$param_set$values$se.method = "jack" + learner$param_set$values$splitrule = "extratrees" + learner$param_set$values$num.random.splits = 1L + learner$param_set$values$num.trees = 1000L + } else if (xdt$rf_type == "smaclike_boot") { + learner$param_set$values$se.method = "simple" + learner$param_set$values$splitrule = "extratrees" + learner$param_set$values$num.random.splits = 1L + learner$param_set$values$num.trees = 10L + learner$param_set$values$replace = TRUE + learner$param_set$values$sample.fraction = 1 + learner$param_set$values$min.node.size = 1 + learner$param_set$values$mtry.ratio = 1 + } else if (xdt$rf_type == "smaclike_no_boot") { + learner$param_set$values$se.method = "simple" + learner$param_set$values$splitrule = "extratrees" + learner$param_set$values$num.random.splits = 1L + learner$param_set$values$num.trees = 10L + learner$param_set$values$replace = FALSE + learner$param_set$values$sample.fraction = 1 + learner$param_set$values$min.node.size = 1 + learner$param_set$values$mtry.ratio = 1 + } + + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% + po("imputeoor", multiplier = 3, affect_columns = selector_type(c("integer", "numeric", "character", "factor", "ordered"))) %>>% + po("colapply", applicator = as.factor, affect_columns = selector_type("character")) %>>% + learner)) + + acq_optimizer = if (xdt$acqopt == "RS_1000") { + AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 1000L)) + } else if (xdt$acqopt == "RS") { + AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 20000L)) + } else if (xdt$acqopt == "FS") { + n_repeats = 2L + maxit = 9L + batch_size = ceiling((20000L / n_repeats) / (1 + maxit)) # 1000L + AcqOptimizer$new(opt("focus_search", n_points = batch_size, maxit = maxit), terminator = trm("evals", n_evals = 20000L)) + } else if (xdt$acqopt == "LS") { + optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10000L), trm("evals", n_evals = 10000L))) + acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20000L)) + acq_optimizer$param_set$values$warmstart = TRUE + acq_optimizer$param_set$values$warmstart_size = "all" + acq_optimizer + } + + acq_function = if (xdt$acqf == "EI") { + if (isTRUE(xdt$acqf_ei_log)) { + AcqFunctionLogEI$new() + } else { + AcqFunctionEI$new() + } + } else if (xdt$acqf == "CB") { + AcqFunctionCB$new(lambda = as.numeric(xdt$lambda)) + } else if (xdt$acqf == "PI") { + AcqFunctionPI$new() + } else if (xdt$acqf == "Mean") { + AcqFunctionMean$new() + } + + if (xdt$loop_function == "ego") { + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + } else if (xdt$loop_function == "ego_log") { + bayesopt_ego_log(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + } + + optim_instance +} + +mlr3mbox_wrapper = function(job, data, instance, ...) { + reticulate::use_condaenv("/home/lschnei8/.conda/envs/env", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + optim_instance = make_optim_instance(instance) + + xdt = data.table(loop_function = "ego_log", init = "sobol", init_size_fraction = "0.25", random_interleave = FALSE, random_interleave_iter = "NA_chr", acqf = "EI", acqf_ei_log = TRUE, lambda = "NA_chr", acqopt = "LS") + + init_design_size = ceiling(as.numeric(xdt$init_size_fraction) * optim_instance$terminator$param_set$values$n_evals) + init_design = if (xdt$init == "random") { + generate_design_random(optim_instance$search_space, n = init_design_size)$data + } else if (xdt$init == "lhs") { + generate_design_lhs(optim_instance$search_space, n = init_design_size)$data + } else if (xdt$init == "sobol") { + generate_design_sobol(optim_instance$search_space, n = init_design_size)$data + } + + optim_instance$eval_batch(init_design) + + random_interleave_iter = if(xdt$random_interleave) as.numeric(xdt$random_interleave_iter) else 0L + + learner = LearnerRegrGowerNNEnsemble$new() + learner$predict_type = "se" + learner$param_set$values$ks = rep(1, 10) + + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% + po("imputeoor", multiplier = 10, affect_columns = selector_type(c("integer", "numeric", "character", "factor", "ordered"))) %>>% + po("colapply", applicator = as.factor, affect_columns = selector_type("character")) %>>% + learner)) + + acq_optimizer = if (xdt$acqopt == "RS_1000") { + AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 1000L)) + } else if (xdt$acqopt == "RS") { + AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 20000L)) + } else if (xdt$acqopt == "FS") { + n_repeats = 2L + maxit = 9L + batch_size = ceiling((20000L / n_repeats) / (1 + maxit)) # 1000L + AcqOptimizer$new(opt("focus_search", n_points = batch_size, maxit = maxit), terminator = trm("evals", n_evals = 20000L)) + } else if (xdt$acqopt == "LS") { + optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10000L), trm("evals", n_evals = 10000L))) + acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20000L)) + acq_optimizer$param_set$values$warmstart = TRUE + acq_optimizer$param_set$values$warmstart_size = "all" + acq_optimizer + } + + acq_function = if (xdt$acqf == "EI") { + if (isTRUE(xdt$acqf_ei_log)) { + AcqFunctionLogEI$new() + } else { + AcqFunctionEI$new() + } + } else if (xdt$acqf == "CB") { + AcqFunctionCB$new(lambda = as.numeric(xdt$lambda)) + } else if (xdt$acqf == "PI") { + AcqFunctionPI$new() + } else if (xdt$acqf == "Mean") { + AcqFunctionMean$new() + } + + if (xdt$loop_function == "ego") { + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + } else if (xdt$loop_function == "ego_log") { + bayesopt_ego_log(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + } + + optim_instance +} + +lfbo_wrapper = function(job, data, instance, ...) { + reticulate::use_condaenv("/home/lschnei8/.conda/envs/env", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + optim_instance = make_optim_instance(instance) + + init_design_size = ceiling(0.25 * optim_instance$terminator$param_set$values$n_evals) + init_design = generate_design_random(optim_instance$search_space, n = init_design_size)$data + + optim_instance$eval_batch(init_design) + + random_interleave_iter = 0L + + learner = lrn("regr.lfbo", lrn("classif.ranger"), lfbo.direction = optim_instance$objective$codomain$tags[[optim_instance$archive$cols_y]]) + learner$param_set$values$keep.inbag = TRUE + + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% + po("imputeoor", multiplier = 3, affect_columns = selector_type(c("integer", "numeric", "character", "factor", "ordered"))) %>>% + po("colapply", applicator = as.factor, affect_columns = selector_type("character")) %>>% + learner)) + + optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10000L), trm("evals", n_evals = 10000L))) + acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20000L)) + acq_optimizer$param_set$values$warmstart = TRUE + acq_optimizer$param_set$values$warmstart_size = "all" + + acq_function = AcqFunctionLFBO$new() + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + + optim_instance +} + +lfbox_wrapper = function(job, data, instance, ...) { + reticulate::use_condaenv("/home/lschnei8/.conda/envs/env", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + optim_instance = make_optim_instance(instance) + + init_design_size = ceiling(0.25 * optim_instance$terminator$param_set$values$n_evals) + init_design = generate_design_random(optim_instance$search_space, n = init_design_size)$data + + optim_instance$eval_batch(init_design) + + random_interleave_iter = 0L + + learner = lrn("regr.lfbo", lrn("classif.ranger", num.trees = 1000, min.node.size = 1), lfbo.direction = optim_instance$objective$codomain$tags[[optim_instance$archive$cols_y]]) + learner$param_set$values$keep.inbag = TRUE + + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% + po("imputeoor", multiplier = 3, affect_columns = selector_type(c("integer", "numeric", "character", "factor", "ordered"))) %>>% + po("colapply", applicator = as.factor, affect_columns = selector_type("character")) %>>% + learner)) + + acq_optimizer = AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 1000L)) + + acq_function = AcqFunctionLFBO$new() + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + + optim_instance +} + +# add algorithms +addAlgorithm("mlr3mbo", fun = mlr3mbo_wrapper) +addAlgorithm("mlr3mbox", fun = mlr3mbox_wrapper) +addAlgorithm("lfbo", fun = lfbo_wrapper) +addAlgorithm("lfbox", fun = lfbox_wrapper) + +# setup scenarios and instances +get_nb301_setup = function(budget_factor = 40L) { + scenario = "nb301" + bench = yahpo_gym$benchmark_set$BenchmarkSet(scenario, instance = "CIFAR10") + fidelity_space = bench$get_fidelity_space() + fidelity_param_id = fidelity_space$get_hyperparameter_names()[1] + min_budget = fidelity_space$get_hyperparameter(fidelity_param_id)$lower + max_budget = fidelity_space$get_hyperparameter(fidelity_param_id)$upper + ndim = length(bench$config_space$get_hyperparameter_names()) - 1L # NOTE: instance is not part of + + instances = "CIFAR10" + target = "val_accuracy" + budget = ceiling(20L * max_budget + sqrt(ndim) * max_budget * budget_factor) + on_integer_scale = TRUE + minimize = bench$config$config$y_minimize[match(target, bench$config$config$y_names)] + setup = setDT(expand.grid(scenario = scenario, instance = instances, target = target, ndim = ndim, max_budget = max_budget, budget = budget, on_integer_scale = on_integer_scale, minimize = minimize, stringsAsFactors = FALSE)) + setup +} + +get_lcbench_setup = function(budget_factor = 40L) { + scenario = "lcbench" + bench = yahpo_gym$benchmark_set$BenchmarkSet(scenario, instance = "167168") + fidelity_space = bench$get_fidelity_space() + fidelity_param_id = fidelity_space$get_hyperparameter_names()[1] + min_budget = fidelity_space$get_hyperparameter(fidelity_param_id)$lower + max_budget = fidelity_space$get_hyperparameter(fidelity_param_id)$upper + ndim = length(bench$config_space$get_hyperparameter_names()) - 2L + + instances = c("167168", "189873", "189906") + target = "val_accuracy" + budget = ceiling(20L * max_budget + sqrt(ndim) * max_budget * budget_factor) + on_integer_scale = TRUE + minimize = bench$config$config$y_minimize[match(target, bench$config$config$y_names)] + setup = setDT(expand.grid(scenario = scenario, instance = instances, target = target, ndim = ndim, max_budget = max_budget, budget = budget, on_integer_scale = on_integer_scale, minimize = minimize, stringsAsFactors = FALSE)) + setup +} + +get_rbv2_setup = function(budget_factor = 40L) { + setup = map_dtr(c("rbv2_glmnet", "rbv2_rpart", "rbv2_ranger", "rbv2_xgboost", "rbv2_super"), function(scenario) { + bench = yahpo_gym$benchmark_set$BenchmarkSet(scenario, instance = "1040") + fidelity_space = bench$get_fidelity_space() + fidelity_param_id = "trainsize" + min_budget = fidelity_space$get_hyperparameter(fidelity_param_id)$lower + max_budget = fidelity_space$get_hyperparameter(fidelity_param_id)$upper + ndim = length(bench$config_space$get_hyperparameter_names()) - 3L # repl and trainsize and instance + + instances = switch(scenario, rbv2_glmnet = c("375", "458"), rbv2_rpart = c("14", "40499"), rbv2_ranger = c("16", "42"), rbv2_xgboost = c("12", "1501", "16", "40499"), rbv2_super = c("1053", "1457", "1063", "1479", "15", "1468")) + target = "acc" + budget = ceiling(20L * max_budget + sqrt(ndim) * max_budget * budget_factor) + on_integer_scale = FALSE + minimize = bench$config$config$y_minimize[match(target, bench$config$config$y_names)] + setup = setDT(expand.grid(scenario = scenario, instance = instances, target = target, ndim = ndim, max_budget = max_budget, budget = budget, on_integer_scale = on_integer_scale, minimize = minimize, stringsAsFactors = FALSE)) + }) +} + +setup = rbind(get_nb301_setup(), get_lcbench_setup(), get_rbv2_setup()) + +setup[, id := seq_len(.N)] + +# add problems +prob_designs = map(seq_len(nrow(setup)), function(i) { + prob_id = paste0(setup[i, ]$scenario, "_", setup[i, ]$instance, "_", setup[i, ]$target) + addProblem(prob_id, data = list(scenario = setup[i, ]$scenario, instance = setup[i, ]$instance, target = setup[i, ]$target, ndim = setup[i, ]$ndim, max_budget = setup[i, ]$max_budget, budget = setup[i, ]$budget, on_integer_scale = setup[i, ]$on_integer_scale, minimize = setup[i, ]$minimize)) + setNames(list(setup[i, ]), nm = prob_id) +}) +nn = sapply(prob_designs, names) +prob_designs = unlist(prob_designs, recursive = FALSE, use.names = FALSE) +names(prob_designs) = nn + +# add jobs for optimizers +optimizers = data.table(algorithm = "mlr3mbox") + +for (i in seq_len(nrow(optimizers))) { + algo_designs = setNames(list(optimizers[i, ]), nm = optimizers[i, ]$algorithm) + + ids = addExperiments( + prob.designs = prob_designs, + algo.designs = algo_designs, + repls = 30L + ) + addJobTags(ids, as.character(optimizers[i, ]$algorithm)) +} + +jobs = getJobTable() +resources.default = list(walltime = 3600 * 9L, memory = 2048L, ntasks = 1L, ncpus = 1L, nodes = 1L, clusters = "beartooth", max.concurrent.jobs = 9999L) +submitJobs(jobs, resources = resources.default) + +done = findDone() +results = reduceResultsList(done, function(x, job) { + x = x$archive$data + budget_var = if (job$instance$scenario %in% c("lcbench", "nb301")) "epoch" else "trainsize" + target_var = job$instance$target + if (!job$instance$minimize) { + x[, (target_var) := - get(target_var)] + } + pars = job$pars + tmp = x[, target_var, with = FALSE] + tmp[, (budget_var) := job$instance$max_budget] + tmp[, method := pars$algo.pars$algorithm] + tmp[, scenario := pars$prob.pars$scenario] + tmp[, instance := pars$prob.pars$instance] + tmp[, repl := job$repl] + tmp[, iter := seq_len(.N)] + colnames(tmp) = c("target", "budget", "method", "scenario", "instance", "repl", "iter") + tmp +}) +results = rbindlist(results, fill = TRUE) +saveRDS(results, "/gscratch/lschnei8/results_yahpo_mbox.rds") + diff --git a/attic/so_config/run_yahpo_fs.R b/attic/so_config/run_yahpo_fs.R new file mode 100644 index 00000000..5612b1d1 --- /dev/null +++ b/attic/so_config/run_yahpo_fs.R @@ -0,0 +1,179 @@ +library(batchtools) +library(data.table) +library(mlr3) +library(mlr3misc) +library(bbotk) # @localsearch +library(paradox) +library(R6) +library(checkmate) + +reticulate::use_condaenv("/home/lschnei8/.conda/envs/env", required = TRUE) +library(reticulate) +yahpo_gym = import("yahpo_gym") + +packages = c("data.table", "mlr3", "mlr3misc", "bbotk", "paradox", "R6", "checkmate") + +#RhpcBLASctl::blas_set_num_threads(1L) +#RhpcBLASctl::omp_set_num_threads(1L) + +reg = makeExperimentRegistry(file.dir = "/gscratch/lschnei8/registry_yahpo_fs", packages = packages) +#reg = makeExperimentRegistry(file.dir = NA, conf.file = NA, packages = packages) # interactive session +saveRegistry(reg) + +fs_wrapper = function(job, data, instance, ...) { + # fs is our baseline with 3000 x more budget + reticulate::use_condaenv("/home/lschnei8/.conda/envs/env", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + make_optim_instance = function(instance) { + benchmark = BenchmarkSet$new(instance$scenario, instance = instance$instance) + benchmark$subset_codomain(instance$target) + objective = benchmark$get_objective(instance$instance, multifidelity = FALSE, check_values = FALSE) + budget = instance$budget + optim_instance = OptimInstanceSingleCrit$new(objective, search_space = benchmark$get_search_space(drop_fidelity_params = TRUE), terminator = trm("evals", n_evals = budget), check_values = FALSE) + optim_instance + } + + n_evals = instance$budget + batch_size = 10000L + maxit = ceiling((n_evals / (batch_size))) + + optimizer = opt("focus_search", n_points = batch_size, maxit = maxit) + + optim_instance = make_optim_instance(instance) + optimizer$optimize(optim_instance) + optim_instance +} + + +# add algorithms +addAlgorithm("fs", fun = fs_wrapper) + +setup = data.table(scenario = rep(c("lcbench", paste0("rbv2_", c("aknn", "glmnet", "ranger", "rpart", "super", "svm", "xgboost"))), each = 4L), + instance = c("167185", "167152", "168910", "189908", + "40499", "1476", "6", "12", + "40979", "1501", "40966", "1478", + "12", "458", "1510", "1515", + "1478", "40979", "12", "28", + "41164", "37", "1515", "1510", + "1478", "1501", "40499", "40979", + "40984", "40979", "40966", "28"), + target = rep(c("val_accuracy", "acc"), c(4L, 28L)), + budget = rep(c(126L, 118L, 90L, 134L, 110L, 267L, 118L, 170L), each = 4L)) +setup[, budget := budget * 3000L] + +setup[, id := seq_len(.N)] + +# add problems +prob_designs = map(seq_len(nrow(setup)), function(i) { + prob_id = paste0(setup[i, ]$scenario, "_", setup[i, ]$instance, "_", setup[i, ]$target) + addProblem(prob_id, data = list(scenario = setup[i, ]$scenario, instance = setup[i, ]$instance, target = setup[i, ]$target, budget = setup[i, ]$budget, on_integer_scale = setup[i, ]$on_integer_scale)) + setNames(list(setup[i, ]), nm = prob_id) +}) +nn = sapply(prob_designs, names) +prob_designs = unlist(prob_designs, recursive = FALSE, use.names = FALSE) +names(prob_designs) = nn + +# add jobs for optimizers +optimizers = data.table(algorithm = c("fs")) + +for (i in seq_len(nrow(optimizers))) { + algo_designs = setNames(list(optimizers[i, ]), nm = optimizers[i, ]$algorithm) + + ids = addExperiments( + prob.designs = prob_designs, + algo.designs = algo_designs, + repls = 30L + ) + addJobTags(ids, as.character(optimizers[i, ]$algorithm)) +} + +# rbv2_super 801000 budget needs ~ 15 minutes so 20 chunks results in roughly 5 hours +jobs = findJobs() +jobs[, chunk := batchtools::chunk(job.id, chunk.size = 20L)] +resources.default = list(walltime = 3600 * 6L, memory = 16000, ntasks = 1L, ncpus = 2L, nodes = 1L, clusters = "beartooth", max.concurrent.jobs = 9999L) +submitJobs(jobs, resources = resources.default) + +done = findDone() +results = reduceResultsList(done, function(x, job) { + x = x$archive$data + pars = job$pars + target_var = pars$prob.pars$target + tmp = x[, eval(target_var), with = FALSE] + colnames(tmp) = "target" + tmp[, best := cummax(target)] + tmp[, method := pars$algo.pars$algorithm] + tmp[, scenario := pars$prob.pars$scenario] + tmp[, instance := pars$prob.pars$instance] + tmp[, repl := job$repl] + tmp[, iter := seq_len(.N)] + tmp +}) +results = rbindlist(results, fill = TRUE) +saveRDS(results, "/gscratch/lschnei8/results_yahpo_fs.rds") + +mean_results = results[, .(mean_best = mean(best), se_best = sd(best) / sqrt(.N)), by = .(scenario, instance, iter)] +saveRDS(mean_results, "/gscratch/lschnei8/results_yahpo_fs_average.rds") + +library(ggplot2) +library(gridExtra) + +lm_data = copy(mean_results) +lm_data[, problem := paste0(scenario, "_", instance)] +models = map_dtr(unique(lm_data$problem), function(problem_) { + tmp = lm_data[problem == problem_] + values = tail(unique(tmp$mean_best), 2L) + dat = map_dtr(values, function(value) { + tmp[mean_best == value][.N, ] + }) + model = lm(mean_best ~ iter, data = dat) + coefs = coef(model) + if (coefs[2L] < .Machine$double.eps) { + stop("Almost constant linear model") + } + max_iter = max(tmp$iter) + estimate_iter = function(mean_best) { + iter = ceiling((mean_best - coefs[1L]) / coefs[2L]) + if (!isTRUE(iter > max_iter)) { + iter = max_iter + 1 + } + iter + } + env = new.env() + environment(estimate_iter) = env + assign("max_iter", value = max_iter, envir = env) + assign("coefs", value = coefs, envir = env) + if (estimate_iter(1.00001 * max(tmp$mean_best)) < max(tmp$iter)) { + # marginal improvements should require more iter than max iter + stop("Model does not interpolate latest iter well.") + } + tmp_p = data.table(iter = ((NROW(tmp) - 1L):ceiling(1.5 * NROW(tmp)))) + p = predict(model, tmp_p) + tmp_p[, mean_best := p] + tmp_p[, method := "interpolation"] + tmp_plot = copy(tmp)[, c("iter", "mean_best")] + tmp_plot[, method := "real"] + tmp_plot = rbind(tmp_plot, tmp_p) + g = ggplot(aes(x = iter, y = mean_best, colour = method), data = tmp_plot[ceiling(0.9 * NROW(tmp)):.N , ]) + + scale_y_log10() + + geom_step(direction = "vh") + + geom_hline(yintercept = max(tmp$mean_best), linetype = 2L) + + labs(title = problem_) + + theme_minimal() + + theme(legend.position = "bottom") + info = strsplit(problem_, "_")[[1L]] + if (length(info) == 3L) { + info = c(paste0(info[1L], "_", info[2L]), info[3L]) # rbv2_ + } + list(model = list(estimate_iter), plot = list(g), scenario = info[1L], instance = info[2L]) +}) + +g = do.call("grid.arrange", c(models$plot, ncol = 8L)) + +ggsave("fs_extrapolation.png", plot = g, width = 32, height = 12) + +saveRDS(models[, - "plot"], "fs_extrapolation.rds") + diff --git a/attic/so_config/run_yahpo_mies.R b/attic/so_config/run_yahpo_mies.R new file mode 100644 index 00000000..59092842 --- /dev/null +++ b/attic/so_config/run_yahpo_mies.R @@ -0,0 +1,197 @@ +library(batchtools) +library(data.table) +library(mlr3) +library(mlr3misc) +library(bbotk) # @localsearch +library(paradox) +library(miesmuschel) # @mlr3mbo_config +library(R6) +library(checkmate) + +reticulate::use_condaenv("/home/lschnei8/.conda/envs/env", required = TRUE) +library(reticulate) +yahpo_gym = import("yahpo_gym") + +packages = c("data.table", "mlr3", "mlr3misc", "bbotk", "paradox", "miesmuschel", "R6", "checkmate") + +#RhpcBLASctl::blas_set_num_threads(1L) +#RhpcBLASctl::omp_set_num_threads(1L) + +reg = makeExperimentRegistry(file.dir = "/gscratch/lschnei8/registry_yahpo_mies", packages = packages) +#reg = makeExperimentRegistry(file.dir = NA, conf.file = NA, packages = packages) # interactive session +saveRegistry(reg) + +mies_wrapper = function(job, data, instance, ...) { + # mies is our baseline with 1000 x more budget + reticulate::use_condaenv("/home/lschnei8/.conda/envs/env", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + make_optim_instance = function(instance) { + benchmark = BenchmarkSet$new(instance$scenario, instance = instance$instance) + benchmark$subset_codomain(instance$target) + objective = benchmark$get_objective(instance$instance, multifidelity = FALSE, check_values = FALSE) + budget = instance$budget + optim_instance = OptimInstanceSingleCrit$new(objective, search_space = benchmark$get_search_space(drop_fidelity_params = TRUE), terminator = trm("evals", n_evals = budget), check_values = FALSE) + optim_instance + } + + recombine = rec("maybe", recombinator = rec("cmpmaybe", recombinator = rec("swap"), p = 0.5), p = 0.7) + + mutate_gauss = mut("cmpmaybe", mutator = mut("gauss", sdev = 0.1, sdev_is_relative = TRUE), p = 0.2) + mutate_uniform = mut("cmpmaybe", mutator = mut("unif", can_mutate_to_same = FALSE), p = 0.2) + + mutate = mut("maybe", + mutator = mut("combine", operators = list(ParamDbl = mutate_gauss$clone(deep = TRUE), + ParamInt = mutate_gauss$clone(deep = TRUE), + ParamFct = mutate_uniform$clone(deep = TRUE), + ParamLgl = mutate_uniform$clone(deep = TRUE))), + p = 0.3) + + select = sel("random", sample_unique = "no") + + survive = sel("best") + + optimizer = opt("mies", recombinator = recombine, mutator = mutate, parent_selector = select, survival_selector = survive, + mu = 100, lambda = 100, survival_strategy = "plus") + + optim_instance = make_optim_instance(instance) + optimizer$optimize(optim_instance) + optim_instance +} + + +# add algorithms +addAlgorithm("mies", fun = mies_wrapper) + +setup = data.table(scenario = rep(c("lcbench", paste0("rbv2_", c("aknn", "glmnet", "ranger", "rpart", "super", "svm", "xgboost"))), each = 4L), + instance = c("167185", "167152", "168910", "189908", + "40499", "1476", "6", "12", + "40979", "1501", "40966", "1478", + "12", "458", "1510", "1515", + "1478", "40979", "12", "28", + "41164", "37", "1515", "1510", + "1478", "1501", "40499", "40979", + "40984", "40979", "40966", "28"), + target = rep(c("val_accuracy", "acc"), c(4L, 28L)), + budget = rep(c(126L, 118L, 90L, 134L, 110L, 267L, 118L, 170L), each = 4L)) +setup[, budget := budget * 1000L] + +setup[, id := seq_len(.N)] + +# add problems +prob_designs = map(seq_len(nrow(setup)), function(i) { + prob_id = paste0(setup[i, ]$scenario, "_", setup[i, ]$instance, "_", setup[i, ]$target) + addProblem(prob_id, data = list(scenario = setup[i, ]$scenario, instance = setup[i, ]$instance, target = setup[i, ]$target, budget = setup[i, ]$budget, on_integer_scale = setup[i, ]$on_integer_scale)) + setNames(list(setup[i, ]), nm = prob_id) +}) +nn = sapply(prob_designs, names) +prob_designs = unlist(prob_designs, recursive = FALSE, use.names = FALSE) +names(prob_designs) = nn + +# add jobs for optimizers +optimizers = data.table(algorithm = c("mies")) + +for (i in seq_len(nrow(optimizers))) { + algo_designs = setNames(list(optimizers[i, ]), nm = optimizers[i, ]$algorithm) + + ids = addExperiments( + prob.designs = prob_designs, + algo.designs = algo_designs, + repls = 30L + ) + addJobTags(ids, as.character(optimizers[i, ]$algorithm)) +} + +# rbv2_super 267000 budget needs ~ 20 minutes so 30 repls results in roughly 10 hours +jobs = findJobs() +jobs[, chunk := batchtools::chunk(job.id, chunk.size = 10L)] +resources.default = list(walltime = 3600 * 12L, memory = 8192, ntasks = 1L, ncpus = 2L, nodes = 1L, clusters = "beartooth", max.concurrent.jobs = 9999L) +submitJobs(jobs, resources = resources.default) + +done = findDone() +results = reduceResultsList(done, function(x, job) { + x = x$archive$data + pars = job$pars + target_var = pars$prob.pars$target + tmp = x[, eval(target_var), with = FALSE] + colnames(tmp) = "target" + tmp[, best := cummax(target)] + tmp[, method := pars$algo.pars$algorithm] + tmp[, scenario := pars$prob.pars$scenario] + tmp[, instance := pars$prob.pars$instance] + tmp[, repl := job$repl] + tmp[, iter := seq_len(.N)] + tmp +}) +results = rbindlist(results, fill = TRUE) +saveRDS(results, "/gscratch/lschnei8/results_yahpo_mies.rds") + +mean_results = results[, .(mean_best = mean(best), se_best = sd(best) / sqrt(.N)), by = .(scenario, instance, iter)] +saveRDS(mean_results, "/gscratch/lschnei8/results_yahpo_mies_average.rds") + +library(ggplot2) +library(gridExtra) + +lm_data = copy(mean_results) +lm_data[, problem := paste0(scenario, "_", instance)] +models = map_dtr(unique(lm_data$problem), function(problem_) { + tmp = lm_data[problem == problem_] + values = tail(unique(tmp$mean_best), 2L) + dat = map_dtr(values, function(value) { + tmp[mean_best == value][.N, ] + }) + model = lm(mean_best ~ iter, data = dat) + coefs = coef(model) + if (coefs[2L] < .Machine$double.eps) { + stop("Almost constant linear model") + } + max_mean_best = max(tmp$mean_best) + max_iter = max(tmp$iter) + estimate_iter = function(mean_best, correct = TRUE) { + stopifnot(mean_best >= max_mean_best) + iter = as.integer(ceiling((mean_best - coefs[1L]) / coefs[2L])) + if (correct) { + if (mean_best > max_mean_best && iter <= max_iter) { + iter = max_iter + 1L + } + } + iter + } + env = new.env() + environment(estimate_iter) = env + assign("max_mean_best", value = max_mean_best, envir = env) + assign("max_iter", value = max_iter, envir = env) + if (estimate_iter(1.00001 * max(tmp$mean_best), correct = FALSE) < max(tmp$iter)) { + # marginal improvements should require more iter than max iter + stop("Model does not interpolate latest iter well.") + } + tmp_p = data.table(iter = ((NROW(tmp) - 1L):ceiling(1.5 * NROW(tmp)))) + p = predict(model, tmp_p) + tmp_p[, mean_best := p] + tmp_p[, method := "interpolation"] + tmp_plot = copy(tmp)[, c("iter", "mean_best")] + tmp_plot[, method := "real"] + tmp_plot = rbind(tmp_plot, tmp_p) + g = ggplot(aes(x = iter, y = mean_best, colour = method), data = tmp_plot[ceiling(0.9 * NROW(tmp)):.N , ]) + + scale_y_log10() + + geom_step(direction = "vh") + + geom_hline(yintercept = max(tmp$mean_best), linetype = 2L) + + labs(title = problem_) + + theme_minimal() + + theme(legend.position = "bottom") + info = strsplit(problem_, "_")[[1L]] + if (length(info) == 3L) { + info = c(paste0(info[1L], "_", info[2L]), info[3L]) # rbv2_ + } + list(model = list(estimate_iter), plot = list(g), scenario = info[1L], instance = info[2L]) +}) + +g = do.call("grid.arrange", c(models$plot, ncol = 8L)) + +ggsave("mies_extrapolation.png", plot = g, width = 32, height = 12) + +saveRDS(models[, - "plot"], "mies_extrapolation.rds") + diff --git a/attic/so_config/run_yahpo_trafbo.R b/attic/so_config/run_yahpo_trafbo.R new file mode 100644 index 00000000..8937555f --- /dev/null +++ b/attic/so_config/run_yahpo_trafbo.R @@ -0,0 +1,162 @@ +library(batchtools) +library(data.table) +library(mlr3) +library(mlr3learners) +library(mlr3pipelines) +library(mlr3misc) +library(mlr3mbo) # @so_config +library(bbotk) # @localsearch +library(paradox) +library(miesmuschel) # @mlr3mbo_config +library(R6) +library(checkmate) +library(trtf) + +reticulate::use_condaenv("/home/lschnei8/.conda/envs/env", required = TRUE) +library(reticulate) +yahpo_gym = import("yahpo_gym") + +packages = c("data.table", "mlr3", "mlr3learners", "mlr3pipelines", "mlr3misc", "mlr3mbo", "bbotk", "paradox", "miesmuschel", "R6", "checkmate", "trtf") + +#RhpcBLASctl::blas_set_num_threads(1L) +#RhpcBLASctl::omp_set_num_threads(1L) + +root = here::here() +experiments_dir = file.path(root) + +source_files = map_chr(c("helpers.R", "AcqFunctionLogEI", "LearnerRegrRangerCustom.R", "OptimizerChain.R", "trafbo.R"), function(x) file.path(experiments_dir, x)) +for (sf in source_files) { + source(sf) +} + +reg = makeExperimentRegistry(file.dir = "/gscratch/lschnei8/registry_mlr3mbo_so_config", packages = packages, source = source_files) +#reg = makeExperimentRegistry(file.dir = NA, conf.file = NA, packages = packages, source = source_files) # interactive session +saveRegistry(reg) + +trafbo_wrapper = function(job, data, instance, ...) { + reticulate::use_condaenv("/home/lschnei8/.conda/envs/env", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + optim_instance = make_optim_instance(instance) + + bayesopt_trafbo(optim_instance) + + optim_instance +} + +# add algorithms +addAlgorithm("trafbo", fun = trafbo_wrapper) + +# setup scenarios and instances +get_nb301_setup = function(budget_factor = 40L) { + scenario = "nb301" + bench = yahpo_gym$benchmark_set$BenchmarkSet(scenario, instance = "CIFAR10") + fidelity_space = bench$get_fidelity_space() + fidelity_param_id = fidelity_space$get_hyperparameter_names()[1] + min_budget = fidelity_space$get_hyperparameter(fidelity_param_id)$lower + max_budget = fidelity_space$get_hyperparameter(fidelity_param_id)$upper + ndim = length(bench$config_space$get_hyperparameter_names()) - 1L # NOTE: instance is not part of + + instances = "CIFAR10" + target = "val_accuracy" + budget = ceiling(20L * max_budget + sqrt(ndim) * max_budget * budget_factor) + on_integer_scale = TRUE + minimize = bench$config$config$y_minimize[match(target, bench$config$config$y_names)] + setup = setDT(expand.grid(scenario = scenario, instance = instances, target = target, ndim = ndim, max_budget = max_budget, budget = budget, on_integer_scale = on_integer_scale, minimize = minimize, stringsAsFactors = FALSE)) + setup +} + +get_lcbench_setup = function(budget_factor = 40L) { + scenario = "lcbench" + bench = yahpo_gym$benchmark_set$BenchmarkSet(scenario, instance = "167168") + fidelity_space = bench$get_fidelity_space() + fidelity_param_id = fidelity_space$get_hyperparameter_names()[1] + min_budget = fidelity_space$get_hyperparameter(fidelity_param_id)$lower + max_budget = fidelity_space$get_hyperparameter(fidelity_param_id)$upper + ndim = length(bench$config_space$get_hyperparameter_names()) - 2L + + instances = c("167168", "189873", "189906") + target = "val_accuracy" + budget = ceiling(20L * max_budget + sqrt(ndim) * max_budget * budget_factor) + on_integer_scale = TRUE + minimize = bench$config$config$y_minimize[match(target, bench$config$config$y_names)] + setup = setDT(expand.grid(scenario = scenario, instance = instances, target = target, ndim = ndim, max_budget = max_budget, budget = budget, on_integer_scale = on_integer_scale, minimize = minimize, stringsAsFactors = FALSE)) + setup +} + +get_rbv2_setup = function(budget_factor = 40L) { + setup = map_dtr(c("rbv2_glmnet", "rbv2_rpart", "rbv2_ranger", "rbv2_xgboost", "rbv2_super"), function(scenario) { + bench = yahpo_gym$benchmark_set$BenchmarkSet(scenario, instance = "1040") + fidelity_space = bench$get_fidelity_space() + fidelity_param_id = "trainsize" + min_budget = fidelity_space$get_hyperparameter(fidelity_param_id)$lower + max_budget = fidelity_space$get_hyperparameter(fidelity_param_id)$upper + ndim = length(bench$config_space$get_hyperparameter_names()) - 3L # repl and trainsize and instance + + instances = switch(scenario, rbv2_glmnet = c("375", "458"), rbv2_rpart = c("14", "40499"), rbv2_ranger = c("16", "42"), rbv2_xgboost = c("12", "1501", "16", "40499"), rbv2_super = c("1053", "1457", "1063", "1479", "15", "1468")) + target = "acc" + budget = ceiling(20L * max_budget + sqrt(ndim) * max_budget * budget_factor) + on_integer_scale = FALSE + minimize = bench$config$config$y_minimize[match(target, bench$config$config$y_names)] + setup = setDT(expand.grid(scenario = scenario, instance = instances, target = target, ndim = ndim, max_budget = max_budget, budget = budget, on_integer_scale = on_integer_scale, minimize = minimize, stringsAsFactors = FALSE)) + }) +} + +setup = rbind(get_nb301_setup(), get_lcbench_setup(), get_rbv2_setup()) + +setup[, id := seq_len(.N)] + +# add problems +prob_designs = map(seq_len(nrow(setup)), function(i) { + prob_id = paste0(setup[i, ]$scenario, "_", setup[i, ]$instance, "_", setup[i, ]$target) + addProblem(prob_id, data = list(scenario = setup[i, ]$scenario, instance = setup[i, ]$instance, target = setup[i, ]$target, ndim = setup[i, ]$ndim, max_budget = setup[i, ]$max_budget, budget = setup[i, ]$budget, on_integer_scale = setup[i, ]$on_integer_scale, minimize = setup[i, ]$minimize)) + setNames(list(setup[i, ]), nm = prob_id) +}) +nn = sapply(prob_designs, names) +prob_designs = unlist(prob_designs, recursive = FALSE, use.names = FALSE) +names(prob_designs) = nn + +# add jobs for optimizers +optimizers = data.table(algorithm = c("mlr3mbo", "trafbo", "mlr3mbo_gp")) + +for (i in seq_len(nrow(optimizers))) { + algo_designs = setNames(list(optimizers[i, ]), nm = optimizers[i, ]$algorithm) + + ids = addExperiments( + prob.designs = prob_designs, + algo.designs = algo_designs, + repls = 30L + ) + addJobTags(ids, as.character(optimizers[i, ]$algorithm)) +} + +jobs = getJobTable() +jobs = jobs[grepl("lcbench", x = problem) & tags == "trafbo", ] +resources.default = list(walltime = 3600 * 3L, memory = 2048L, ntasks = 1L, ncpus = 1L, nodes = 1L, clusters = "teton", max.concurrent.jobs = 9999L) +submitJobs(jobs, resources = resources.default) + +done = findDone() +results = reduceResultsList(done, function(x, job) { + x = x$archive$data + budget_var = if (job$instance$scenario %in% c("lcbench", "nb301")) "epoch" else "trainsize" + target_var = job$instance$target + if (!job$instance$minimize) { + x[, (target_var) := - get(target_var)] + } + pars = job$pars + tmp = x[, target_var, with = FALSE] + tmp[, (budget_var) := job$instance$max_budget] + tmp[, method := pars$algo.pars$algorithm] + tmp[, scenario := pars$prob.pars$scenario] + tmp[, instance := pars$prob.pars$instance] + tmp[, repl := job$repl] + tmp[, iter := seq_len(.N)] + colnames(tmp) = c("target", "budget", "method", "scenario", "instance", "repl", "iter") + tmp +}) +results = rbindlist(results, fill = TRUE) +saveRDS(results, "results_yahpo_trafbo.rds") + diff --git a/attic/so_config/trafbo.R b/attic/so_config/trafbo.R new file mode 100644 index 00000000..71212b33 --- /dev/null +++ b/attic/so_config/trafbo.R @@ -0,0 +1,248 @@ +# FIXME: check log EI again + +source("LearnerRegrRangerCustom.R") + +bayesopt_trafbo = function(instance, init_design_size = NULL) { + if (is.null(init_design_size)) { + init_design_size = ceiling(instance$terminator$param_set$values$n_evals / 4L) + } + design = generate_design_sobol(instance$search_space, init_design_size)$data + instance$eval_batch(design) + repeat { + if (runif(1) <= 0.1) { + candidate = generate_design_random(instance$search_space, 1L)$data + instance$eval_batch(candidate) + } else { + # FIXME: adapt for arbitrary target, optim direction and search space + dat = copy(instance$archive$data[, c(instance$archive$cols_x, instance$archive$cols_y), with = FALSE]) + fix_NA_and_chars(dat) + max_to_min = instance$archive$codomain$tags[[instance$archive$cols_y]] == "maximize" + + y = dat[[instance$archive$cols_y]] + if (max_to_min) y = - y + min_y = min(y) - 0.05 * diff(range(y)) + max_y = max(y) + y_log = log((y - min_y) / (max_y - min_y), base = 10L) + + lower = min(y_log) + upper = max(y_log) + support = c(lower - 0.2 * (upper - lower), upper + 0.2 * (upper - lower)) + #support = c(pmin(support[1L], exp(support[1L])), pmax(support[2L], exp(support[2L]))) # density transformation FIXME: exp() + + dat$y_log = y_log + + y_var = numeric_var("y_log", support = support, bounds = support) + y_bern = Bernstein_basis(y_var, order = 8, ui = "increasing") + + ctmm = ctm(response = y_bern, todistr = "Normal") + ctrl = ctree_control(minbucket = 10L, alpha = 0.05, maxdepth = Inf) + tree = trafotree(ctmm, formula = as.formula(paste0("y_log ~ ", paste0(instance$archive$cols_x, collapse = " + "))), data = dat, control = ctrl, min_update = 1L) + plot(tree, tp_args = list(type = "density", id = FALSE, ylines = 0, K = 200)) + + q = seq(support[1L], lower, length.out = 1001L) + #imp = pmax(lower - q, 0) + imp = pmax(exp(lower) - exp(q), 0) # density transformation + + nodes = nodeids(tree, terminal = TRUE) + + xdt = copy(dat) + xdt[, .node := predict(tree, dat[, instance$archive$cols_x, with = FALSE])] + + if (length(nodes) > 1L) { + # actually log EI + eis = map_dbl(nodes, function(node) { + xdt_node = xdt[.node == node, ][, instance$archive$cols_x, with = FALSE][1L, ] + d = predict(tree, xdt_node, type = "density", q = q)[, 1L] + d = d * abs((1 / exp(q))) # density transformation FIXME: exp + + ei = sum(d * rev(diff(rev(c(imp, 0)))), na.rm = TRUE) + if (ei < sqrt(.Machine$double.eps)) ei = 0 + ei + }) + names(eis) = nodes + + best_node = as.numeric(names(shuffle(which(eis == max(eis)))[1L])) # random tie breaks which can occur due to min_update > minbucket + + # FIXME: we could also proceed to evaluate every other second best node + + tmp = dat[predict(tree, dat[, instance$archive$cols_x, with = FALSE]) == best_node, ][, c("y_log", instance$archive$cols_x), with = FALSE] + tmp_ss = create_tmp_ss(instance$search_space, tree = tree, node = best_node) + } else { + tmp = dat[, c("y_log", instance$archive$cols_x), with = FALSE] + tmp_ss = instance$search_space$clone(deep = TRUE) + } + + task = TaskRegr$new("inner", target = "y_log", backend = tmp) + + ranger = LearnerRegrRangerCustom$new() + ranger$param_set$values$se.method = "simple" + ranger$param_set$values$splitrule = "extratrees" + ranger$param_set$values$num.random.splits = 1L + ranger$param_set$values$num.trees = 10L + ranger$param_set$values$replace = TRUE + ranger$param_set$values$sample.fraction = 1 + ranger$param_set$values$min.node.size = 1 + ranger$param_set$values$mtry.ratio = 1 + learner = GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor", multiplier = 3) %>>% ranger) + learner$predict_type = "se" + learner$train(task) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + if (runif(1) <= 0.5) { + f_instance = make_greedy_f_instance(tmp_ss, learner = learner, n_evals = 10010L, lower = lower) + best = copy(instance$archive$data) + best_tmp = best[, instance$archive$cols_x, with = FALSE] + fix_NA_and_chars(best_tmp) + if (length(nodes) > 1L) { + best[, .node := predict(tree, best_tmp)] + best = best[.node == best_node, ] + } + if (max_to_min) { + setorderv(best, col = instance$archive$cols_y, order = -1L) + } else { + setorderv(best, col = instance$archive$cols_y, order = 1L) + } + # FIXME: actually, we should evaluate all in the archive and then choose the 10 best within Local Search + best = best[1:min(10, nrow(best)), instance$archive$cols_x, with = FALSE] + f_instance$eval_batch(best) + opt("local_search", n_points = 100L)$optimize(f_instance) + f_instance$terminator$param_set$values$n_evals = 20010L + opt("random_search", batch_size = 10000L)$optimize(f_instance) + } else { + f_instance = make_greedy_f_instance(tmp_ss, learner = learner, n_evals = 20000L, lower = lower) + opt("random_search", batch_size = 10000L)$optimize(f_instance) + } + if ("x_domain" %nin% colnames(f_instance$archive$data)) { + f_instance$archive$data[, x_domain := list()] # FIXME: why is this necessary + } + + # FIXME: we want to make sure we evaluate only within the best node + #if (length(nodes) > 1L) { + # tmp = f_instance$archive$data[, f_instance$archive$cols_x, with = FALSE] + # fix_NA_and_chars(tmp) + # f_instance$archive$data = f_instance$archive$data[predict(tree, tmp) == best_node, ] + #} + + candidate = mlr3mbo:::get_best_not_evaluated(f_instance, evaluated = instance$archive$data) + logger$set_threshold("info") + instance$eval_batch(candidate[, instance$archive$cols_x, with = FALSE]) + } + if (instance$is_terminated) break + } +} + +create_tmp_ss = function(ss, tree, node) { + tmp_ss = ss$clone(deep = TRUE) + splits = partykit:::.list.rules.party(tree)[[as.character(node)]] + split_info = strsplit(splits, " & ")[[1L]] + + larger = which(sapply(split_info, grepl, pattern = ">")) + smaller = which(sapply(split_info, grepl, pattern = "<=")) + subset = which(sapply(split_info, grepl, pattern = "%in%")) + + split_info = strsplit(split_info, " > | <= | %in%") + + for (i in larger) { + if (tmp_ss$params[[split_info[[i]][[1]]]]$class == "ParamDbl") { + tmp_ss$params[[split_info[[i]][1]]]$lower = as.numeric(split_info[[i]][2]) + } else if (tmp_ss$params[[split_info[[i]][[1]]]]$class == "ParamInt") { + tmp_ss$params[[split_info[[i]][1]]]$lower = min(as.integer(split_info[[i]][2]) + 1L, tmp_ss$params[[split_info[[i]][1]]]$upper) + } + } + + for (i in smaller) { + tmp_ss$params[[split_info[[i]][1]]]$upper = as.numeric(split_info[[i]][2]) + } + + for (i in subset) { + new_levels = unlist(map(tmp_ss$params[[split_info[[i]][1]]]$levels, function(pattern) if (grepl(pattern, x = split_info[[i]][2])) pattern else NULL)) + new_levels = setdiff(new_levels, c(".MISSING")) # FIXME: depends on imputeoor + if (length(new_levels) > 0L) { + tmp_ss$params[[split_info[[i]][1]]]$levels = new_levels + } + } + + tmp_ss +} + +## Mean +#make_greedy_f_instance = function(ss, learner, n_evals, lower) { +# ss = ss$clone(deep = TRUE) +# ss$trafo = NULL +# f = ObjectiveRFunDt$new( +# fun = function(xdt) data.table(mean = learner$predict_newdata(fix_NA_and_chars(xdt))$response), +# domain = ss, +# codomain = ps(mean = p_dbl(tags = "minimize")) +# ) +# f_instance = OptimInstanceSingleCrit$new( +# objective = f, +# search_space = ss, +# terminator = trm("evals", n_evals = n_evals) +# ) +# f_instance +#} + +## EI +#make_greedy_f_instance = function(ss, learner, n_evals, lower) { +# ss = ss$clone(deep = TRUE) +# ss$trafo = NULL +# f = ObjectiveRFunDt$new( +# fun = function(xdt) { +# p = learner$predict_newdata(fix_NA_and_chars(xdt)) +# mu = p$response +# se = p$se +# d = lower - mu +# d_norm = d / se +# ei = d * pnorm(d_norm) + se * dnorm(d_norm) +# ei = ifelse(se < 1e-20, 0, ei) +# data.table(acq_ei = ei) +# }, +# domain = ss, +# codomain = ps(acq_ei = p_dbl(tags = "maximize")) +# ) +# f_instance = OptimInstanceSingleCrit$new( +# objective = f, +# search_space = ss, +# terminator = trm("evals", n_evals = n_evals) +# ) +# f_instance +#} + +# Log EI +make_greedy_f_instance = function(ss, learner, n_evals, lower) { + ss = ss$clone(deep = TRUE) + ss$trafo = NULL + f = ObjectiveRFunDt$new( + fun = function(xdt) { + p = learner$predict_newdata(fix_NA_and_chars(xdt)) + mu = p$response + se = p$se + d = lower - mu + d_norm = d / se + ei = (exp(lower) * pnorm(d_norm)) - (exp((0.5 * se ^ 2) + mu) * pnorm(d_norm - se)) + ei = ifelse(se < 1e-20, 0, ei) + data.table(acq_log_ei = ei) + }, + domain = ss, + codomain = ps(acq_log_ei = p_dbl(tags = "maximize")) + ) + f_instance = OptimInstanceSingleCrit$new( + objective = f, + search_space = ss, + terminator = trm("evals", n_evals = n_evals) + ) + f_instance +} + +fix_NA_and_chars = function(xydt) { + chr_cols = names(xydt)[map_chr(xydt, class) == "character"] + if (length(chr_cols)) { + xydt[, `:=`((chr_cols), map(.SD, as_factor_NA_fix)), .SDcols = chr_cols] + } + xydt +} + +as_factor_NA_fix = function(x) { + x[is.na(x)] = ".MISSING" + as.factor(x) +} diff --git a/attic/so_config/trafbo_old.R b/attic/so_config/trafbo_old.R new file mode 100644 index 00000000..23f6e9f6 --- /dev/null +++ b/attic/so_config/trafbo_old.R @@ -0,0 +1,123 @@ +bayesopt_trafbo = function(instance, init_design_size = NULL) { + if (is.null(init_design_size)) { + init_design_size = ceiling(instance$terminator$param_set$values$n_evals / 4L) + } + design = generate_design_random(instance$search_space, init_design_size)$data + instance$eval_batch(design) + repeat { + if (runif(1) <= 0.25) { + candidate = generate_design_random(instance$search_space, 1L)$data + instance$eval_batch(candidate) + } else { + # FIXME: adapt for arbitrary target, optim direction and search space + dat = copy(instance$archive$data[, c(instance$archive$cols_x, instance$archive$cols_y), with = FALSE]) + + y = - dat$val_accuracy + min_y = min(y) - 0.05 * diff(range(y)) + max_y = max(y) + y_log = log((y - min_y) / (max_y - min_y)) + + lower = min(y_log) + upper = max(y_log) + support = c(lower - 0.05 * (upper - lower), upper + 0.05 * (upper - lower)) + + dat$val_accuracy = y_log + + y = numeric_var("val_accuracy", support = support, bounds = support) + yy = Bernstein_basis(y, order = 4, ui = "increasing") + + ctmm = ctm(response = yy, todistr = "Normal") + ctrl = ctree_control(minbucket = 10L, alpha = 0.05, maxdepth = 10L) + tree = trafotree(ctmm, formula = val_accuracy ~ ., data = dat, control = ctrl, min_update = 10L) + plot(tree, tp_args = list(type = "density", id = FALSE, ylines = 0, K = 200)) + + q = seq(support[1L], support[2L], length.out = 10001L) + imp = pmax(lower - q, 0) + + nodes = nodeids(tree, terminal = TRUE) + + if (length(nodes) > 1L) { + eis = map_dbl(nodes, function(node) { + xdt = generate_design_random(create_tmp_ss(instance$search_space, tree = tree, node = node), 100L)$data + xdt[predict(tree, xdt) == node, ][1L, ] + d = predict(tree, xdt, type = "density", q = q)[, 1L] + ei = sum(d / sum(d, na.rm = TRUE) * imp, na.rm = TRUE) + if (ei < sqrt(.Machine$double.eps)) ei = 0 + ei + }) + + best_node = nodes[which.max(eis)] + tmp = dat[predict(tree, dat) == best_node, ] + tmp_ss = create_tmp_ss(instance$search_space, tree = tree, node = best_node) + } else { + tmp = dat + tmp_ss = instance$search_space$clone(deep = TRUE) + } + + task = TaskRegr$new("inner", target = "val_accuracy", backend = tmp) + learner = as_learner(po("imputeoor", multiplier = 10L) %>>% lrn("regr.ranger", splitrule = "extratrees", num.trees = 1000L)) + learner$train(task) + f_instance = make_greedy_f_instance(tmp_ss, learner = learner, n_evals = 10010L) + setorderv(tmp, col = "val_accuracy") + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + f_instance$eval_batch(tmp[1:10, ]) + opt("local_search")$optimize(f_instance) + f_instance$archive$data = f_instance$archive$data[11:10010, ] + f_instance$terminator$param_set$values$n_evals = 20000L + opt("random_search", batch_size = 10000L)$optimize(f_instance) + f_instance$archive$data[, x_domain := list()] # FIXME: needed due to f_instance ObjectiveRFunDt + + if (length(nodes) > 1L) { + f_instance$archive$data = f_instance$archive$data[predict(tree, f_instance$archive$data[, f_instance$archive$cols_x, with = FALSE]) == best_node, ] + } + + candidate = mlr3mbo:::get_best_not_evaluated(f_instance, evaluated = instance$archive$data) + logger$set_threshold("info") + instance$eval_batch(candidate[, instance$archive$cols_x, with = FALSE]) + } + if (instance$is_terminated) break + } +} + +# FIXME: fails with everything not numeric/integer +create_tmp_ss = function(ss, tree, node) { + tmp_ss = ss$clone(deep = TRUE) + splits = partykit:::.list.rules.party(tree)[[as.character(node)]] + split_info = strsplit(splits, " & ")[[1L]] + + larger = which(sapply(split_info, grepl, pattern = ">")) + smaller = which(sapply(split_info, grepl, pattern = "<=")) + + split_info = strsplit(split_info, " > | <= ") + + for (i in larger) { + if (tmp_ss$params[[split_info[[i]][[1]]]]$class == "ParamDbl") { + tmp_ss$params[[split_info[[i]][1]]]$lower = as.numeric(split_info[[i]][2]) + } else if (tmp_ss$params[[split_info[[i]][[1]]]]$class == "ParamInt") { + tmp_ss$params[[split_info[[i]][1]]]$lower = min(as.integer(split_info[[i]][2]) + 1L, tmp_ss$params[[split_info[[i]][1]]]$upper) + } + } + + for (i in smaller) { + tmp_ss$params[[split_info[[i]][1]]]$upper = as.numeric(split_info[[i]][2]) + } + + tmp_ss +} + +make_greedy_f_instance = function(ss, learner, n_evals) { + ss = ss$clone(deep = TRUE) + ss$trafo = NULL + f = ObjectiveRFunDt$new( + fun = function(xdt) data.table(mean = learner$predict_newdata(xdt)$response), + domain = ss, + codomain = ps(mean = p_dbl(tags = "minimize")) + ) + f_instance = OptimInstanceSingleCrit$new( + objective = f, + search_space = ss, + terminator = trm("evals", n_evals = n_evals) + ) + f_instance +} diff --git a/attic/so_config_old/OptimizerChain.R b/attic/so_config_old/OptimizerChain.R new file mode 100644 index 00000000..4f04a373 --- /dev/null +++ b/attic/so_config_old/OptimizerChain.R @@ -0,0 +1,83 @@ +OptimizerChain = R6Class("OptimizerChain", inherit = bbotk::Optimizer, + public = list( + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + #' + #' @param optimizers (list of [Optimizer]s). + #' @param terminators (list of [Terminator]s | NULL). + initialize = function(optimizers, terminators = rep(list(NULL), length(optimizers))) { + assert_list(optimizers, types = "Optimizer", any.missing = FALSE) + assert_list(terminators, types = c("Terminator", "NULL"), len = length(optimizers)) + + param_sets = vector(mode = "list", length = length(optimizers)) + ids_taken = character(0L) + # for each optimizer check whether the id of the param_set + # (decuded from the optimizer class) is already taken; + # if necessary postfix the id + for (i_opt in seq_along(optimizers)) { + opt = optimizers[[i_opt]] + ps = opt$param_set$clone(deep = TRUE) + ps$set_id = class(opt)[[1L]] + try_postfix = 0L + while (ps$set_id %in% ids_taken) { + try_postfix = try_postfix + 1L + ps$set_id = paste0(class(opt)[[1L]], "_", try_postfix) + } + ids_taken[[i_opt]] = ps$set_id + param_sets[[i_opt]] = ps + } + private$.ids = map_chr(param_sets, "set_id") + super$initialize( + param_set = ParamSetCollection$new(param_sets), + param_classes = Reduce(intersect, mlr3misc::map(optimizers, "param_classes")), + properties = Reduce(intersect, mlr3misc::map(optimizers, "properties")), + packages = unique(unlist(mlr3misc::map(optimizers, "packages"))) + ) + private$.optimizers = optimizers + private$.terminators = terminators + } + ), + + private = list( + .optimizers = NULL, + .terminators = NULL, + .ids = NULL, + + .optimize = function(inst) { + terminator = inst$terminator + on.exit({inst$terminator = terminator}) + inner_inst = inst$clone(deep = TRUE) + + for (i_opt in seq_along(private$.optimizers)) { + inner_term = private$.terminators[[i_opt]] + if (!is.null(inner_term)) { + inner_inst$terminator = TerminatorCombo$new(list(inner_term, terminator)) + } else { + inner_inst$terminator = terminator + } + opt = private$.optimizers[[i_opt]] + opt$param_set$values = self$param_set$.__enclos_env__$private$.sets[[i_opt]]$values + opt$optimize(inner_inst) + inner_inst$archive$data$batch_nr = max(inst$archive$data$batch_nr, 0L) + + inner_inst$archive$data$batch_nr + inner_inst$archive$data$optimizer = private$.ids[i_opt] + inst$archive$data = rbind(inst$archive$data, inner_inst$archive$data, fill = TRUE) + inner_inst$archive$data = data.table() + if (terminator$is_terminated(inst$archive)) { + break + } + } + }, + + deep_clone = function(name, value) { + switch( + name, + .optimizers = mlr3misc::map(value, .f = function(x) x$clone(deep = TRUE)), + .terminators = mlr3misc::map(value, .f = function(x) if (!is.null(x)) x$clone(deep = TRUE)), + value + ) + } + ) +) + diff --git a/attic/so_config_old/analyze.R b/attic/so_config_old/analyze.R new file mode 100644 index 00000000..3fe51370 --- /dev/null +++ b/attic/so_config_old/analyze.R @@ -0,0 +1,57 @@ +library(data.table) +library(mlr3) +library(mlr3learners) +library(mlr3pipelines) +library(mlr3mbo) +library(ggplot2) +library(GGally) + +x1 = readRDS("ac_instance_1.rds") +x2 = readRDS("ac_instance_2.rds") +x3 = readRDS("ac_instance_3.rds") +x4 = readRDS("ac_instance_4.rds") # FIXME: currently only 166/230 evals +x5 = readRDS("ac_instance_5.rds") + +data = rbind(x1$archive$data, x2$archive$data, x3$archive$data, x4$archive$data, x5$archive$data)[, c(x1$archive$cols_x, x1$archive$cols_y), with = FALSE] +data[is.na(random_interleave_iter), random_interleave_iter := 0] +data[is.na(num.random.splits), num.random.splits := 0] +data[is.na(lambda), lambda := 0] +data[is.na(fs_behavior), fs_behavior := "none"] +chars = c("init", "splitrule", "acqf", "acqopt", "fs_behavior") +data[, (chars) := lapply(.SD, as.factor), .SDcols = chars] + +lmx = lm(mean_perf ~ init * init_size_factor + random_interleave * random_interleave_iter + num.trees + splitrule * num.random.splits + acqf * lambda + acqopt_iter_factor * acqopt * fs_behavior, data = data) + +task = TaskRegr$new("mbo", backend = data, target = "mean_perf") +learner = default_surrogate(x1)$model +learner$param_set$values$regr.ranger.importance = "permutation" +learner$param_set$values$regr.ranger.num.trees = 2000L + +rr = resample(task, learner, rsmp("cv", folds = 10L)) +rr$aggregate(msr("regr.rsq")) +rr$aggregate(msr("regr.ktau")) + +learner$train(task) +p = learner$predict(task) +task$data()[which.max(p$response), ] + +imp = ranger::importance(learner$model$regr.ranger$model) +imp = data.table(importance = imp, feature = names(imp)) +ggplot(aes(x = feature, y = importance), data = imp) + + geom_bar(stat = "identity") + +data[, mean_perf := p$response] +top5 = quantile(data$mean_perf, 0.95) +data[, top5 := as.factor(mean_perf >= top5)] + +best = data[top5 == "TRUE", - "top5"] +best[, init := as.factor(init)] +best[, splitrule := as.factor(splitrule)] +best[, acqf := as.factor(acqf)] +best[, acqopt := as.factor(acqopt)] +best[, fs_behavior := as.factor(fs_behavior)] + +summary(best[random_interleave == FALSE & acqopt == "FS" & fs_behavior == "global"]) + +# we go with +# init = lhs, init_size_factor = 6, random_interleave = FALSE, num.trees = 250, splitrule = extratrees, num.random.splits = 8, acqf = CB, lambda = 2.8, acqopt_iter_factor = 6, acqopt = FS, fs_behavior = global diff --git a/attic/so_config_old/analyze_glmnet_458.R b/attic/so_config_old/analyze_glmnet_458.R new file mode 100644 index 00000000..008e6bf4 --- /dev/null +++ b/attic/so_config_old/analyze_glmnet_458.R @@ -0,0 +1,204 @@ +library(mlr3) +library(mlr3learners) +library(mlr3pipelines) +library(mlr3mbo) + +set.seed(1) + +reticulate::use_virtualenv("/home/lps/.local/share/virtualenvs/yahpo_gym-4ygV7ggv/", required = TRUE) +source("helpers.R") +source("OptimizerChain.R") +library(yahpogym) +library(bbotk) +instance = make_optim_instance(data.table(scenario = "rbv2_glmnet", instance = "458", target = "acc", ndi = 3L, max_budget = 1, budget = 10^6, on_integer_scale = FALSE, minimize = FALSE)) +xdt = generate_design_grid(instance$search_space, resolution = 100)$data +instance$eval_batch(xdt) +ref = copy(instance$archive$data) +breaks = quantile(ref$acc, c(0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 0.99, 0.999, 1)) + +instance = make_optim_instance(data.table(scenario = "rbv2_glmnet", instance = "458", target = "acc", ndi = 3L, max_budget = 1, budget = 90L, on_integer_scale = FALSE, minimize = FALSE)) +d = instance$search_space$length +init_design_size = 4L * d +init_design = generate_design_lhs(instance$search_space, n = init_design_size)$data +instance$eval_batch(init_design) + +random_interleave_iter = 0L + +learner = lrn("regr.ranger_custom", num.trees = 10L, mtry.ratio = 1) +surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor", multiplier = 10) %>>% learner)) + +acq_function = AcqFunctionEI$new() + +optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) +acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20010L)) + +acq_optimizer$param_set$values$warmstart = TRUE +acq_optimizer$param_set$values$warmstart_size = "all" + +bayesopt_ego(instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + +mbo = copy(instance$archive$data)[, c("alpha", "num.impute.selected.cpo", "s", "acc")] +mbo[, method := "mbo"] +mbo[, repl := 1] +mbo[, iter := seq_len(.N)] + +instance$archive$clear() +instance$eval_batch(init_design) + +bayesopt_ego_log(instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + +mbo_log = copy(instance$archive$data)[, c("alpha", "num.impute.selected.cpo", "s", "acc")] +mbo_log[, method := "mbo_log"] +mbo_log[, repl := 1] +mbo_log[, iter := seq_len(.N)] + +glmnet_458_smac = readRDS("rbv2_glmnet_458_smac.rds")[repl == 2] +glmnet_458_smac[, method := "smac4hpo"] +glmnet_458_smac[, s := log(s)] + +glmnet_458_mbo = readRDS("rbv2_glmnet_458_mlr3mbo_new_rf.rds")[repl == 1] +glmnet_458_mbo[, method := "mlr3mbo_new_rf"] + +glmnet_458 = rbind(glmnet_458_smac, glmnet_458_mbo, mbo, mbo_log, fill = TRUE) + +glmnet_458_best = rbind(glmnet_458[method == "smac4hpo"][which.max(acc), ], + glmnet_458[method == "mlr3mbo_new_rf"][which.max(acc), ], + glmnet_458[method == "mbo"][which.max(acc), ], + glmnet_458[method == "mbo_log"][which.max(acc), ]) + +g1 = ggplot(data = ref, aes(x = alpha, y = s, z = acc)) + + geom_point(aes(x = alpha, y = s, colour = iter, shape = method), data = glmnet_458[method %in% c("smac4hpo", "mbo", "mbo_log")], size = 3) + + geom_contour(breaks = breaks) + + geom_contour_filled(breaks = breaks, alpha = 0.1) + + facet_grid(method ~ num.impute.selected.cpo) + + theme(legend.position = "bottom") + +g2 = ggplot(data = glmnet_458[method %in% c("smac4hpo", "mbo", "mbo_log") & iter > 12], aes(x = iter, y = cummax(acc), colour = method)) + + geom_step() + +instance_x = instance$clone(deep = TRUE) +instance$archive$data = glmnet_458[method == "mbo_log"] + +learner = lrn("regr.ranger", mtry.ratio = 1, num.trees = 10L, se.method = "jack", min.node.size = 1L, splitrule = "extratrees", num.random.splits = 1L) +surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% learner)) +acq_function = AcqFunctionEI$new() +surrogate$archive = instance$archive +acq_function$surrogate = surrogate + +surrogate$y_cols = "y_trafo" +acq_function$surrogate_max_to_min = 1 + +y = instance$archive$data[[instance$archive$cols_y]] * instance$objective_multiplicator +min_y = min(y) - 0.01 * diff(range(y)) +max_y = max(y) +y_log = log((y - min_y) / (max_y - min_y)) +instance$archive$data[, y_trafo := y_log] +acq_function$surrogate$update() +acq_function$y_best = min(y_log) # manual update + + +acq_instance = OptimInstanceSingleCrit$new(objective = acq_function, search_space = acq_function$domain, terminator = trm("none"), check_values = FALSE, keep_evals = "all") +acq_instance$eval_batch(xdt) + +sp = surrogate$predict(xdt) +acq_data = copy(acq_instance$archive$data) +acq_data = cbind(acq_data, sp) +acq_data[which.max(acq_ei), ] + +acq_instance$archive$clear() + +optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) +acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20010L)) + +acq_optimizer$acq_function = acq_function +candidate = acq_optimizer$optimize() +acq_best = acq_data[which.max(acq_ei), ] + +g3 = ggplot(data = acq_data, aes(x = alpha, y = s, z = mean)) + + geom_contour() + + geom_contour_filled() + + facet_wrap(~ num.impute.selected.cpo) + + theme(legend.position = "bottom") + +g4 = ggplot(data = acq_data, aes(x = alpha, y = s, z = se)) + + geom_contour() + + geom_contour_filled() + + facet_wrap(~ num.impute.selected.cpo) + + theme(legend.position = "bottom") + +g5 = ggplot(data = acq_data, aes(x = alpha, y = s, z = acq_ei)) + + geom_contour() + + geom_contour_filled() + + geom_point(data = candidate, colour = "red", size = 2) + + geom_point(data = acq_best, colour = "white", size = 2) + + facet_wrap(~ num.impute.selected.cpo) + + theme(legend.position = "bottom") + +ggsave("/tmp/lol.png", plot = g5, width = 12, height = 3) + +#library(mlr3) +#library(mlr3learners) +#library(mlr3pipelines) +#library(mlr3viz) +# +#tdat = readRDS("glmnet_458_mbo_data.rds") +# +#tdat[, num.impute.selected.cpo := as.factor(num.impute.selected.cpo)] +#task = TaskRegr$new("test", backend = tdat, target = "acc") +#resampling = rsmp("repeated_cv", folds = 10L)$instantiate(task) +#gp = as_learner(po("encode") %>>% po("fixfactors") %>>% lrn("regr.km", covtype = "matern3_2", optim.method = "gen")) +#gp2 = as_learner(po("encodeimpact") %>>% po("fixfactors") %>>% lrn("regr.km", covtype = "matern3_2", optim.method = "gen")) +#rp = as_learner(po("fixfactors") %>>% lrn("regr.rpart")) +#rf = as_learner(po("fixfactors") %>>% lrn("regr.ranger")) +#lm = as_learner(po("fixfactors") %>>% lrn("regr.lm")) +#kknn = as_learner(po("fixfactors") %>>% lrn("regr.kknn")) +#kknn1 = as_learner(po("fixfactors") %>>% lrn("regr.kknn", k = 1)) +#kknn1$id = "kknn1" +#mars = as_learner(po("encode") %>>% po("fixfactors") %>>% lrn("regr.mars")) +#gam = as_learner(po("encode") %>>% po("fixfactors") %>>% lrn("regr.gam")) +#lm = as_learner(po("fixfactors") %>>% lrn("regr.lm")) +#learners = list(learner, gp, gp2, rp, rf, kknn, mars, gam, lm) +#bg = benchmark_grid(task, learners, resampling) +#b = benchmark(bg) +#scores = b$aggregate(msrs(c("regr.mse", "regr.mae", "regr.rsq", "regr.ktau"))) +#scores_long = melt(scores, id.vars = "learner_id", measure.vars = c("regr.mse", "regr.mae", "regr.rsq", "regr.ktau")) +# +#ggplot(aes(x = learner_id, y = value), data = scores_long) + +# geom_point(size = 3) + +# facet_wrap(~ variable, scales = "free") + +# theme_minimal() + +# theme(axis.text.x = element_text(angle = 45, vjust = 0.5, hjust = 1)) +# +#breaks = quantile(task$data()$acc, c(seq(0, 1, by = 0.1))) +#weights = as.numeric(cut(tdat$acc, breaks)) +#weights[is.na(weights)] = 1 +#task2 = TaskRegr$new("test", backend = cbind(tdat, weights), target = "acc") +#task2$col_roles$weight = "weights" +#task2$col_roles$feature = setdiff(task2$col_roles$feature, "weights") +#learners = list(rf, kknn, kknn1, lm, gp) +# +#map(learners, function(l) { +# l$train(task2) +#}) +# +#diffs = map_dtr(1:100, function(r) { +# print(r) +# map_dtr(c(2^(5:13)), function(k) { +# xdt_sample = ref[sample(.N, size = k, replace = FALSE), ] +# best = xdt_sample[which.max(acc), ] +# preds = map_dbl(learners, function(l) { +# which.max(l$predict_newdata(xdt_sample)$response) +# }) +# xdt_learner_p_best = xdt_sample[preds, ] +# data.table(diff = best$acc - xdt_learner_p_best$acc, k = k, learner = map_chr(learners, "id"), repl = r) +# }) +#}) +# +#mean_diffs = diffs[, .(mean_diff = mean(diff), se_diff = sd(diff) / sqrt(.N)), by = .(k, learner)] +# +#ggplot(aes(x = k, y = mean_diff, colour = learner), data = mean_diffs) + +# geom_point() + +# geom_errorbar(aes(ymin = mean_diff - se_diff, ymax = mean_diff + se_diff)) + +# geom_line() + +# theme_minimal() + diff --git a/attic/so_config_old/analyze_yahpo.R b/attic/so_config_old/analyze_yahpo.R new file mode 100644 index 00000000..4c8b7b8f --- /dev/null +++ b/attic/so_config_old/analyze_yahpo.R @@ -0,0 +1,92 @@ +library(data.table) +library(ggplot2) +library(pammtools) +library(mlr3misc) + +#dat = rbind(readRDS("results_yahpo.rds"), readRDS("results_yahpo_own.rds"))[method != "mlrintermbo"] +dat = rbind(readRDS("results_yahpo.rds"), readRDS("results_yahpo_own.rds"))[method %in% c("mlr3mbo", "mlrintermbo", "mlr3mbo_new_rf", "mlr3mbo_new_rf_ls", "mlr3mbo_kknn_ls", "smac4hpo", "random")] +dat = dat[scenario %nin% c("nb301", "rbv2_super")] +dat[, cumbudget := cumsum(budget), by = .(method, scenario, instance, repl)] +dat[, cumbudget_scaled := cumbudget / max(cumbudget), by = .(method, scenario, instance, repl)] +dat[, normalized_regret := (target - min(target)) / (max(target) - min(target)), by = .(scenario, instance)] +dat[, incumbent := cummin(normalized_regret), by = .(method, scenario, instance, repl)] + +get_incumbent_cumbudget = function(incumbent, cumbudget_scaled) { + budgets = seq(0, 1, length.out = 101) + map_dbl(budgets, function(budget) { + ind = which(cumbudget_scaled <= budget) + if (length(ind) == 0L) { + max(incumbent) + } else { + min(incumbent[ind]) + } + }) +} + +dat_budget = dat[, .(incumbent_budget = get_incumbent_cumbudget(incumbent, cumbudget_scaled), cumbudget_scaled = seq(0, 1, length.out = 101)), by = .(method, scenario, instance, repl)] + +agg_budget = dat_budget[, .(mean = mean(incumbent_budget), se = sd(incumbent_budget) / sqrt(.N)), by = .(cumbudget_scaled, method, scenario, instance)] +#agg_budget[, method := factor(method, levels = c("random", "smac4hpo", "hb", "bohb", "dehb", "smac4mf", "optuna", "mlr3mbo", "mlr3mbo_default"), labels = c("Random", "SMAC", "HB", "BOHB", "DEHB", "SMAC-HB", "optuna", "mlr3mbo", "mlr3mbo_default"))] + +g = ggplot(aes(x = cumbudget_scaled, y = mean, colour = method, fill = method), data = agg_budget[cumbudget_scaled > 0.10]) + + scale_y_log10() + + geom_step() + + geom_stepribbon(aes(min = mean - se, max = mean + se), colour = NA, alpha = 0.3) + + labs(x = "Fraction of Budget Used", y = "Mean Normalized Regret", colour = "Optimizer", fill = "Optimizer") + + facet_wrap(~ scenario + instance, scales = "free", ncol = 5) + + theme_minimal() + + theme(legend.position = "bottom", legend.title = element_text(size = rel(0.75)), legend.text = element_text(size = rel(0.5))) +ggsave("anytime.pdf", plot = g, device = "pdf", width = 15, height = 10) + +overall_budget = agg_budget[, .(mean = mean(mean), se = sd(mean) / sqrt(.N)), by = .(method, cumbudget_scaled)] + +g = ggplot(aes(x = cumbudget_scaled, y = mean, colour = method, fill = method), data = overall_budget[cumbudget_scaled > 0.10]) + + scale_y_log10() + + geom_step() + + geom_stepribbon(aes(min = mean - se, max = mean + se), colour = NA, alpha = 0.1) + + labs(x = "Fraction of Budget Used", y = "Mean Normalized Regret", colour = "Optimizer", fill = "Optimizer") + + theme_minimal() + + theme(legend.position = "bottom", legend.title = element_text(size = rel(0.75)), legend.text = element_text(size = rel(0.75))) +ggsave("anytime_average.pdf", plot = g, device = "pdf", width = 6, height = 4) + +methods = unique(agg_budget$method) +ranks = map_dtr(unique(agg_budget$scenario), function(scenario_) { + map_dtr(unique(agg_budget$instance), function(instance_) { + map_dtr(unique(agg_budget$cumbudget_scaled), function(cumbudget_scaled_) { + res = agg_budget[scenario == scenario_ & instance == instance_ & cumbudget_scaled == cumbudget_scaled_] + if (nrow(res) == 0L) { + return(data.table()) + } + setorderv(res, "mean") + data.table(rank = match(methods, res$method), method = methods, scenario = scenario_, instance = instance_, cumbudget_scaled = cumbudget_scaled_) + }) + }) +}) + +ranks_overall = ranks[, .(mean = mean(rank), se = sd(rank) / sqrt(.N)), by = .(method, cumbudget_scaled)] + +g = ggplot(aes(x = cumbudget_scaled, y = mean, colour = method, fill = method), data = ranks_overall[cumbudget_scaled > 0.10]) + + geom_line() + + geom_ribbon(aes(min = mean - se, max = mean + se), colour = NA, alpha = 0.3) + + labs(x = "Fraction of Budget Used", y = "Mean Rank", colour = "Optimizer", fill = "Optimizer") + + theme_minimal() + + theme(legend.position = "bottom", legend.title = element_text(size = rel(0.75)), legend.text = element_text(size = rel(0.75))) +ggsave("anytime_average_rank.pdf", plot = g, device = "pdf", width = 6, height = 4) + +library(scmamp) +best_agg = agg_budget[cumbudget_scaled == 0.25] +best_agg[, problem := paste0(scenario, "_", instance)] +tmp = - as.matrix(dcast(best_agg, problem ~ method, value.var = "mean")[, -1]) +friedmanTest(tmp) +pdf("cd_025_mf.pdf", width = 6, height = 4, pointsize = 10) +plotCD(tmp, cex = 1) +dev.off() + +best_agg = agg_budget[cumbudget_scaled == 1] +best_agg[, problem := paste0(scenario, "_", instance)] +tmp = - as.matrix(dcast(best_agg, problem ~ method, value.var = "mean")[, -1]) +friedmanTest(tmp) +pdf("cd_1_mf.pdf", width = 6, height = 4, pointsize = 10) +plotCD(tmp, cex = 1) +dev.off() + diff --git a/attic/so_config_old/coordinate_descent_old.R b/attic/so_config_old/coordinate_descent_old.R new file mode 100644 index 00000000..a9d212e4 --- /dev/null +++ b/attic/so_config_old/coordinate_descent_old.R @@ -0,0 +1,205 @@ +#!/usr/bin/env Rscript +# chmod ug+x +library(data.table) +library(mlr3) +library(mlr3misc) +library(mlr3learners) +library(mlr3pipelines) +library(bbotk) +library(paradox) +library(R6) +library(checkmate) +library(mlr3mbo) # @so_config +reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) +library(reticulate) +library(yahpogym) +library("future") +library("future.batchtools") +library("future.apply") + +source("OptimizerCoordinateDescent.R") +source("LearnerRegrRangerCustom.R") + +search_space = ps( + loop_function = p_fct(c("ego", "ego_log")), + init = p_fct(c("random", "lhs", "sobol")), + init_size_factor = p_int(lower = 1L, upper = 4L), + random_interleave = p_lgl(), + random_interleave_iter = p_fct(c("2", "4", "10"), depends = random_interleave == TRUE), + + rf_type = p_fct(c("standard", "smaclike_boot", "smaclike_no_boot", "smaclike_variance_boot")), + + acqf = p_fct(c("EI", "TTEI", "CB", "PI", "Mean")), + lambda = p_int(lower = 1, upper = 3, depends = acqf == "CB"), + acqopt = p_fct(c("RS_1000", "RS", "FS", "LS")) +) + +instances = readRDS("instances.rds") + +evaluate = function(xdt, instance) { + library(data.table) + library(mlr3) + library(mlr3misc) + library(mlr3learners) + library(mlr3pipelines) + library(bbotk) + library(paradox) + library(R6) + library(checkmate) + library(mlr3mbo) # @so_config + reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) + library(reticulate) + library(yahpogym) + + source("LearnerRegrRangerCustom.R") + + logger = lgr::get_logger("mlr3") + logger$set_threshold("warn") + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + + make_optim_instance = function(instance) { + benchmark = BenchmarkSet$new(instance$scenario, instance = instance$instance) + benchmark$subset_codomain(instance$target) + objective = benchmark$get_objective(instance$instance, multifidelity = FALSE, check_values = FALSE) + budget = instance$budget + optim_instance = OptimInstanceSingleCrit$new(objective, search_space = benchmark$get_search_space(drop_fidelity_params = TRUE), terminator = trm("evals", n_evals = budget), check_values = FALSE) + optim_instance + } + + optim_instance = make_optim_instance(instance) + + d = optim_instance$search_space$length + init_design_size = d * xdt$init_size_factor + init_design = if (xdt$init == "random") { + generate_design_random(optim_instance$search_space, n = init_design_size)$data + } else if (xdt$init == "lhs") { + generate_design_lhs(optim_instance$search_space, n = init_design_size)$data + } else if (xdt$init == "sobol") { + generate_design_sobol(optim_instance$search_space, n = init_design_size)$data + } + + optim_instance$eval_batch(init_design) + + random_interleave_iter = if(xdt$random_interleave) as.numeric(xdt$random_interleave_iter) else 0L + + learner = LearnerRegrRangerCustom$new() + learner$predict_type = "se" + learner$param_set$values$keep.inbag = TRUE + + if (xdt$rf_type == "standard") { + learner$param_set$values$se.method = "jack" + learner$param_set$values$splitrule = "variance" + learner$param_set$values$num.trees = 1000L + } else if (xdt$rf_type == "smaclike_boot") { + learner$param_set$values$se.method = "simple" + learner$param_set$values$splitrule = "extratrees" + learner$param_set$values$num.random.splits = 1L + learner$param_set$values$num.trees = 10 + learner$param_set$values$replace = TRUE + learner$param_set$values$sample.fraction = 1 + learner$param_set$values$min.node.size = 1 + learner$param_set$values$mtry.ratio = 1 + } else if (xdt$rf_type == "smaclike_no_boot") { + learner$param_set$values$se.method = "simple" + learner$param_set$values$splitrule = "extratrees" + learner$param_set$values$num.random.splits = 1L + learner$param_set$values$num.trees = 10 + learner$param_set$values$replace = FALSE + learner$param_set$values$sample.fraction = 1 + learner$param_set$values$min.node.size = 1 + learner$param_set$values$mtry.ratio = 1 + } else if (xdt$rf_type == "smaclike_variance_boot") { + learner$param_set$values$se.method = "simple" + learner$param_set$values$splitrule = "variance" + learner$param_set$values$num.trees = 10 + learner$param_set$values$replace = TRUE + learner$param_set$values$sample.fraction = 1 + learner$param_set$values$min.node.size = 1 + learner$param_set$values$mtry.ratio = 1 + } + + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor", multiplier = 3) %>>% learner)) + + acq_optimizer = if (xdt$acqopt == "RS_1000") { + AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 1000L)) + } else if (xdt$acqopt == "RS") { + AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 20000L)) + } else if (xdt$acqopt == "FS") { + n_repeats = 2L + maxit = 9L + batch_size = ceiling((20000L / n_repeats) / (1 + maxit)) # 1000L + AcqOptimizer$new(opt("focus_search", n_points = batch_size, maxit = maxit), terminator = trm("evals", n_evals = 20000L)) + } else if (xdt$acqopt == "LS") { + optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) + acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20020L)) + acq_optimizer$param_set$values$warmstart = TRUE + acq_optimizer$param_set$values$warmstart_size = 10L + acq_optimizer + } + + acq_function = if (xdt$acqf == "EI") { + AcqFunctionEI$new() + } else if (xdt$acqf == "TTEI") { + AcqFunctionTTEI$new(toplvl_acq_optimizer = acq_optimizer$clone(deep = TRUE)) + } else if (xdt$acqf == "CB") { + AcqFunctionCB$new(lambda = xdt$lambda) + } else if (xdt$acqf == "PI") { + AcqFunctionPI$new() + } else if (xdt$acqf == "Mean") { + AcqFunctionMean$new() + } + + if (xdt$loop_function == "ego") { + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + } else if (xdt$loop_function == "ego_log") { + bayesopt_ego_log(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + } + + best = optim_instance$archive$best()[[instance$target]] + ecdf_best = instance$ecdf(best) # evaluate the precomputed ecdf for the best value found; our target is effectively P(X <= best) + cat("scenario:", instance$scenario, "instance:", instance$instance, "ECDF_best:", ecdf_best, "\n") + ecdf_best +} + +evaluate_all = function(xs, instances) { + # evaluate_all will be carried out after future.batchtools scheduled a job to evaluate xs on all instances + library(future) + library(future.apply) + library(future.batchtools) + plan("batchtools_slurm", resources = list(walltime = 3600L * 7L, ncpus = 28L, memory = 1000L), template = "slurm_wyoming.tmpl") + tmp = future_lapply(transpose_list(instances), function(instance) { + res_instance = evaluate(xs, instance) + #res_instance = tryCatch(evaluate(xs, instance), error = function(error_condition) 0) + }, future.seed = TRUE) + data.table(mean_perf = mean(unlist(tmp), na.rm = TRUE), raw_perfs = list(tmp), n_na = sum(is.na(unlist(tmp)))) +} + +objective = ObjectiveRFunDt$new( + fun = function(xdt) { + library(future) + library(future.apply) + library(future.batchtools) + # FIXME: walltime can be set adaptively based on xdt + # FIXME: we could continuously model the walltime with a surrogate and set this for each xs in xdt + plan("batchtools_slurm", resources = list(walltime = 3600L * 7L, ncpus = 1L, memory = 1000L), template = "slurm_wyoming.tmpl") + res = future_lapply(transpose_list(xdt), function(xs) { + evaluate_all(xs, instances = instances) + }, future.seed = TRUE) + rbindlist(res) + }, + domain = search_space, + codomain = ps(mean_perf = p_dbl(tags = "maximize")), + check_values = FALSE +) + +cd_instance = OptimInstanceSingleCrit$new( + objective = objective, + search_space = search_space, + terminator = trm("none") # OptimizerChain currently terminates on its own +) + +optimizer = OptimizerCoordinateDescent$new() +optimizer$param_set$values$max_gen = 1L + +optimizer$optimize(cd_instance) diff --git a/attic/so_config_old/get_filling_candidate.R b/attic/so_config_old/get_filling_candidate.R new file mode 100644 index 00000000..0e9f8ff5 --- /dev/null +++ b/attic/so_config_old/get_filling_candidate.R @@ -0,0 +1,8 @@ +get_filling_candidate = function(instance) { + newdata = generate_design_random(instance$search_space, n = 10000L)$data + gw_dists = get_gower_dist(fct_to_char(newdata), instance$archive$data[, instance$archive$cols_x, with = FALSE]) # 0 if identical, 1 if maximally dissimilar + min_gw_dists = apply(gw_dists, MARGIN = 1L, FUN = min) # get the minium for each new point to the points in the archive + which.max(min_gw_dists) + newdata[which.max(min_gw_dists), ] +} + diff --git a/attic/so_config_old/helpers.R b/attic/so_config_old/helpers.R new file mode 100644 index 00000000..a66f06b4 --- /dev/null +++ b/attic/so_config_old/helpers.R @@ -0,0 +1,9 @@ +make_optim_instance = function(instance) { + benchmark = BenchmarkSet$new(as.character(instance$scenario), instance = as.character(instance$instance)) + benchmark$subset_codomain(instance$target) + objective = benchmark$get_objective(as.character(instance$instance), multifidelity = FALSE, check_values = FALSE) + n_evals = as.integer(ceiling(instance$budget / instance$max_budget)) # full budget + optim_instance = OptimInstanceSingleCrit$new(objective, search_space = benchmark$get_search_space(drop_fidelity_params = TRUE), terminator = trm("evals", n_evals = n_evals), check_values = FALSE) + optim_instance +} + diff --git a/attic/so_config_old/min_max.R b/attic/so_config_old/min_max.R new file mode 100644 index 00000000..7f396a90 --- /dev/null +++ b/attic/so_config_old/min_max.R @@ -0,0 +1,40 @@ +library(data.table) +library(bbotk) +library(mlr3misc) +library(paradox) +reticulate::use_virtualenv("/home/lps/.local/share/virtualenvs/yahpo_gym-4ygV7ggv/", required = TRUE) +library(reticulate) +library(yahpogym) + +instances = data.table(scenario = rep(paste0("rbv2_", c("aknn", "glmnet", "ranger", "rpart", "super", "svm", "xgboost")), each = 5L), + instance = c("40499", "1476", "6", "12", "41150", + "40979", "1501", "40966", "1478", "40984", + "12", "458", "1510", "1515", "307", + "1478", "40979", "12", "28", "1501", + "41164", "37", "1515", "1510", "42", + "1478", "1501", "40499", "40979", "300", + "40984", "40979", "40966", "28", "22"), + target = "acc", + budget = rep(c(118, 90, 134, 110, 267, 118, 170), each = 5L)) + +make_optim_instance = function(instance) { + benchmark = BenchmarkSet$new(instance$scenario, instance = instance$instance) + benchmark$subset_codomain(instance$target) + objective = benchmark$get_objective(instance$instance, multifidelity = FALSE, check_values = FALSE) + budget = instance$budget + optim_instance = OptimInstanceSingleCrit$new(objective, search_space = benchmark$get_search_space(drop_fidelity_params = TRUE), terminator = trm("evals", n_evals = 1000000L), check_values = FALSE) + optim_instance +} + +evaluate = function(instance) { + optim_instance = make_optim_instance(instance) + opt("random_search", batch_size = 10000L)$optimize(optim_instance) + ys = optim_instance$archive$data[, optim_instance$archive$cols_y, with = FALSE][[1L]] + data.table(min = min(ys), max = max(ys), mean = mean(ys), sd = sd(ys), ecdf = list(ecdf(ys))) +} + +y_stats = map_dtr(seq_len(nrow(instances)), function(i) { + evaluate(instances[i, ]) +}) + +saveRDS(cbind(instances, y_stats), "instances.rds") diff --git a/attic/so_config_old/run.R b/attic/so_config_old/run.R new file mode 100755 index 00000000..28d9b93f --- /dev/null +++ b/attic/so_config_old/run.R @@ -0,0 +1,187 @@ +#!/usr/bin/env Rscript +# chmod ug+x +library(argparse) +library(data.table) +library(mlr3) +library(mlr3misc) +library(mlr3learners) +library(mlr3pipelines) +library(bbotk) +library(paradox) +library(R6) +library(checkmate) +library(mlr3mbo) # @so_config +reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) +library(reticulate) +library(yahpogym) +library("future") +library("future.apply") + +source("OptimizerChain.R") + +parser = ArgumentParser() +parser$add_argument("-r", "--run", type = "integer", default = 1, help = "Id of run, should be within 1-5") +args = parser$parse_args() +run_id = args$run +stopifnot(run_id %in% 1:5) +cat("run_id:", run_id, "\n") + +seeds = c(2409, 2906, 0905, 2412, 3112) + +set.seed(seeds[run_id]) + +search_space = ps( + init = p_fct(c("random", "lhs", "sobol")), + init_size_factor = p_int(lower = 1L, upper = 4L), + random_interleave = p_lgl(), + random_interleave_iter = p_fct(c("2", "3", "5", "10"), depends = random_interleave == TRUE), + + surrogate = p_fct(c("ranger", "ranger_custom")), + mtry.ratio = p_fct(c("default", "1")), + num.trees = p_fct(c("10", "100", "1000")), + replace = p_lgl(depends = surrogate == "ranger"), + sample.fraction = p_fct(c("0.6", "0.8", "1"), depends = surrogate == "ranger"), + splitrule = p_fct(c("variance", "extratrees"), depends = surrogate == "ranger"), + num.random.splits = p_fct(c("1", "2", "10"), depends = surrogate == "ranger" && splitrule == "extratrees"), + min.node.size = p_fct(c("1", "5"), depends = surrogate == "ranger"), + se.method = p_fct(c("jack", "infjack"), depends = surrogate == "ranger"), + se.simple.spatial = p_lgl(depends = surrogate == "ranger_custom"), + + acqf = p_fct(c("EI", "CB")), + lambda = p_fct(c("1", "2", "3"), depends = acqf == "CB"), + acqopt = p_fct(c("RS_1000", "RS", "FS", "LS")) +) +search_space$trafo = function(x, param_set) { + x[["random_interleave_iter"]] = as.integer(x[["random_interleave_iter"]]) + x[["num.trees"]] = as.integer(x[["num.trees"]]) + x[["sample.fraction"]] = as.numeric(x[["sample.fraction"]]) + x[["num.random.splits"]] = as.integer(x[["num.random.splits"]]) + x[["min.node.size"]] = as.integer(x[["min.node.size"]]) + x[["lambda"]] = as.numeric(x[["lambda"]]) + x[map_lgl(x, function(y) length(y) == 0L)] = NA + x +} + +instances = readRDS("instances.rds") +#instances = data.table(scenario = rep(paste0("rbv2_", c("aknn", "glmnet", "ranger", "rpart", "super", "svm", "xgboost")), each = 5L), +# instance = c("40499", "1476", "6", "12", "41150", +# "40979", "1501", "40966", "1478", "40984", +# "12", "458", "1510", "1515", "307", +# "1478", "40979", "12", "28", "1501", +# "41164", "37", "1515", "1510", "42", +# "1478", "1501", "40499", "40979", "300", +# "40984", "40979", "40966", "28", "22"), +# target = "acc", +# budget = rep(c(118, 90, 134, 110, 267, 118, 170), each = 5L)) + +make_optim_instance = function(instance) { + benchmark = BenchmarkSet$new(instance$scenario, instance = instance$instance) + benchmark$subset_codomain(instance$target) + objective = benchmark$get_objective(instance$instance, multifidelity = FALSE, check_values = FALSE) + budget = instance$budget + optim_instance = OptimInstanceSingleCrit$new(objective, search_space = benchmark$get_search_space(drop_fidelity_params = TRUE), terminator = trm("evals", n_evals = budget), check_values = FALSE) + optim_instance +} + +evaluate = function(xdt, instance) { + logger = lgr::get_logger("mlr3") + logger$set_threshold("warn") + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + + optim_instance = make_optim_instance(instance) + + d = optim_instance$search_space$length + init_design_size = d * xdt$init_size_factor + init_design = if (xdt$init == "random") { + generate_design_random(optim_instance$search_space, n = init_design_size)$data + } else if (xdt$init == "lhs") { + generate_design_lhs(optim_instance$search_space, n = init_design_size)$data + } else if (xdt$init == "sobol") { + generate_design_sobol(optim_instance$search_space, n = init_design_size)$data + } + + optim_instance$eval_batch(init_design) + + random_interleave_iter = if(xdt$random_interleave) xdt$random_interleave_iter else 0L + + if (xdt$surrogate == "ranger") { + learner = lrn("regr.ranger", keep.inbag = TRUE) + values = as.list(xdt[, c("num.trees", "replace", "sample.fraction", "splitrule", "num.random.splits", "min.node.size", "se.method")]) + values = values[!map_lgl(values, function(x) is.na(x))] + } else if (xdt$surrogate == "ranger_custom") { + learner = lrn("regr.ranger_custom") + values = as.list(xdt[, c("num.trees", "se.simple.spatial")]) + values = values[!map_lgl(values, function(x) is.na(x))] + } + if (xdt$mtry.ratio == "1") { + values$mtry.ratio = 1 + } + learner$param_set$values = insert_named(learner$param_set$values %??% list(), values) + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% learner)) + + acq_function = if (xdt$acqf == "EI") { + AcqFunctionEI$new() + } else if (xdt$acqf == "CB") { + AcqFunctionCB$new(lambda = xdt$lambda) + } + + acq_optimizer = if (xdt$acqopt == "RS_1000") { + AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 1000L)) + } else if (xdt$acqopt == "RS") { + AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 20000L)) + } else if (xdt$acqopt == "FS") { + n_repeats = 2L + maxit = 9L + batch_size = ceiling((20000L / n_repeats) / (1 + maxit)) # 1000L + AcqOptimizer$new(opt("focus_search", n_points = batch_size, maxit = maxit), terminator = trm("evals", n_evals = 20000L)) + } else if (xdt$acqopt == "LS") { + optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) + acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20020L)) + acq_optimizer$param_set$values$warmstart = TRUE + acq_optimizer$param_set$values$warmstart_size = "all" + acq_optimizer + } + + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + + best = optim_instance$archive$best()[[instance$target]] + ecdf_best = instance$ecdf(best) # evaluate the precomputed ecdf for the best value found; our target is effectively P(X <= best) + cat("scenario:", instance$scenario, "instance:", instance$instance, "ECDF_best:", ecdf_best, "\n") + ecdf_best +} + +objective = ObjectiveRFunDt$new( + fun = function(xdt) { + saveRDS(ac_instance, paste0("ac_instance_", run_id, ".rds")) + map_dtr(seq_len(nrow(xdt)), function(i) { + plan("multicore") + tmp = future_lapply(transpose_list(instances), function(instance) { + res_instance = tryCatch(evaluate(xdt[i, ], instance), error = function(error_condition) 0) + }, future.seed = TRUE) + data.table(mean_perf = mean(unlist(tmp), na.rm = TRUE), raw_perfs = list(tmp), n_na = sum(is.na(unlist(tmp)))) + }) + }, + domain = search_space, + codomain = ps(mean_perf = p_dbl(tags = "maximize")), + check_values = FALSE +) + +ac_instance = OptimInstanceSingleCrit$new( + objective = objective, + search_space = search_space, + terminator = trm("evals", n_evals = 200L) # 100 init design + 100 +) + +surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% lrn("regr.ranger", num.trees = 1000L, se.method = "jack", keep.inbag = TRUE))) +acq_function = AcqFunctionCB$new(lambda = 3) +optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) +acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20020L)) +acq_optimizer$param_set$values$warmstart = TRUE +acq_optimizer$param_set$values$warmstart_size = "all" +design = generate_design_sobol(ac_instance$search_space, n = 100L)$data +ac_instance$eval_batch(design) +saveRDS(ac_instance, paste0("ac_instance_", run_id, ".rds")) +bayesopt_ego(ac_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = 5L) +saveRDS(ac_instance, paste0("ac_instance_", run_id, ".rds")) + diff --git a/attic/so_config_old/run_yahpo.R b/attic/so_config_old/run_yahpo.R new file mode 100644 index 00000000..d40a3217 --- /dev/null +++ b/attic/so_config_old/run_yahpo.R @@ -0,0 +1,465 @@ +library(batchtools) +library(data.table) +library(mlr3) +library(mlr3learners) +library(mlr3pipelines) +library(mlr3misc) +library(mlr3mbo) # @so_config +library(bbotk) +library(paradox) +library(R6) +library(checkmate) + +reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) +library(reticulate) +yahpo_gym = import("yahpo_gym") + +packages = c("data.table", "mlr3", "mlr3learners", "mlr3pipelines", "mlr3misc", "mlr3mbo", "bbotk", "paradox", "mlrintermbo", "R6", "checkmate") + +RhpcBLASctl::blas_set_num_threads(1L) +RhpcBLASctl::omp_set_num_threads(1L) + +root = here::here() +experiments_dir = file.path(root) + +source_files = map_chr(c("helpers.R", "OptimizerChain.R"), function(x) file.path(experiments_dir, x)) +for (sf in source_files) { + source(sf) +} + +reg = makeExperimentRegistry(file.dir = "/gscratch/lschnei8/registry_mlr3mbo_so_config", packages = packages, source = source_files) +#reg = makeExperimentRegistry(file.dir = NA, conf.file = NA, packages = packages, source = source_files) # interactive session +saveRegistry(reg) + +# FIXME: also compare jack vs. infjack? +mlr3mbo_wrapper = function(job, data, instance, ...) { + reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + optim_instance = make_optim_instance(instance) + + xdt = list(init = "lhs", init_size_factor = 4L, random_interleave = FALSE, num.trees = 250L, splitrule = "extratrees", num.random.splits = 8, acqf = "CB", lambda = 2.8, acqopt_iter_factor = 6L, acqopt = "FS", fs_behavior = "global") + + d = optim_instance$search_space$length + init_design_size = d * xdt$init_size_factor + init_design = if (xdt$init == "random") generate_design_random(optim_instance$search_space, n = init_design_size)$data else if (xdt$init == "lhs") generate_design_lhs(optim_instance$search_space, n = init_design_size)$data + optim_instance$eval_batch(init_design) + + random_interleave_iter = if(xdt$random_interleave) xdt$random_interleave_iter else 0L + + learner = if (xdt$splitrule == "extratrees") { + lrn("regr.ranger", num.trees = xdt$num.trees, keep.inbag = TRUE, splitrule = xdt$splitrule, num.random.splits = xdt$num.random.splits) + } else if (xdt$splitrule == "variance") { + lrn("regr.ranger", num.trees = xdt$num.trees, keep.inbag = TRUE, splitrule = xdt$splitrule) + } + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% learner)) + + acq_function = if (xdt$acqf == "EI") { + AcqFunctionEI$new() + } else if (xdt$acqf == "CB") { + AcqFunctionCB$new(lambda = xdt$lambda) + } else if (xdt$acqf == "PI") { + AcqFunctionPI$new() + } + + acq_budget = 1000 * xdt$acqopt_iter_factor + + acq_optimizer = if (xdt$acqopt == "RS") { + AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = acq_budget)) + } else if (xdt$acqopt == "FS") { + if (xdt$fs_behavior == "global") { + n_repeats = 10L + maxit = 2L + batch_size = ceiling((acq_budget / n_repeats) / (1 + maxit)) + } else if (xdt$fs_behavior == "local") { + n_repeats = 2L + maxit = 10L + batch_size = ceiling((acq_budget / n_repeats) / (1 + maxit)) + } + AcqOptimizer$new(opt("focus_search", n_points = batch_size, maxit = maxit), terminator = trm("evals", n_evals = acq_budget)) + } + + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + optim_instance +} + +mlrintermbo_wrapper = function(job, data, instance, ...) { + reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + optim_instance = make_optim_instance(instance) + optimizer = opt("intermbo", on.surrogate.error = "stop") + learner = lrn("regr.ranger", se.method = "jack", keep.inbag = TRUE) + learner$predict_type = "se" + learner = GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% learner) + learner$predict_type = "se" + #learner = mlr::makeLearner("regr.randomForest", se.method = "jackknife", keep.inbag = TRUE) + #learner = mlr::makeImputeWrapper(learner, classes = list(numeric = mlr::imputeMax(2), factor = mlr::imputeConstant("__miss__"), logical = mlr::imputeUniform())) + #learner = mlr::setPredictType(learner, "se") + #optimizer$param_set$values$surrogate.learner = learner + optimizer$optimize(optim_instance) + optim_instance +} + +mlr3mbo_default_wrapper = function(job, data, instance, ...) { + reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + optim_instance = make_optim_instance(instance) + + xdt = list(init = "random", init_size_factor = 4L, random_interleave = FALSE, num.trees = 500L, splitrule = "variance", num.random.splits = NA_integer_, acqf = "EI", lambda = NA_real_, acqopt_iter_factor = 10L, acqopt = "RS", fs_behavior = NA_character_) + + d = optim_instance$search_space$length + init_design_size = d * xdt$init_size_factor + init_design = if (xdt$init == "random") generate_design_random(optim_instance$search_space, n = init_design_size)$data else if (xdt$init == "lhs") generate_design_lhs(optim_instance$search_space, n = init_design_size)$data + optim_instance$eval_batch(init_design) + + random_interleave_iter = if(xdt$random_interleave) xdt$random_interleave_iter else 0L + + learner = if (xdt$splitrule == "extratrees") { + lrn("regr.ranger", num.trees = xdt$num.trees, keep.inbag = TRUE, splitrule = xdt$splitrule, num.random.splits = xdt$num.random.splits) + } else if (xdt$splitrule == "variance") { + lrn("regr.ranger", num.trees = xdt$num.trees, keep.inbag = TRUE, splitrule = xdt$splitrule) + } + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% learner)) + + acq_function = if (xdt$acqf == "EI") { + AcqFunctionEI$new() + } else if (xdt$acqf == "CB") { + AcqFunctionCB$new(lambda = xdt$lambda) + } else if (xdt$acqf == "PI") { + AcqFunctionPI$new() + } + + acq_budget = 1000 * xdt$acqopt_iter_factor + + acq_optimizer = if (xdt$acqopt == "RS") { + AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = acq_budget)) + } else if (xdt$acqopt == "FS") { + if (xdt$fs_behavior == "global") { + n_repeats = 10L + maxit = 2L + batch_size = ceiling((acq_budget / n_repeats) / (1 + maxit)) + } else if (xdt$fs_behavior == "local") { + n_repeats = 2L + maxit = 10L + batch_size = ceiling((acq_budget / n_repeats) / (1 + maxit)) + } + AcqOptimizer$new(opt("focus_search", n_points = batch_size, maxit = maxit), terminator = trm("evals", n_evals = acq_budget)) + } + + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + optim_instance +} + +mlr3mbo_wrapper_custom = function(job, data, instance, ...) { + reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + optim_instance = make_optim_instance(instance) + + xdt = list(init = "lhs", init_size_factor = 1L, random_interleave = FALSE, num.trees = 250L, splitrule = "extratrees", num.random.splits = 10, acqf = "CB", lambda = 3, acqopt_iter_factor = 30L, acqopt = "FS", fs_behavior = "global") + + d = optim_instance$search_space$length + init_design_size = d * xdt$init_size_factor + init_design = if (xdt$init == "random") generate_design_random(optim_instance$search_space, n = init_design_size)$data else if (xdt$init == "lhs") generate_design_lhs(optim_instance$search_space, n = init_design_size)$data + optim_instance$eval_batch(init_design) + + random_interleave_iter = if(xdt$random_interleave) xdt$random_interleave_iter else 0L + + learner = if (xdt$splitrule == "extratrees") { + lrn("regr.ranger", num.trees = xdt$num.trees, keep.inbag = TRUE, splitrule = xdt$splitrule, num.random.splits = xdt$num.random.splits) + } else if (xdt$splitrule == "variance") { + lrn("regr.ranger", num.trees = xdt$num.trees, keep.inbag = TRUE, splitrule = xdt$splitrule) + } + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% learner)) + + acq_function = if (xdt$acqf == "EI") { + AcqFunctionEI$new() + } else if (xdt$acqf == "CB") { + AcqFunctionCB$new(lambda = xdt$lambda) + } else if (xdt$acqf == "PI") { + AcqFunctionPI$new() + } + + acq_budget = 1000 * xdt$acqopt_iter_factor + + acq_optimizer = if (xdt$acqopt == "RS") { + AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = acq_budget)) + } else if (xdt$acqopt == "FS") { + if (xdt$fs_behavior == "global") { + n_repeats = 5L + maxit = 5L + batch_size = ceiling((acq_budget / n_repeats) / (1 + maxit)) + } else if (xdt$fs_behavior == "local") { + n_repeats = 2L + maxit = 10L + batch_size = ceiling((acq_budget / n_repeats) / (1 + maxit)) + } + AcqOptimizer$new(opt("focus_search", n_points = batch_size, maxit = maxit), terminator = trm("evals", n_evals = acq_budget)) + } + + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + optim_instance +} + +mlr3mbo_wrapper_new_rf = function(job, data, instance, ...) { + reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + optim_instance = make_optim_instance(instance) + + d = optim_instance$search_space$length + init_design_size = 4L * d + init_design = generate_design_lhs(optim_instance$search_space, n = init_design_size)$data + optim_instance$eval_batch(init_design) + + random_interleave_iter = 0L + + learner = lrn("regr.ranger_custom") + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% learner)) + + acq_function = AcqFunctionEI$new() + + acq_budget = 20000L + + acq_optimizer = { + n_repeats = 2L + maxit = 9L + batch_size = ceiling((acq_budget / n_repeats) / (1 + maxit)) + AcqOptimizer$new(opt("focus_search", n_points = batch_size, maxit = maxit), terminator = trm("evals", n_evals = acq_budget)) + } + + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + optim_instance +} + +mlr3mbo_wrapper_new_rf_ls = function(job, data, instance, ...) { + reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + optim_instance = make_optim_instance(instance) + + d = optim_instance$search_space$length + init_design_size = 4L * d + init_design = generate_design_lhs(optim_instance$search_space, n = init_design_size)$data + optim_instance$eval_batch(init_design) + + random_interleave_iter = 0L + + learner = lrn("regr.ranger_custom") + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% learner)) + + acq_function = AcqFunctionEI$new() + + optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) + acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20010L)) + + acq_optimizer$param_set$values$warmstart = TRUE + acq_optimizer$param_set$values$warmstart_size = "all" + + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + optim_instance +} + +mlr3mbo_wrapper_xxx_ls = function(job, data, instance, ...) { + reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + optim_instance = make_optim_instance(instance) + + d = optim_instance$search_space$length + init_design_size = 4L * d + init_design = generate_design_sobol(optim_instance$search_space, n = init_design_size)$data + optim_instance$eval_batch(init_design) + + random_interleave_iter = 4L + + learner = lrn("regr.ranger_custom") + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor", multiplier = 2) %>>% learner)) + + acq_function = AcqFunctionEI$new() + + optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) + acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20010L)) + + acq_optimizer$param_set$values$warmstart = TRUE + acq_optimizer$param_set$values$warmstart_size = "all" + + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + optim_instance +} + +mlr3mbo_wrapper_kknn_ls = function(job, data, instance, ...) { + reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + optim_instance = make_optim_instance(instance) + + d = optim_instance$search_space$length + init_design_size = 4L * d + init_design = generate_design_sobol(optim_instance$search_space, n = init_design_size)$data + optim_instance$eval_batch(init_design) + + random_interleave_iter = 4L + + learner = lrn("regr.kknn") + learner = as_learner(po("imputeoor", multiplier = 2) %>>% po("fixfactors") %>>% po("imputesample") %>>% learner) + learner$predict_types = "response" + surrogate = SurrogateLearner$new(learner) + + acq_function = AcqFunctionMean$new() + + optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) + acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20010L)) + + acq_optimizer$param_set$values$warmstart = TRUE + acq_optimizer$param_set$values$warmstart_size = "all" + + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + optim_instance +} + +# add algorithms +addAlgorithm("mlr3mbo", fun = mlr3mbo_wrapper) +addAlgorithm("mlrintermbo", fun = mlrintermbo_wrapper) +addAlgorithm("mlr3mbo_default", fun = mlr3mbo_default_wrapper) +addAlgorithm("mlr3mbo_custom", fun = mlr3mbo_wrapper_custom) +addAlgorithm("mlr3mbo_new_rf", fun = mlr3mbo_wrapper_new_rf) +addAlgorithm("mlr3mbo_new_rf_ls", fun = mlr3mbo_wrapper_new_rf_ls) +addAlgorithm("mlr3mbo_xxx_ls", fun = mlr3mbo_wrapper_xxx_ls) +addAlgorithm("mlr3mbo_kknn_ls", fun = mlr3mbo_wrapper_kknn_ls) + +# setup scenarios and instances +get_nb301_setup = function(budget_factor = 40L) { + scenario = "nb301" + bench = yahpo_gym$benchmark_set$BenchmarkSet(scenario, instance = "CIFAR10") + fidelity_space = bench$get_fidelity_space() + fidelity_param_id = fidelity_space$get_hyperparameter_names()[1] + min_budget = fidelity_space$get_hyperparameter(fidelity_param_id)$lower + max_budget = fidelity_space$get_hyperparameter(fidelity_param_id)$upper + ndim = length(bench$config_space$get_hyperparameter_names()) - 1L # NOTE: instance is not part of + + instances = "CIFAR10" + target = "val_accuracy" + budget = ceiling(20L * max_budget + sqrt(ndim) * max_budget * budget_factor) + on_integer_scale = TRUE + minimize = bench$config$config$y_minimize[match(target, bench$config$config$y_names)] + setup = setDT(expand.grid(scenario = scenario, instance = instances, target = target, ndim = ndim, max_budget = max_budget, budget = budget, on_integer_scale = on_integer_scale, minimize = minimize, stringsAsFactors = FALSE)) + setup +} + +get_lcbench_setup = function(budget_factor = 40L) { + scenario = "lcbench" + bench = yahpo_gym$benchmark_set$BenchmarkSet(scenario, instance = "167168") + fidelity_space = bench$get_fidelity_space() + fidelity_param_id = fidelity_space$get_hyperparameter_names()[1] + min_budget = fidelity_space$get_hyperparameter(fidelity_param_id)$lower + max_budget = fidelity_space$get_hyperparameter(fidelity_param_id)$upper + ndim = length(bench$config_space$get_hyperparameter_names()) - 2L + + instances = c("167168", "189873", "189906") + target = "val_accuracy" + budget = ceiling(20L * max_budget + sqrt(ndim) * max_budget * budget_factor) + on_integer_scale = TRUE + minimize = bench$config$config$y_minimize[match(target, bench$config$config$y_names)] + setup = setDT(expand.grid(scenario = scenario, instance = instances, target = target, ndim = ndim, max_budget = max_budget, budget = budget, on_integer_scale = on_integer_scale, minimize = minimize, stringsAsFactors = FALSE)) + setup +} + +get_rbv2_setup = function(budget_factor = 40L) { + setup = map_dtr(c("rbv2_glmnet", "rbv2_rpart", "rbv2_ranger", "rbv2_xgboost", "rbv2_super"), function(scenario) { + bench = yahpo_gym$benchmark_set$BenchmarkSet(scenario, instance = "1040") + fidelity_space = bench$get_fidelity_space() + fidelity_param_id = "trainsize" + min_budget = fidelity_space$get_hyperparameter(fidelity_param_id)$lower + max_budget = fidelity_space$get_hyperparameter(fidelity_param_id)$upper + ndim = length(bench$config_space$get_hyperparameter_names()) - 3L # repl and trainsize and instance + + instances = switch(scenario, rbv2_glmnet = c("375", "458"), rbv2_rpart = c("14", "40499"), rbv2_ranger = c("16", "42"), rbv2_xgboost = c("12", "1501", "16", "40499"), rbv2_super = c("1053", "1457", "1063", "1479", "15", "1468")) + target = "acc" + budget = ceiling(20L * max_budget + sqrt(ndim) * max_budget * budget_factor) + on_integer_scale = FALSE + minimize = bench$config$config$y_minimize[match(target, bench$config$config$y_names)] + setup = setDT(expand.grid(scenario = scenario, instance = instances, target = target, ndim = ndim, max_budget = max_budget, budget = budget, on_integer_scale = on_integer_scale, minimize = minimize, stringsAsFactors = FALSE)) + }) +} + +setup = rbind(get_nb301_setup(), get_lcbench_setup(), get_rbv2_setup()) + +setup[, id := seq_len(.N)] + +# add problems +prob_designs = map(seq_len(nrow(setup)), function(i) { + prob_id = paste0(setup[i, ]$scenario, "_", setup[i, ]$instance, "_", setup[i, ]$target) + addProblem(prob_id, data = list(scenario = setup[i, ]$scenario, instance = setup[i, ]$instance, target = setup[i, ]$target, ndim = setup[i, ]$ndim, max_budget = setup[i, ]$max_budget, budget = setup[i, ]$budget, on_integer_scale = setup[i, ]$on_integer_scale, minimize = setup[i, ]$minimize)) + setNames(list(setup[i, ]), nm = prob_id) +}) +nn = sapply(prob_designs, names) +prob_designs = unlist(prob_designs, recursive = FALSE, use.names = FALSE) +names(prob_designs) = nn + +# add jobs for optimizers +optimizers = data.table(algorithm = c("mlr3mbo", "mlrintermbo", "mlr3mbo_default", "mlr3mbo_custom", "mlr3mbo_new_rf", "mlr3mbo_new_rf_ls", "mlr3mbo_xxx_ls", "mlr3mbo_kknn_ls")) + +for (i in seq_len(nrow(optimizers))) { + algo_designs = setNames(list(optimizers[i, ]), nm = optimizers[i, ]$algorithm) + + ids = addExperiments( + prob.designs = prob_designs, + algo.designs = algo_designs, + repls = 30L + ) + addJobTags(ids, as.character(optimizers[i, ]$algorithm)) +} + +jobs = findJobs() +resources.default = list(walltime = 3600 * 12L, memory = 2048L, ntasks = 1L, ncpus = 1L, nodes = 1L, clusters = "teton", max.concurrent.jobs = 9999L) +submitJobs(jobs, resources = resources.default) + +done = findDone() +results = reduceResultsList(done, function(x, job) { + x = x$archive$data + budget_var = if (job$instance$scenario %in% c("lcbench", "nb301")) "epoch" else "trainsize" + target_var = job$instance$target + if (!job$instance$minimize) { + x[, (target_var) := - get(target_var)] + } + pars = job$pars + tmp = x[, target_var, with = FALSE] + tmp[, (budget_var) := job$instance$max_budget] + tmp[, method := pars$algo.pars$algorithm] + tmp[, scenario := pars$prob.pars$scenario] + tmp[, instance := pars$prob.pars$instance] + tmp[, repl := job$repl] + tmp[, iter := seq_len(.N)] + colnames(tmp) = c("target", "budget", "method", "scenario", "instance", "repl", "iter") + tmp +}) +results = rbindlist(results, fill = TRUE) +saveRDS(results, "results_yahpo_own.rds") + diff --git a/attic/so_config_old/submit.sh b/attic/so_config_old/submit.sh new file mode 100644 index 00000000..0069211b --- /dev/null +++ b/attic/so_config_old/submit.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +sbatch < lower)) + + # set default value for sigma + if (is.null(params$sigma)) { + params$sigma <- stats::median(upper-lower) / 4 + } + + optimFunBlock <- function(x) { + browser() + if (is.null(cl)) + return(apply(x,2,optimFun)) + else + return(parallel::parApply(cl,x,2,optimFun)) + } + Rlibcmaes::cmaesOptim(x0, params$sigma, optimFun, optimFunBlock,lower, upper, cmaAlgo = as.integer(params$cmaAlgorithm), lambda = ifelse(is.null(params$lambda),-1,params$lambda), maxEvals = params$maxEvals, xtol=params$xtol, ftol=params$ftol, traceFreq =params$trace, seed = params$seed, quietRun=params$quiet) +} + + +fun = get_private(acq_function)$.fun +constants = acq_function$constants$values +direction = acq_function$codomain$direction + +wrapper = function(x) { + xdt = set_names(as.data.table(t(xmat)), acq_function$domain$ids()) + res = mlr3misc::invoke(fun, xdt = xdt, .args = constants)[[1]] + res * acq_function$codomain$direction +} + +x0 = set_names(as.numeric(generate_design_random(acq_function$domain, n = 1)$data), acq_function$domain$ids()) + + +res = cmaes(x0 = x0, optimFun = wrapper, lower = acq_function$domain$lower, upper = acq_function$domain$upper, params = cmaEsParams(xtol = 1e-3,ftol = 1e-3)) + +res = cmaes(x0 = x0, optimFun = wrapper, lower = acq_function$domain$lower, upper = acq_function$domain$upper, params = cmaEsParams(cmaAlgorithm = cmaEsAlgo()$IPOP_CMAES,maxEvals=1e4)) + + +wrapper = function(xmat) { # fun, constants, direction + xdt = set_names(as.data.table(t(xmat)), acq_function$domain$ids()) + res = mlr3misc::invoke(fun, xdt = xdt, .args = constants)[[1]] + res * direction +} + +optimFunBlock = function(x) { + wrapper(x) +} + +lower = acq_function$domain$lower +upper = acq_function$domain$upper + +res = Rlibcmaes::cmaesOptim( + x0 = x0, + sigma = median(upper - lower) / 4, + optimFun = wrapper, + optimFunBlock = optimFunBlock, + lower = lower, + upper = upper, + cmaAlgo = as.integer(cmaEsAlgo()$IPOP_CMAES), + lambda = -1, + maxEvals = 1e4, + xtol = 1e-3, +) + +wrapper(res) + + +## random search +acq_optimizer_random_search = AcqOptimizer$new(opt("random_search", batch_size = 1000L), trm("evals", n_evals = 1000L)) +acq_optimizer_random_search$acq_function = acq_function +res_random_search = acq_optimizer_random_search$optimize() + +res_random_search + + + +# cmaes +acq_optimizer_cmaes = AcqOptimizerCmaes$new() +acq_optimizer_cmaes$param_set$set_values( + maxEvals = 30000L, + xtol = 1e-4 +) +acq_optimizer_cmaes$acq_function = acq_function +acq_optimizer_cmaes$optimize() diff --git a/man-roxygen/learner.R b/man-roxygen/learner.R new file mode 100644 index 00000000..20c30b5c --- /dev/null +++ b/man-roxygen/learner.R @@ -0,0 +1,14 @@ +#' @section Dictionary: +#' This [Learner] can be instantiated via the [dictionary][mlr3misc::Dictionary] [mlr3::mlr_learners] or with the associated sugar function [lrn()]: +#' ``` +#' mlr_learners$get("<%= id %>") +#' lrn("<%= id %>") +#' ``` +#' +#' @section Meta Information: +#' `r mlr3misc::rd_info(mlr3::lrn("<%= id %>"))` +#' @md +#' +#' @section Parameters: +#' `r mlr3misc::rd_info(mlr3::lrn("<%= id %>")$param_set)` +#' @md diff --git a/man-roxygen/seealso_learner.R b/man-roxygen/seealso_learner.R new file mode 100644 index 00000000..ba1aefae --- /dev/null +++ b/man-roxygen/seealso_learner.R @@ -0,0 +1,17 @@ +#' @seealso +#' +#' * Chapter in the [mlr3book](https://mlr3book.mlr-org.com/): +#' \url{https://mlr3book.mlr-org.com/basics.html#learners} +#' * Package \CRANpkg{mlr3learners} for a solid collection of essential learners. +#' * Package [mlr3extralearners](https://github.com/mlr-org/mlr3extralearners) for more learners. +#' * [Dictionary][mlr3misc::Dictionary] of [Learners][Learner]: [mlr_learners] +#' * `as.data.table(mlr_learners)` for a table of available [Learners][Learner] in the running session (depending on the loaded packages). +#' * \CRANpkg{mlr3pipelines} to combine learners with pre- and postprocessing steps. +#' * Package \CRANpkg{mlr3viz} for some generic visualizations. +#' * Extension packages for additional task types: +#' * \CRANpkg{mlr3proba} for probabilistic supervised regression and survival analysis. +#' * \CRANpkg{mlr3cluster} for unsupervised clustering. +#' * \CRANpkg{mlr3tuning} for tuning of hyperparameters, \CRANpkg{mlr3tuningspaces} +#' for established default tuning spaces. +#' +#' @family Learner diff --git a/man/AcqOptimizerCmaes.Rd b/man/AcqOptimizerCmaes.Rd new file mode 100644 index 00000000..2a5a35dc --- /dev/null +++ b/man/AcqOptimizerCmaes.Rd @@ -0,0 +1,109 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/AcqOptimizerCmaes.R +\name{AcqOptimizerCmaes} +\alias{AcqOptimizerCmaes} +\title{CMA-ES Acquisition Function Optimizer} +\description{ +CMA-ES Acquisition Function Optimizer + +CMA-ES Acquisition Function Optimizer +} +\section{Super class}{ +\code{\link[mlr3mbo:AcqOptimizer]{mlr3mbo::AcqOptimizer}} -> \code{AcqOptimizerCmaes} +} +\section{Public fields}{ +\if{html}{\out{
}} +\describe{ +\item{\code{state}}{(\code{list()})\cr +\code{\link[libcmaesr:cmaes]{libcmaesr::cmaes()}} results.} +} +\if{html}{\out{
}} +} +\section{Active bindings}{ +\if{html}{\out{
}} +\describe{ +\item{\code{print_id}}{(\code{character})\cr +Id used when printing.} + +\item{\code{param_set}}{(\link[paradox:ParamSet]{paradox::ParamSet})\cr +Set of hyperparameters.} +} +\if{html}{\out{
}} +} +\section{Methods}{ +\subsection{Public methods}{ +\itemize{ +\item \href{#method-AcqOptimizerCmaes-new}{\code{AcqOptimizerCmaes$new()}} +\item \href{#method-AcqOptimizerCmaes-optimize}{\code{AcqOptimizerCmaes$optimize()}} +\item \href{#method-AcqOptimizerCmaes-reset}{\code{AcqOptimizerCmaes$reset()}} +\item \href{#method-AcqOptimizerCmaes-clone}{\code{AcqOptimizerCmaes$clone()}} +} +} +\if{html}{\out{ +
Inherited methods + +
+}} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerCmaes-new}{}}} +\subsection{Method \code{new()}}{ +Creates a new instance of this \link[R6:R6Class]{R6} class. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{AcqOptimizerCmaes$new(acq_function = NULL)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{acq_function}}{(\code{NULL} | \link{AcqFunction}).} +} +\if{html}{\out{
}} +} +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerCmaes-optimize}{}}} +\subsection{Method \code{optimize()}}{ +Optimize the acquisition function. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{AcqOptimizerCmaes$optimize()}\if{html}{\out{
}} +} + +\subsection{Returns}{ +\code{\link[data.table:data.table]{data.table::data.table()}} with 1 row per candidate. +} +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerCmaes-reset}{}}} +\subsection{Method \code{reset()}}{ +Reset the acquisition function optimizer. + +Currently not used. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{AcqOptimizerCmaes$reset()}\if{html}{\out{
}} +} + +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerCmaes-clone}{}}} +\subsection{Method \code{clone()}}{ +The objects of this class are cloneable with this method. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{AcqOptimizerCmaes$clone(deep = FALSE)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{deep}}{Whether to make a deep clone.} +} +\if{html}{\out{
}} +} +} +} diff --git a/man/AcqOptimizerDirect.Rd b/man/AcqOptimizerDirect.Rd new file mode 100644 index 00000000..f2cf9b58 --- /dev/null +++ b/man/AcqOptimizerDirect.Rd @@ -0,0 +1,112 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/AcqOptimizerDirect.R +\name{AcqOptimizerDirect} +\alias{AcqOptimizerDirect} +\title{Direct Optimization Acquisition Function Optimizer} +\description{ +If the restart strategy is \code{"none"}, the optimizer starts with the best point in the archive. +The optimization stops when one of the stopping criteria is met. + +If \code{restart_strategy} is \code{"random"}, the optimizer runs at least for \code{maxeval / n_restarts} iterations. +The first iteration starts with the best point in the archive. +The next iterations start from a random point. +} +\section{Super class}{ +\code{\link[mlr3mbo:AcqOptimizer]{mlr3mbo::AcqOptimizer}} -> \code{AcqOptimizerDirect} +} +\section{Public fields}{ +\if{html}{\out{
}} +\describe{ +\item{\code{state}}{(\code{list()})\cr +List of \code{\link[nloptr:nloptr]{nloptr::nloptr()}} results.} +} +\if{html}{\out{
}} +} +\section{Active bindings}{ +\if{html}{\out{
}} +\describe{ +\item{\code{print_id}}{(\code{character})\cr +Id used when printing.} + +\item{\code{param_set}}{(\link[paradox:ParamSet]{paradox::ParamSet})\cr +Set of hyperparameters.} +} +\if{html}{\out{
}} +} +\section{Methods}{ +\subsection{Public methods}{ +\itemize{ +\item \href{#method-AcqOptimizerDirect-new}{\code{AcqOptimizerDirect$new()}} +\item \href{#method-AcqOptimizerDirect-optimize}{\code{AcqOptimizerDirect$optimize()}} +\item \href{#method-AcqOptimizerDirect-reset}{\code{AcqOptimizerDirect$reset()}} +\item \href{#method-AcqOptimizerDirect-clone}{\code{AcqOptimizerDirect$clone()}} +} +} +\if{html}{\out{ +
Inherited methods + +
+}} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerDirect-new}{}}} +\subsection{Method \code{new()}}{ +Creates a new instance of this \link[R6:R6Class]{R6} class. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{AcqOptimizerDirect$new(acq_function = NULL)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{acq_function}}{(\code{NULL} | \link{AcqFunction}).} +} +\if{html}{\out{
}} +} +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerDirect-optimize}{}}} +\subsection{Method \code{optimize()}}{ +Optimize the acquisition function. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{AcqOptimizerDirect$optimize()}\if{html}{\out{
}} +} + +\subsection{Returns}{ +\code{\link[data.table:data.table]{data.table::data.table()}} with 1 row per candidate. +} +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerDirect-reset}{}}} +\subsection{Method \code{reset()}}{ +Reset the acquisition function optimizer. + +Currently not used. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{AcqOptimizerDirect$reset()}\if{html}{\out{
}} +} + +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerDirect-clone}{}}} +\subsection{Method \code{clone()}}{ +The objects of this class are cloneable with this method. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{AcqOptimizerDirect$clone(deep = FALSE)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{deep}}{Whether to make a deep clone.} +} +\if{html}{\out{
}} +} +} +} diff --git a/man/AcqOptimizerLbfgsb.Rd b/man/AcqOptimizerLbfgsb.Rd new file mode 100644 index 00000000..c4e4954b --- /dev/null +++ b/man/AcqOptimizerLbfgsb.Rd @@ -0,0 +1,112 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/AcqOptimizerLbfgsb.R +\name{AcqOptimizerLbfgsb} +\alias{AcqOptimizerLbfgsb} +\title{L-BFGS-B Acquisition Function Optimizer} +\description{ +If \code{restart_strategy} is \code{"random"}, the optimizer runs for \code{n_iterations} iterations. +Each iteration starts with a random search of size \code{random_restart_size}. +The best point is used as the start point for the L-BFGS-B optimization. + +If \code{restart_strategy} is \code{"none"}, the only the L-BFGS-B optimization is performed. +The start point is the best point in the archive. +} +\section{Super class}{ +\code{\link[mlr3mbo:AcqOptimizer]{mlr3mbo::AcqOptimizer}} -> \code{AcqOptimizerLbfgsb} +} +\section{Public fields}{ +\if{html}{\out{
}} +\describe{ +\item{\code{state}}{(\code{list()})\cr +List of \code{\link[nloptr:nloptr]{nloptr::nloptr()}} results.} +} +\if{html}{\out{
}} +} +\section{Active bindings}{ +\if{html}{\out{
}} +\describe{ +\item{\code{print_id}}{(\code{character})\cr +Id used when printing.} + +\item{\code{param_set}}{(\link[paradox:ParamSet]{paradox::ParamSet})\cr +Set of hyperparameters.} +} +\if{html}{\out{
}} +} +\section{Methods}{ +\subsection{Public methods}{ +\itemize{ +\item \href{#method-AcqOptimizerLbfgsb-new}{\code{AcqOptimizerLbfgsb$new()}} +\item \href{#method-AcqOptimizerLbfgsb-optimize}{\code{AcqOptimizerLbfgsb$optimize()}} +\item \href{#method-AcqOptimizerLbfgsb-reset}{\code{AcqOptimizerLbfgsb$reset()}} +\item \href{#method-AcqOptimizerLbfgsb-clone}{\code{AcqOptimizerLbfgsb$clone()}} +} +} +\if{html}{\out{ +
Inherited methods + +
+}} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerLbfgsb-new}{}}} +\subsection{Method \code{new()}}{ +Creates a new instance of this \link[R6:R6Class]{R6} class. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{AcqOptimizerLbfgsb$new(acq_function = NULL)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{acq_function}}{(\code{NULL} | \link{AcqFunction}).} +} +\if{html}{\out{
}} +} +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerLbfgsb-optimize}{}}} +\subsection{Method \code{optimize()}}{ +Optimize the acquisition function. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{AcqOptimizerLbfgsb$optimize()}\if{html}{\out{
}} +} + +\subsection{Returns}{ +\code{\link[data.table:data.table]{data.table::data.table()}} with 1 row per candidate. +} +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerLbfgsb-reset}{}}} +\subsection{Method \code{reset()}}{ +Reset the acquisition function optimizer. + +Currently not used. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{AcqOptimizerLbfgsb$reset()}\if{html}{\out{
}} +} + +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerLbfgsb-clone}{}}} +\subsection{Method \code{clone()}}{ +The objects of this class are cloneable with this method. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{AcqOptimizerLbfgsb$clone(deep = FALSE)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{deep}}{Whether to make a deep clone.} +} +\if{html}{\out{
}} +} +} +} diff --git a/man/AcqOptimizerLocalSearch.Rd b/man/AcqOptimizerLocalSearch.Rd new file mode 100644 index 00000000..37a96b30 --- /dev/null +++ b/man/AcqOptimizerLocalSearch.Rd @@ -0,0 +1,109 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/AcqOptimizerLocalSearch.R +\name{AcqOptimizerLocalSearch} +\alias{AcqOptimizerLocalSearch} +\title{Local Search Acquisition Function Optimizer} +\description{ +Local Search Acquisition Function Optimizer + +Local Search Acquisition Function Optimizer +} +\section{Super class}{ +\code{\link[mlr3mbo:AcqOptimizer]{mlr3mbo::AcqOptimizer}} -> \code{AcqOptimizerLocalSearch} +} +\section{Public fields}{ +\if{html}{\out{
}} +\describe{ +\item{\code{state}}{(\code{list()})\cr +List of \code{\link[cmaes:cma_es]{cmaes::cma_es()}} results.} +} +\if{html}{\out{
}} +} +\section{Active bindings}{ +\if{html}{\out{
}} +\describe{ +\item{\code{print_id}}{(\code{character})\cr +Id used when printing.} + +\item{\code{param_set}}{(\link[paradox:ParamSet]{paradox::ParamSet})\cr +Set of hyperparameters.} +} +\if{html}{\out{
}} +} +\section{Methods}{ +\subsection{Public methods}{ +\itemize{ +\item \href{#method-AcqOptimizerLocalSearch-new}{\code{AcqOptimizerLocalSearch$new()}} +\item \href{#method-AcqOptimizerLocalSearch-optimize}{\code{AcqOptimizerLocalSearch$optimize()}} +\item \href{#method-AcqOptimizerLocalSearch-reset}{\code{AcqOptimizerLocalSearch$reset()}} +\item \href{#method-AcqOptimizerLocalSearch-clone}{\code{AcqOptimizerLocalSearch$clone()}} +} +} +\if{html}{\out{ +
Inherited methods + +
+}} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerLocalSearch-new}{}}} +\subsection{Method \code{new()}}{ +Creates a new instance of this \link[R6:R6Class]{R6} class. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{AcqOptimizerLocalSearch$new(acq_function = NULL)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{acq_function}}{(\code{NULL} | \link{AcqFunction}).} +} +\if{html}{\out{
}} +} +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerLocalSearch-optimize}{}}} +\subsection{Method \code{optimize()}}{ +Optimize the acquisition function. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{AcqOptimizerLocalSearch$optimize()}\if{html}{\out{
}} +} + +\subsection{Returns}{ +\code{\link[data.table:data.table]{data.table::data.table()}} with 1 row per candidate. +} +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerLocalSearch-reset}{}}} +\subsection{Method \code{reset()}}{ +Reset the acquisition function optimizer. + +Currently not used. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{AcqOptimizerLocalSearch$reset()}\if{html}{\out{
}} +} + +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerLocalSearch-clone}{}}} +\subsection{Method \code{clone()}}{ +The objects of this class are cloneable with this method. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{AcqOptimizerLocalSearch$clone(deep = FALSE)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{deep}}{Whether to make a deep clone.} +} +\if{html}{\out{
}} +} +} +} diff --git a/man/AcqOptimizerRandomSearch.Rd b/man/AcqOptimizerRandomSearch.Rd new file mode 100644 index 00000000..60a07970 --- /dev/null +++ b/man/AcqOptimizerRandomSearch.Rd @@ -0,0 +1,101 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/AcqOptimzerRandomSearch.R +\name{AcqOptimizerRandomSearch} +\alias{AcqOptimizerRandomSearch} +\title{Random Search Acquisition Function Optimizer} +\description{ +Random Search Acquisition Function Optimizer + +Random Search Acquisition Function Optimizer +} +\section{Super class}{ +\code{\link[mlr3mbo:AcqOptimizer]{mlr3mbo::AcqOptimizer}} -> \code{AcqOptimizerRandomSearch} +} +\section{Active bindings}{ +\if{html}{\out{
}} +\describe{ +\item{\code{print_id}}{(\code{character})\cr +Id used when printing.} + +\item{\code{param_set}}{(\link[paradox:ParamSet]{paradox::ParamSet})\cr +Set of hyperparameters.} +} +\if{html}{\out{
}} +} +\section{Methods}{ +\subsection{Public methods}{ +\itemize{ +\item \href{#method-AcqOptimizerRandomSearch-new}{\code{AcqOptimizerRandomSearch$new()}} +\item \href{#method-AcqOptimizerRandomSearch-optimize}{\code{AcqOptimizerRandomSearch$optimize()}} +\item \href{#method-AcqOptimizerRandomSearch-reset}{\code{AcqOptimizerRandomSearch$reset()}} +\item \href{#method-AcqOptimizerRandomSearch-clone}{\code{AcqOptimizerRandomSearch$clone()}} +} +} +\if{html}{\out{ +
Inherited methods + +
+}} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerRandomSearch-new}{}}} +\subsection{Method \code{new()}}{ +Creates a new instance of this \link[R6:R6Class]{R6} class. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{AcqOptimizerRandomSearch$new(acq_function = NULL)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{acq_function}}{(\code{NULL} | \link{AcqFunction}).} +} +\if{html}{\out{
}} +} +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerRandomSearch-optimize}{}}} +\subsection{Method \code{optimize()}}{ +Optimize the acquisition function. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{AcqOptimizerRandomSearch$optimize()}\if{html}{\out{
}} +} + +\subsection{Returns}{ +\code{\link[data.table:data.table]{data.table::data.table()}} with 1 row per candidate. +} +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerRandomSearch-reset}{}}} +\subsection{Method \code{reset()}}{ +Reset the acquisition function optimizer. + +Currently not used. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{AcqOptimizerRandomSearch$reset()}\if{html}{\out{
}} +} + +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerRandomSearch-clone}{}}} +\subsection{Method \code{clone()}}{ +The objects of this class are cloneable with this method. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{AcqOptimizerRandomSearch$clone(deep = FALSE)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{deep}}{Whether to make a deep clone.} +} +\if{html}{\out{
}} +} +} +} diff --git a/man/SurrogateGP.Rd b/man/SurrogateGP.Rd new file mode 100644 index 00000000..cf488404 --- /dev/null +++ b/man/SurrogateGP.Rd @@ -0,0 +1,184 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/SurrogateGP.R +\name{SurrogateGP} +\alias{SurrogateGP} +\title{Surrogate Model Containing a Gaussian Process} +\description{ +Surrogate model containing a single Gaussian Process via \code{\link[DiceKriging:km]{DiceKriging::km()}} from package \CRANpkg{DiceKriging}. +Update and predict methods are inspired from \link[mlr3learners:mlr_learners_regr.km]{mlr3learners::LearnerRegrKM} from package \CRANpkg{mlr3learners}. + +Compared to using \link[mlr3learners:mlr_learners_regr.km]{mlr3learners::LearnerRegrKM} within a \link{SurrogateLearner} the update and predict methods of this class are much more efficient +as they skip many assertions and checks naturally arising when using a \link{SurrogateLearner} wrapping a \link[mlr3learners:mlr_learners_regr.km]{mlr3learners::LearnerRegrKM}. +} +\section{Parameters}{ + +\describe{ +\item{\code{catch_errors}}{\code{logical(1)}\cr +Should errors during updating the surrogate be caught and propagated to the \code{loop_function} which can then handle +the failed acquisition function optimization (as a result of the failed surrogate) appropriately by, e.g., proposing a randomly sampled point for evaluation? +Default is \code{TRUE}. +} +\item{\code{impute_method}}{\code{character(1)}\cr +Method to impute missing values in the case of updating on an asynchronous \link[bbotk:ArchiveAsync]{bbotk::ArchiveAsync} with pending evaluations. +Can be \code{"mean"} to use mean imputation or \code{"random"} to sample values uniformly at random between the empirical minimum and maximum. +Default is \code{"random"}. +} +} +For a description of all other parameters related to \code{\link[DiceKriging:km]{DiceKriging::km()}} directly, see the documentation of \code{\link[DiceKriging:km]{DiceKriging::km()}}. +\itemize{ +\item The predict type hyperparameter "type" defaults to "SK" (simple kriging). +\item The additional hyperparameter \code{nugget.stability} is used to overwrite the +hyperparameter \code{nugget} with \code{nugget.stability * var(y)} before training to +improve the numerical stability. We recommend a value of \code{1e-8}. +\item The additional hyperparameter \code{jitter} can be set to add +\verb{N(0, [jitter])}-distributed noise to the data before prediction to avoid +perfect interpolation. We recommend a value of \code{1e-12}. +} +} + +\examples{ +if (requireNamespace("DiceKriging") & + requireNamespace("rgenoud")) { + library(bbotk) + library(paradox) + + fun = function(xs) { + list(y = xs$x ^ 2) + } + domain = ps(x = p_dbl(lower = -10, upper = 10)) + codomain = ps(y = p_dbl(tags = "minimize")) + objective = ObjectiveRFun$new(fun = fun, domain = domain, codomain = codomain) + + instance = OptimInstanceBatchSingleCrit$new( + objective = objective, + terminator = trm("evals", n_evals = 5)) + + xdt = generate_design_random(instance$search_space, n = 4)$data + + instance$eval_batch(xdt) + + surrogate = SurrogateGP$new(archive = instance$archive) + surrogate$param_set$set_values( + covtype = "matern5_2", + optim.method = "gen", + control = list(trace = FALSE), + nugget.stability = 10^-8 + ) + + surrogate$update() + + surrogate$learner$model +} +} +\section{Super class}{ +\code{\link[mlr3mbo:Surrogate]{mlr3mbo::Surrogate}} -> \code{SurrogateGP} +} +\section{Active bindings}{ +\if{html}{\out{
}} +\describe{ +\item{\code{print_id}}{(\code{character})\cr +Id used when printing.} + +\item{\code{n_learner}}{(\code{integer(1)})\cr +Returns the number of surrogate models.} + +\item{\code{packages}}{(\code{character()})\cr +Set of required packages. +A warning is signaled if at least one of the packages is not installed, but loaded (not attached) later on-demand via \code{\link[=requireNamespace]{requireNamespace()}}.} + +\item{\code{feature_types}}{(\code{character()})\cr +Stores the feature types the surrogate can handle, e.g. \code{"logical"}, \code{"numeric"}, or \code{"factor"}. +A complete list of candidate feature types, grouped by task type, is stored in \code{\link[mlr3:mlr_reflections]{mlr_reflections$task_feature_types}}.} + +\item{\code{properties}}{(\code{character()})\cr +Stores a set of properties/capabilities the surrogate has. +A complete list of candidate properties, grouped by task type, is stored in \code{\link[mlr3:mlr_reflections]{mlr_reflections$learner_properties}}.} + +\item{\code{predict_type}}{(\code{character(1)})\cr +Retrieves the currently active predict type, e.g. \code{"response"}.} +} +\if{html}{\out{
}} +} +\section{Methods}{ +\subsection{Public methods}{ +\itemize{ +\item \href{#method-SurrogateGP-new}{\code{SurrogateGP$new()}} +\item \href{#method-SurrogateGP-predict}{\code{SurrogateGP$predict()}} +\item \href{#method-SurrogateGP-clone}{\code{SurrogateGP$clone()}} +} +} +\if{html}{\out{ +
Inherited methods + +
+}} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-SurrogateGP-new}{}}} +\subsection{Method \code{new()}}{ +Creates a new instance of this \link[R6:R6Class]{R6} class. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{SurrogateGP$new(archive = NULL, cols_x = NULL, col_y = NULL)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{archive}}{(\link[bbotk:Archive]{bbotk::Archive} | \code{NULL})\cr +\link[bbotk:Archive]{bbotk::Archive} of the \link[bbotk:OptimInstance]{bbotk::OptimInstance}.} + +\item{\code{cols_x}}{(\code{character()} | \code{NULL})\cr +Column id's of variables that should be used as features. +By default, automatically inferred based on the archive.} + +\item{\code{col_y}}{(\code{character(1)} | \code{NULL})\cr +Column id of variable that should be used as a target. +By default, automatically inferred based on the archive.} +} +\if{html}{\out{
}} +} +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-SurrogateGP-predict}{}}} +\subsection{Method \code{predict()}}{ +Predict mean response and standard error. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{SurrogateGP$predict(xdt)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{xdt}}{(\code{\link[data.table:data.table]{data.table::data.table()}})\cr +New data. One row per observation.} +} +\if{html}{\out{
}} +} +\subsection{Returns}{ +\code{\link[data.table:data.table]{data.table::data.table()}} with the columns \code{mean} and \code{se}. +} +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-SurrogateGP-clone}{}}} +\subsection{Method \code{clone()}}{ +The objects of this class are cloneable with this method. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{SurrogateGP$clone(deep = FALSE)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{deep}}{Whether to make a deep clone.} +} +\if{html}{\out{
}} +} +} +} diff --git a/man/SurrogateRF.Rd b/man/SurrogateRF.Rd new file mode 100644 index 00000000..368876f5 --- /dev/null +++ b/man/SurrogateRF.Rd @@ -0,0 +1,184 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/SurrogateRF.R +\name{SurrogateRF} +\alias{SurrogateRF} +\title{Surrogate Model Containing a Random Forest} +\description{ +Surrogate model containing a single Random Forest via \code{\link[ranger:ranger]{ranger::ranger()}} from package \CRANpkg{ranger}. +Update and predict methods are inspired from \link[mlr3learners:mlr_learners_regr.ranger]{mlr3learners::LearnerRegrRanger} from package \CRANpkg{mlr3learners}. + +Compared to using \link[mlr3learners:mlr_learners_regr.ranger]{mlr3learners::LearnerRegrRanger} within a \link{SurrogateLearner} the update and predict methods of this class are much more efficient +as they skip many assertions and checks naturally arising when using a \link{SurrogateLearner} wrapping a \link[mlr3learners:mlr_learners_regr.ranger]{mlr3learners::LearnerRegrRanger}. +} +\section{Parameters}{ + +\describe{ +\item{\code{catch_errors}}{\code{logical(1)}\cr +Should errors during updating the surrogate be caught and propagated to the \code{loop_function} which can then handle +the failed acquisition function optimization (as a result of the failed surrogate) appropriately by, e.g., proposing a randomly sampled point for evaluation? +Default is \code{TRUE}. +} +\item{\code{impute_method}}{\code{character(1)}\cr +Method to impute missing values in the case of updating on an asynchronous \link[bbotk:ArchiveAsync]{bbotk::ArchiveAsync} with pending evaluations. +Can be \code{"mean"} to use mean imputation or \code{"random"} to sample values uniformly at random between the empirical minimum and maximum. +Default is \code{"random"}. +} +} +For a description of all other parameters related to \code{\link[DiceKriging:km]{DiceKriging::km()}} directly, see the documentation of \code{\link[DiceKriging:km]{DiceKriging::km()}}. +\itemize{ +\item The predict type hyperparameter "type" defaults to "SK" (simple kriging). +\item The additional hyperparameter \code{nugget.stability} is used to overwrite the +hyperparameter \code{nugget} with \code{nugget.stability * var(y)} before training to +improve the numerical stability. We recommend a value of \code{1e-8}. +\item The additional hyperparameter \code{jitter} can be set to add +\verb{N(0, [jitter])}-distributed noise to the data before prediction to avoid +perfect interpolation. We recommend a value of \code{1e-12}. +} +} + +\examples{ +if (requireNamespace("DiceKriging") & + requireNamespace("rgenoud")) { + library(bbotk) + library(paradox) + + fun = function(xs) { + list(y = xs$x ^ 2) + } + domain = ps(x = p_dbl(lower = -10, upper = 10)) + codomain = ps(y = p_dbl(tags = "minimize")) + objective = ObjectiveRFun$new(fun = fun, domain = domain, codomain = codomain) + + instance = OptimInstanceBatchSingleCrit$new( + objective = objective, + terminator = trm("evals", n_evals = 5)) + + xdt = generate_design_random(instance$search_space, n = 4)$data + + instance$eval_batch(xdt) + + surrogate = SurrogateGP$new(archive = instance$archive) + surrogate$param_set$set_values( + covtype = "matern5_2", + optim.method = "gen", + control = list(trace = FALSE), + nugget.stability = 10^-8 + ) + + surrogate$update() + + surrogate$learner$model +} +} +\section{Super class}{ +\code{\link[mlr3mbo:Surrogate]{mlr3mbo::Surrogate}} -> \code{SurrogateRF} +} +\section{Active bindings}{ +\if{html}{\out{
}} +\describe{ +\item{\code{print_id}}{(\code{character})\cr +Id used when printing.} + +\item{\code{n_learner}}{(\code{integer(1)})\cr +Returns the number of surrogate models.} + +\item{\code{packages}}{(\code{character()})\cr +Set of required packages. +A warning is signaled if at least one of the packages is not installed, but loaded (not attached) later on-demand via \code{\link[=requireNamespace]{requireNamespace()}}.} + +\item{\code{feature_types}}{(\code{character()})\cr +Stores the feature types the surrogate can handle, e.g. \code{"logical"}, \code{"numeric"}, or \code{"factor"}. +A complete list of candidate feature types, grouped by task type, is stored in \code{\link[mlr3:mlr_reflections]{mlr_reflections$task_feature_types}}.} + +\item{\code{properties}}{(\code{character()})\cr +Stores a set of properties/capabilities the surrogate has. +A complete list of candidate properties, grouped by task type, is stored in \code{\link[mlr3:mlr_reflections]{mlr_reflections$learner_properties}}.} + +\item{\code{predict_type}}{(\code{character(1)})\cr +Retrieves the currently active predict type, e.g. \code{"response"}.} +} +\if{html}{\out{
}} +} +\section{Methods}{ +\subsection{Public methods}{ +\itemize{ +\item \href{#method-SurrogateRF-new}{\code{SurrogateRF$new()}} +\item \href{#method-SurrogateRF-predict}{\code{SurrogateRF$predict()}} +\item \href{#method-SurrogateRF-clone}{\code{SurrogateRF$clone()}} +} +} +\if{html}{\out{ +
Inherited methods + +
+}} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-SurrogateRF-new}{}}} +\subsection{Method \code{new()}}{ +Creates a new instance of this \link[R6:R6Class]{R6} class. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{SurrogateRF$new(archive = NULL, cols_x = NULL, col_y = NULL)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{archive}}{(\link[bbotk:Archive]{bbotk::Archive} | \code{NULL})\cr +\link[bbotk:Archive]{bbotk::Archive} of the \link[bbotk:OptimInstance]{bbotk::OptimInstance}.} + +\item{\code{cols_x}}{(\code{character()} | \code{NULL})\cr +Column id's of variables that should be used as features. +By default, automatically inferred based on the archive.} + +\item{\code{col_y}}{(\code{character(1)} | \code{NULL})\cr +Column id of variable that should be used as a target. +By default, automatically inferred based on the archive.} +} +\if{html}{\out{
}} +} +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-SurrogateRF-predict}{}}} +\subsection{Method \code{predict()}}{ +Predict mean response and standard error. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{SurrogateRF$predict(xdt)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{xdt}}{(\code{\link[data.table:data.table]{data.table::data.table()}})\cr +New data. One row per observation.} +} +\if{html}{\out{
}} +} +\subsection{Returns}{ +\code{\link[data.table:data.table]{data.table::data.table()}} with the columns \code{mean} and \code{se}. +} +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-SurrogateRF-clone}{}}} +\subsection{Method \code{clone()}}{ +The objects of this class are cloneable with this method. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{SurrogateRF$clone(deep = FALSE)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{deep}}{Whether to make a deep clone.} +} +\if{html}{\out{
}} +} +} +} diff --git a/man/mlr_learners_regr.ranger.Rd b/man/mlr_learners_regr.ranger.Rd new file mode 100644 index 00000000..f96d5a80 --- /dev/null +++ b/man/mlr_learners_regr.ranger.Rd @@ -0,0 +1,96 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/LearnerRegrRangerMbo.R +\name{mlr_learners_regr.ranger} +\alias{mlr_learners_regr.ranger} +\alias{LearnerRegrRangerMbo} +\title{Custom Ranger Regression Learner for Model Based Optimization} +\description{ +Random regression forest. +Calls \code{\link[ranger:ranger]{ranger::ranger()}} from package \CRANpkg{ranger}. +} +\section{Super classes}{ +\code{\link[mlr3:Learner]{mlr3::Learner}} -> \code{\link[mlr3:LearnerRegr]{mlr3::LearnerRegr}} -> \code{LearnerRegrRangerMbo} +} +\section{Methods}{ +\subsection{Public methods}{ +\itemize{ +\item \href{#method-LearnerRegrRangerMbo-new}{\code{LearnerRegrRangerMbo$new()}} +\item \href{#method-LearnerRegrRangerMbo-importance}{\code{LearnerRegrRangerMbo$importance()}} +\item \href{#method-LearnerRegrRangerMbo-oob_error}{\code{LearnerRegrRangerMbo$oob_error()}} +\item \href{#method-LearnerRegrRangerMbo-clone}{\code{LearnerRegrRangerMbo$clone()}} +} +} +\if{html}{\out{ +
Inherited methods + +
+}} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-LearnerRegrRangerMbo-new}{}}} +\subsection{Method \code{new()}}{ +Creates a new instance of this \link[R6:R6Class]{R6} class. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{LearnerRegrRangerMbo$new()}\if{html}{\out{
}} +} + +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-LearnerRegrRangerMbo-importance}{}}} +\subsection{Method \code{importance()}}{ +The importance scores are extracted from the model slot \code{variable.importance}. +Parameter \code{importance.mode} must be set to \code{"impurity"}, \code{"impurity_corrected"}, or +\code{"permutation"} +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{LearnerRegrRangerMbo$importance()}\if{html}{\out{
}} +} + +\subsection{Returns}{ +Named \code{numeric()}. +} +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-LearnerRegrRangerMbo-oob_error}{}}} +\subsection{Method \code{oob_error()}}{ +The out-of-bag error, extracted from model slot \code{prediction.error}. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{LearnerRegrRangerMbo$oob_error()}\if{html}{\out{
}} +} + +\subsection{Returns}{ +\code{numeric(1)}. +} +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-LearnerRegrRangerMbo-clone}{}}} +\subsection{Method \code{clone()}}{ +The objects of this class are cloneable with this method. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{LearnerRegrRangerMbo$clone(deep = FALSE)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{deep}}{Whether to make a deep clone.} +} +\if{html}{\out{
}} +} +} +} diff --git a/tests/testthat/test_AcqOptimizerCmaes.R b/tests/testthat/test_AcqOptimizerCmaes.R new file mode 100644 index 00000000..3a1e9baf --- /dev/null +++ b/tests/testthat/test_AcqOptimizerCmaes.R @@ -0,0 +1,45 @@ +test_that("AcqOptimizerCmaes works", { + instance = oi(OBJ_1D, terminator = trm("evals", n_evals = 5L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerCmaes$new(acq_function = acqfun) + acqopt$param_set$set_values(max_fevals = 100L) + acqfun$surrogate$update() + acqfun$update() + expect_data_table(acqopt$optimize(), nrows = 1L) + expect_list(acqopt$state) +}) + +test_that("AcqOptimizerCmaes works with 2D", { + instance = oi(OBJ_2D, terminator = trm("evals", n_evals = 5L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerCmaes$new(acq_function = acqfun) + acqopt$param_set$set_values(max_fevals = 100L) + acqfun$surrogate$update() + acqfun$update() + expect_data_table(acqopt$optimize(), nrows = 1L) + expect_list(acqopt$state) + expect_list(acqopt$state) +}) + +test_that("AcqOptimizerCmaes works with instance", { + instance = oi(OBJ_1D, terminator = trm("evals", n_evals = 10L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerCmaes$new(acq_function = acqfun) + acqopt$param_set$set_values(max_fevals = 100L) + + optimizer = opt("mbo", acq_optimizer = acqopt, acq_function = acqfun, surrogate = surrogate) + expect_data_table(optimizer$optimize(instance), nrow = 1L) +}) + diff --git a/tests/testthat/test_AcqOptimizerDirect.R b/tests/testthat/test_AcqOptimizerDirect.R new file mode 100644 index 00000000..3124d9d2 --- /dev/null +++ b/tests/testthat/test_AcqOptimizerDirect.R @@ -0,0 +1,70 @@ +test_that("AcqOptimizerDirect works", { + instance = oi(OBJ_1D, terminator = trm("evals", n_evals = 5L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerDirect$new(acq_function = acqfun) + acqopt$param_set$set_values(maxeval = 200L, restart_strategy = "none") + acqfun$surrogate$update() + acqfun$update() + + expect_data_table(acqopt$optimize(), nrows = 1L) + expect_list(acqopt$state) + expect_names(names(acqopt$state), must.include = "iteration_1") + expect_class(acqopt$state$iteration_1$model, "nloptr") + expect_true(acqopt$state$iteration_1$model$iterations <= 200L) +}) + +test_that("AcqOptimizerDirect works with 2D", { + instance = oi(OBJ_2D, terminator = trm("evals", n_evals = 5L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerDirect$new(acq_function = acqfun) + acqopt$param_set$set_values(maxeval = 200L) + acqfun$surrogate$update() + acqfun$update() + + expect_data_table(acqopt$optimize(), nrows = 1L) + expect_list(acqopt$state) + expect_names(names(acqopt$state), must.include = "iteration_1") + expect_class(acqopt$state$iteration_1$model, "nloptr") + expect_true(acqopt$state$iteration_1$model$iterations <= 200L) +}) + +test_that("AcqOptimizerDirect works with instance", { + instance = oi(OBJ_1D, terminator = trm("evals", n_evals = 10L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerDirect$new(acq_function = acqfun) + acqopt$param_set$set_values(maxeval = 10L) + + optimizer = opt("mbo", acq_optimizer = acqopt, acq_function = acqfun, surrogate = surrogate) + expect_data_table(optimizer$optimize(instance), nrow = 1L) +}) + +test_that("AcqOptimizerDirect works with random restart", { + instance = oi(OBJ_2D, terminator = trm("evals", n_evals = 5L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerDirect$new(acq_function = acqfun) + acqopt$param_set$set_values(maxeval = -1, ftol_rel = 1e-6, restart_strategy = "random", max_restarts = 3L) + acqfun$surrogate$update() + acqfun$update() + + expect_data_table(acqopt$optimize(), nrows = 1L) + expect_list(acqopt$state, min.len = 4L) + expect_names(names(acqopt$state), must.include = c("iteration_1", "iteration_2", "iteration_3", "iteration_4")) + walk(acqopt$state, function(x) expect_class(x$model, "nloptr")) + expect_true(all(sapply(acqopt$state, function(x) x$model$iterations) <= 50L)) +}) diff --git a/tests/testthat/test_AcqOptimizerLbfgsb.R b/tests/testthat/test_AcqOptimizerLbfgsb.R new file mode 100644 index 00000000..55edcc7e --- /dev/null +++ b/tests/testthat/test_AcqOptimizerLbfgsb.R @@ -0,0 +1,70 @@ +test_that("AcqOptimizerLbfgsb works", { + instance = oi(OBJ_1D, terminator = trm("evals", n_evals = 5L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerLbfgsb$new(acq_function = acqfun) + acqopt$param_set$set_values(maxeval = 200L, restart_strategy = "none") + acqfun$surrogate$update() + acqfun$update() + + expect_data_table(acqopt$optimize(), nrows = 1L) + expect_list(acqopt$state) + expect_names(names(acqopt$state), must.include = "iteration_1") + expect_class(acqopt$state$iteration_1$model, "nloptr") + expect_true(acqopt$state$iteration_1$model$iterations <= 200L) +}) + +test_that("AcqOptimizerLbfgsb works with 2D", { + instance = oi(OBJ_2D, terminator = trm("evals", n_evals = 5L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerLbfgsb$new(acq_function = acqfun) + acqopt$param_set$set_values(maxeval = 200L) + acqfun$surrogate$update() + acqfun$update() + + expect_data_table(acqopt$optimize(), nrows = 1L) + expect_list(acqopt$state) + expect_names(names(acqopt$state), must.include = "iteration_1") + expect_class(acqopt$state$iteration_1$model, "nloptr") + expect_true(acqopt$state$iteration_1$model$iterations <= 200L) +}) + +test_that("AcqOptimizerLbfgsb works with instance", { + instance = oi(OBJ_1D, terminator = trm("evals", n_evals = 10L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerLbfgsb$new(acq_function = acqfun) + acqopt$param_set$set_values(maxeval = 10L) + + optimizer = opt("mbo", acq_optimizer = acqopt, acq_function = acqfun, surrogate = surrogate) + expect_data_table(optimizer$optimize(instance), nrow = 1L) +}) + +test_that("AcqOptimizerLbfgsb works with random restart", { + instance = oi(OBJ_2D, terminator = trm("evals", n_evals = 5L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerLbfgsb$new(acq_function = acqfun) + acqopt$param_set$set_values(maxeval = -1, ftol_rel = 1e-6, restart_strategy = "random", max_restarts = 3L) + acqfun$surrogate$update() + acqfun$update() + + expect_data_table(acqopt$optimize(), nrows = 1L) + expect_list(acqopt$state, min.len = 4L) + expect_names(names(acqopt$state), must.include = c("iteration_1", "iteration_2", "iteration_3", "iteration_4")) + walk(acqopt$state, function(x) expect_class(x$model, "nloptr")) + expect_true(all(sapply(acqopt$state, function(x) x$model$iterations) <= 50L)) +}) diff --git a/tests/testthat/test_AcqOptimizerLocalSearch.R b/tests/testthat/test_AcqOptimizerLocalSearch.R new file mode 100644 index 00000000..bc3e4f56 --- /dev/null +++ b/tests/testthat/test_AcqOptimizerLocalSearch.R @@ -0,0 +1,45 @@ +test_that("AcqOptimizerLocalSearch works", { + instance = oi(OBJ_1D, terminator = trm("evals", n_evals = 5L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerLocalSearch$new(acq_function = acqfun) + acqfun$surrogate$update() + acqfun$update() + + res = acqopt$optimize() + expect_data_table(res, nrows = 1L, ncols = 2L) + expect_names(names(res), must.include = c("x", "acq_ei")) +}) + +test_that("AcqOptimizerLocalSearch works with 2D", { + instance = oi(OBJ_2D, terminator = trm("evals", n_evals = 5L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerLocalSearch$new(acq_function = acqfun) + acqfun$surrogate$update() + acqfun$update() + + res = acqopt$optimize() + expect_data_table(res, nrows = 1L, ncols = 3L) + expect_names(names(res), must.include = c("x1", "x2", "acq_ei")) +}) + +test_that("AcqOptimizerLocalSearch works with instance", { + instance = oi(OBJ_1D, terminator = trm("evals", n_evals = 10L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerLocalSearch$new(acq_function = acqfun) + + optimizer = opt("mbo", acq_optimizer = acqopt, acq_function = acqfun, surrogate = surrogate) + expect_data_table(optimizer$optimize(instance), nrow = 1L) +}) + diff --git a/tests/testthat/test_AcqOptimizerRandomSearch.R b/tests/testthat/test_AcqOptimizerRandomSearch.R new file mode 100644 index 00000000..5d30ae20 --- /dev/null +++ b/tests/testthat/test_AcqOptimizerRandomSearch.R @@ -0,0 +1,70 @@ +test_that("AcqOptimizerRandomSearch works", { + instance = oi(OBJ_1D, terminator = trm("evals", n_evals = 5L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerRandomSearch$new(acq_function = acqfun) + acqopt$param_set$set_values(max_fevals = 200L) + acqfun$surrogate$update() + acqfun$update() + + expect_data_table(acqopt$optimize(), nrows = 1L) + expect_list(acqopt$state) + expect_names(names(acqopt$state), must.include = "iteration_1") + expect_class(acqopt$state$iteration_1$model, "nloptr") + expect_true(acqopt$state$iteration_1$model$iterations <= 200L) +}) + +test_that("AcqOptimizerLbfgsb works with 2D", { + instance = oi(OBJ_2D, terminator = trm("evals", n_evals = 5L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerLbfgsb$new(acq_function = acqfun) + acqopt$param_set$set_values(maxeval = 200L) + acqfun$surrogate$update() + acqfun$update() + + expect_data_table(acqopt$optimize(), nrows = 1L) + expect_list(acqopt$state) + expect_names(names(acqopt$state), must.include = "iteration_1") + expect_class(acqopt$state$iteration_1$model, "nloptr") + expect_true(acqopt$state$iteration_1$model$iterations <= 200L) +}) + +test_that("AcqOptimizerLbfgsb works with instance", { + instance = oi(OBJ_1D, terminator = trm("evals", n_evals = 10L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerLbfgsb$new(acq_function = acqfun) + acqopt$param_set$set_values(maxeval = 10L) + + optimizer = opt("mbo", acq_optimizer = acqopt, acq_function = acqfun, surrogate = surrogate) + expect_data_table(optimizer$optimize(instance), nrow = 1L) +}) + +test_that("AcqOptimizerLbfgsb works with random restart", { + instance = oi(OBJ_2D, terminator = trm("evals", n_evals = 5L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerLbfgsb$new(acq_function = acqfun) + acqopt$param_set$set_values(maxeval = 200L, restart_strategy = "random", n_restarts = 3L) + acqfun$surrogate$update() + acqfun$update() + + expect_data_table(acqopt$optimize(), nrows = 1L) + expect_list(acqopt$state, min.len = 4L) + expect_names(names(acqopt$state), must.include = c("iteration_1", "iteration_2", "iteration_3", "iteration_4")) + walk(acqopt$state, function(x) expect_class(x$model, "nloptr")) + expect_true(all(sapply(acqopt$state, function(x) x$model$iterations) <= 50L)) +})