From 42c02309874f5f806b8f59440dd33e6df7d144b3 Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Tue, 7 Jun 2022 09:35:52 +0200 Subject: [PATCH 001/126] push latest so_config attic --- .gitignore | 1 + attic/so_config/analyze.R | 17 ++++++++++++----- attic/so_config/instances.rds | Bin 1377 -> 0 bytes attic/so_config/min_max.R | 7 ++++--- 4 files changed, 17 insertions(+), 8 deletions(-) delete mode 100644 attic/so_config/instances.rds diff --git a/.gitignore b/.gitignore index 4c780408..80d03c8f 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,4 @@ doc Meta attic/so_config/fanova.py attic/so_config/Pipfile +attic/so_config/*.rds diff --git a/attic/so_config/analyze.R b/attic/so_config/analyze.R index 3db606e9..5218d764 100644 --- a/attic/so_config/analyze.R +++ b/attic/so_config/analyze.R @@ -9,8 +9,19 @@ library(GGally) x1 = readRDS("ac_instance_1.rds") x2 = readRDS("ac_instance_2.rds") x3 = readRDS("ac_instance_3.rds") +x4 = readRDS("ac_instance_4.rds") +x5 = readRDS("ac_instance_5.rds") + +data = rbind(x1$archive$data, x2$archive$data, x3$archive$data, x4$archive$data, x5$archive$data)[, c(x1$archive$cols_x, x1$archive$cols_y), with = FALSE] +data[is.na(random_interleave_iter), random_interleave_iter := 0] +data[is.na(num.random.splits), num.random.splits := 0] +data[is.na(lambda), lambda := 0] +data[is.na(fs_behavior), fs_behavior := "none"] +chars = c("init", "splitrule", "acqf", "acqopt", "fs_behavior") +data[, (chars) := lapply(.SD, as.factor), .SDcols = chars] + +lmx = lm(mean_perf ~ init * init_size_factor + random_interleave * random_interleave_iter + num.trees + splitrule * num.random.splits + acqf * lambda + acqopt_iter_factor * acqopt * fs_behavior, data = data) -data = rbind(x1$archive$data, x2$archive$data, x3$archive$data)[, c(x1$archive$cols_x, x1$archive$cols_y), with = FALSE] task = TaskRegr$new("mbo", backend = data, target = "mean_perf") learner = default_surrogate(x1)$model learner$param_set$values$regr.ranger.importance = "permutation" @@ -40,10 +51,6 @@ best[, acqf := as.factor(acqf)] best[, acqopt := as.factor(acqopt)] best[, fs_behavior := as.factor(fs_behavior)] -ggparcoord(data = data, columns = c(1, 2, 4, 6, 8, 9), - alphaLines = 0.2, - groupColumn = "top5", - order = "Outlying") write.table(task$data(), "tmp.csv") data = read.table("tmp.csv") diff --git a/attic/so_config/instances.rds b/attic/so_config/instances.rds deleted file mode 100644 index 0279351efe9be4d373fe63ff3853593f6d739820..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1377 zcmV-n1)lmJiwFP!000001MOCAOjB1FzP3>Ln2O?Dgc(5=(PT7jZA*)|SEE*@!70Is z;E+qXLZRGZTk&IPK%#CsS;VnSjLea2ZeTz}K_;+5l@E0U3-V2wKt*7s%qXG@vi4j` z3u76{bo;X(H_v&W^PJ~A=k3QmK@J#((J`ADbc}9m9AdWH0!3Z|Bj;=^?2M0-@$tZH zFnc4~%fsV2yYLv2e1(J~A|j=+B%yq5lpH4`3HfWF94IOdmn&Zjqu3iaX|I3I*ilS; z>?^9@K=uB}aG6Y@eC6E4ub?fL#T5$23L7@pn?E+5VII)foN+Ci&0|>>3cW20c`UZo z1fOr^D6pCkaR1~==kf#=8#c>o%ri??u)P1ma6E#=kj4>A8H>&4kMF+1dx~wWx2JJA zQ|vhDoR-c_h+>S(n(E9GL=)O%iA>{gCVfJ5TrB>7_@4|hY_{oxSp89=>8~97Tl{dE z7^a#Yh9pi8<1SARr+W@!^|TNh>ks5*9XAzl(>K(V94=+|xtTC8BG{w^)e|k^3uCS! zzUbWj3nnble24go+J%&Eqz3sk4}~FJqi@PDChWYi0O<;BpQClx=@m$KDh4SMY=+UBl5o#Ku4J6 z$3vKY*9Bp5Wi|52=SHD*MF-0eUg8aH4!$sP%h>P=5XA%PIeSU&>E1NbOs)fkN+ID%Y=Ss?N!#a>o(W zx)(bs937_kthLBzkZKATUJ5Yl{^%-dle!AipvSdqKDR#|bPh>jNy~(wW4N7gT9ye9 zePvx<-*`YfR+^Dmxe{94T#vLQz7OrRl+}H_Gw@*5)6>_kJA<}#_S5sr+h8Dw!C94? z0t0EHz%^J83@lE4_B&Pr56|WAzj#;&j~t|(w>9mcJFFZWKDGlo2myB_JqdUckh@)kl_z4tV`QMzI+dw6yoX^tsc<4cq{G5wI$G?+xR5nfDJT;&p!Bg zraye)#GP~X#%9RAox#7X$%Jg_;Qf-eQ&2v*r&f&XpyWc`{8_3ipxGceyRIAuO}^KN zN6!wZ`@l0VCnp*-=dv>k7*FBOs3UKESvORcev+`WHwJzb2nXW5+@V%fw0Y&4IiT+O zSsb$66>95Gyct?t1@&(g%eN#gg62*aeQ#7MG|(afecU{uWn)@vONM4=2}83lsG9S#%hKVmq(6YZW-ZwBP Date: Tue, 5 Jul 2022 23:01:15 +0200 Subject: [PATCH 002/126] new so_config --- attic/so_config/run.R | 18 ++++++++++-------- attic/so_config/submit.sh | 2 +- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/attic/so_config/run.R b/attic/so_config/run.R index 4cd4c7ff..ee629bfb 100755 --- a/attic/so_config/run.R +++ b/attic/so_config/run.R @@ -41,7 +41,7 @@ search_space = ps( num.random.splits = p_int(lower = 1L, upper = 10L, depends = splitrule == "extratrees"), acqf = p_fct(c("EI", "CB", "PI")), lambda = p_dbl(lower = 0, upper = 3L, depends = acqf == "CB"), - acqopt_iter_factor = p_int(lower = 10L, upper = 50L), + acqopt_iter_factor = p_int(lower = 20L, upper = 60L), # lowest acqopt_iter is 20 * 90 for rbv2_gl,met acqopt = p_fct(c("RS", "FS")), # FIXME: miesmuschel fs_behavior = p_fct(c("global", "local"), depends = acqopt == "FS") ) @@ -111,24 +111,26 @@ evaluate = function(xdt, instance) { } acq_budget = optim_instance$terminator$param_set$values$n_evals * xdt$acqopt_iter_factor + batch_size = optim_instance$terminator$param_set$values$n_evals acq_optimizer = if (xdt$acqopt == "RS") { - AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = acq_budget)) + AcqOptimizer$new(opt("random_search", batch_size = batch_size), terminator = trm("evals", n_evals = acq_budget)) } else if (xdt$acqopt == "FS") { if (xdt$fs_behavior == "global") { n_repeats = 10L - maxit = ceiling(((acq_budget / n_repeats) - 1000L) / 1000L) + maxit = floor(((acq_budget / n_repeats) - batch_size) / batch_size) } else if (xdt$fs_behavior == "local") { - n_repeats = 3L - maxit = ceiling(((acq_budget / n_repeats) - 1000L) / 1000L) + n_repeats = 1L + maxit = floor(((acq_budget / n_repeats) - batch_size) / batch_size) } - AcqOptimizer$new(opt("focus_search", n_points = 1000L, maxit = maxit), terminator = trm("evals", n_evals = acq_budget)) + AcqOptimizer$new(opt("focus_search", n_points = batch_size, maxit = maxit), terminator = trm("evals", n_evals = acq_budget)) } bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) best = optim_instance$archive$best()[[instance$target]] - (best - instance$mean) / instance$sd # normalize best w.r.t min_max.R empirical mean and sd + # (best - instance$mean) / instance$sd # normalize best w.r.t min_max.R empirical mean and sd + instance$ecdf(best) # evaluate the precomputed ecdf for the best value found; our target is effectively P(X <= best) } # FIXME: maybe add that the objective stores its state on disk after each eval @@ -148,7 +150,7 @@ objective = ObjectiveRFunDt$new( ac_instance = OptimInstanceSingleCrit$new( objective = objective, - terminator = trm("evals", n_evals = 230L) # 30 init design + 200 + terminator = trm("evals", n_evals = 280L) # 30 init design + 250 ) surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% lrn("regr.ranger", num.trees = 2000L, keep.inbag = TRUE))) diff --git a/attic/so_config/submit.sh b/attic/so_config/submit.sh index e3f0154f..0069211b 100644 --- a/attic/so_config/submit.sh +++ b/attic/so_config/submit.sh @@ -2,7 +2,7 @@ sbatch < Date: Wed, 6 Jul 2022 11:06:16 +0200 Subject: [PATCH 003/126] fix: always add dropped columns to xdt during surrogate predict --- R/SurrogateLearner.R | 4 +++- R/SurrogateLearners.R | 4 +++- R/helper.R | 12 ++++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/R/SurrogateLearner.R b/R/SurrogateLearner.R index 88d91976..36defa0b 100644 --- a/R/SurrogateLearner.R +++ b/R/SurrogateLearner.R @@ -46,8 +46,10 @@ SurrogateLearner = R6Class("SurrogateLearner", #' @return [data.table::data.table()] with the columns `mean` and `se`. predict = function(xdt) { assert_xdt(xdt) + xdt = fix_xdt_missing(xdt, archive = self$archive) + xdt = char_to_fct(xdt) - pred = self$model$predict_newdata(newdata = char_to_fct(xdt)) + pred = self$model$predict_newdata(newdata = xdt) if (self$model$predict_type == "se") { data.table(mean = pred$response, se = pred$se) } else { diff --git a/R/SurrogateLearners.R b/R/SurrogateLearners.R index aa0075c3..c99bcae3 100644 --- a/R/SurrogateLearners.R +++ b/R/SurrogateLearners.R @@ -55,9 +55,11 @@ SurrogateLearners = R6Class("SurrogateLearners", #' @return list of [data.table::data.table()]s with the columns `mean` and `se`. predict = function(xdt) { assert_xdt(xdt) + xdt = fix_xdt_missing(xdt, archive = self$archive) + xdt = char_to_fct(xdt) preds = lapply(self$model, function(model) { - pred = model$predict_newdata(newdata = char_to_fct(xdt)) + pred = model$predict_newdata(newdata = xdt) if (model$predict_type == "se") { data.table(mean = pred$response, se = pred$se) } else { diff --git a/R/helper.R b/R/helper.R index 25099401..ca7bd8a1 100644 --- a/R/helper.R +++ b/R/helper.R @@ -40,6 +40,18 @@ archive_x = function(archive) { archive$data[, archive$cols_x, with = FALSE] } +# durring surrogate prediction it may have happened the whole columns where dropped (e.g., during focussearch if the search space was shrinked) +fix_xdt_missing = function(xdt, archive) { + missing = archive$cols_x[archive$cols_x %nin% colnames(xdt)] + types = map_chr(missing, function(x) typeof(archive$data[[x]])) + NA_types = list(double = NA_real_, integer = NA_integer_, character = NA_character_)[types] + for (i in seq_along(missing)) { + xdt[, eval(missing[i]) := NA_types[i]] + } + assert_set_equal(colnames(xdt), archive$cols_x) + xdt +} + get_gower_dist = function(x, y = NULL) { # if y is NULL we get the Gower distance of x pairwise with itself and set the diagonal to ones # otherwise we get the Gower dist_threshold of x pairwise with y From 483b643712820835850efc11f10b17908549ecf6 Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Wed, 6 Jul 2022 11:06:44 +0200 Subject: [PATCH 004/126] new so_config --- attic/so_config/run.R | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/attic/so_config/run.R b/attic/so_config/run.R index ee629bfb..51f3e0ed 100755 --- a/attic/so_config/run.R +++ b/attic/so_config/run.R @@ -41,7 +41,7 @@ search_space = ps( num.random.splits = p_int(lower = 1L, upper = 10L, depends = splitrule == "extratrees"), acqf = p_fct(c("EI", "CB", "PI")), lambda = p_dbl(lower = 0, upper = 3L, depends = acqf == "CB"), - acqopt_iter_factor = p_int(lower = 20L, upper = 60L), # lowest acqopt_iter is 20 * 90 for rbv2_gl,met + acqopt_iter_factor = p_int(lower = 1L, upper = 20L), # lowest acqopt_iter is 1000 * 1 acqopt = p_fct(c("RS", "FS")), # FIXME: miesmuschel fs_behavior = p_fct(c("global", "local"), depends = acqopt == "FS") ) @@ -110,18 +110,19 @@ evaluate = function(xdt, instance) { AcqFunctionPI$new() } - acq_budget = optim_instance$terminator$param_set$values$n_evals * xdt$acqopt_iter_factor - batch_size = optim_instance$terminator$param_set$values$n_evals + acq_budget = 1000 * xdt$acqopt_iter_factor acq_optimizer = if (xdt$acqopt == "RS") { - AcqOptimizer$new(opt("random_search", batch_size = batch_size), terminator = trm("evals", n_evals = acq_budget)) + AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = acq_budget)) } else if (xdt$acqopt == "FS") { if (xdt$fs_behavior == "global") { n_repeats = 10L - maxit = floor(((acq_budget / n_repeats) - batch_size) / batch_size) + maxit = 2L + batch_size = ceiling((acq_budget / n_repeats) / (1 + maxit)) } else if (xdt$fs_behavior == "local") { - n_repeats = 1L - maxit = floor(((acq_budget / n_repeats) - batch_size) / batch_size) + n_repeats = 2L + maxit = 10L + batch_size = ceiling((acq_budget / n_repeats) / (1 + maxit)) } AcqOptimizer$new(opt("focus_search", n_points = batch_size, maxit = maxit), terminator = trm("evals", n_evals = acq_budget)) } @@ -130,7 +131,9 @@ evaluate = function(xdt, instance) { best = optim_instance$archive$best()[[instance$target]] # (best - instance$mean) / instance$sd # normalize best w.r.t min_max.R empirical mean and sd - instance$ecdf(best) # evaluate the precomputed ecdf for the best value found; our target is effectively P(X <= best) + ecdf_best = instance$ecdf(best) # evaluate the precomputed ecdf for the best value found; our target is effectively P(X <= best) + cat("scenario:", instance$scenario, "instance:", instance$instance, "ECDF_best:", ecdf_best, "\n") + ecdf_best } # FIXME: maybe add that the objective stores its state on disk after each eval @@ -150,7 +153,7 @@ objective = ObjectiveRFunDt$new( ac_instance = OptimInstanceSingleCrit$new( objective = objective, - terminator = trm("evals", n_evals = 280L) # 30 init design + 250 + terminator = trm("evals", n_evals = 230L) # 30 init design + 200 ) surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% lrn("regr.ranger", num.trees = 2000L, keep.inbag = TRUE))) @@ -158,6 +161,7 @@ acq_function = AcqFunctionEI$new() acq_optimizer = AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 10000L)) design = generate_design_lhs(ac_instance$search_space, n = 30L)$data ac_instance$eval_batch(design) +saveRDS(ac_instance, paste0("ac_instance_", run_id, ".rds")) bayesopt_ego(ac_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = 5L) saveRDS(ac_instance, paste0("ac_instance_", run_id, ".rds")) From 86e3c3f51e62810fed874f17fcca8fd9f8e81def Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Wed, 6 Jul 2022 11:36:03 +0200 Subject: [PATCH 005/126] use current bbotk main branch --- attic/so_config/run.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/attic/so_config/run.R b/attic/so_config/run.R index 7ba1d817..327a5aa3 100755 --- a/attic/so_config/run.R +++ b/attic/so_config/run.R @@ -6,7 +6,7 @@ library(mlr3) library(mlr3misc) library(mlr3learners) library(mlr3pipelines) -library(bbotk) # @focussearch +library(bbotk) library(paradox) library(mlr3mbo) # @so_config reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) From 9dac61996d8210ba60ad3f2814625a40cbd2fc7b Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Thu, 25 Aug 2022 21:06:11 +0200 Subject: [PATCH 006/126] LearnerRegrRangerCustom --- DESCRIPTION | 4 +- NAMESPACE | 2 + R/LearnerRegrRangerCustom.R | 205 +++++++++++++++++++++++++ R/helper.R | 14 +- R/zzz.R | 4 + man-roxygen/learner.R | 14 ++ man-roxygen/seealso_learner.R | 17 ++ man/mlr_learners_regr.ranger_custom.Rd | 200 ++++++++++++++++++++++++ 8 files changed, 457 insertions(+), 3 deletions(-) create mode 100644 R/LearnerRegrRangerCustom.R create mode 100644 man-roxygen/learner.R create mode 100644 man-roxygen/seealso_learner.R create mode 100644 man/mlr_learners_regr.ranger_custom.Rd diff --git a/DESCRIPTION b/DESCRIPTION index d01d786e..304c3a00 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -2,7 +2,7 @@ Type: Package Package: mlr3mbo Title: Flexible Bayesian Optimization in R Version: 0.0.0.9000 -Authors@R:c( +Authors@R: c( person("Lennart", "Schneider", , "lennart.sch@web.de", role = c("cre", "aut"), comment = c(ORCID = "0000-0003-4152-5308")), person("Jakob", "Richter", , "jakob1richter@gmail.com", role = "aut", @@ -37,6 +37,7 @@ Imports: Suggests: DiceKriging, lhs, + ggplot2, mlr3learners, mlr3pipelines, ranger, @@ -60,6 +61,7 @@ Collate: 'AcqFunctionPI.R' 'AcqFunctionSmsEgo.R' 'AcqOptimizer.R' + 'LearnerRegrRangerCustom.R' 'OptimizerMbo.R' 'Surrogate.R' 'SurrogateLearner.R' diff --git a/NAMESPACE b/NAMESPACE index e06e4847..ee0aa93f 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,5 +1,6 @@ # Generated by roxygen2: do not edit by hand +S3method(default_values,LearnerRegrRangerCustom) export(AcqFunction) export(AcqFunctionAEI) export(AcqFunctionCB) @@ -8,6 +9,7 @@ export(AcqFunctionEIPS) export(AcqFunctionPI) export(AcqFunctionSmsEgo) export(AcqOptimizer) +export(LearnerRegrRangerCustom) export(OptimizerMbo) export(Surrogate) export(SurrogateLearner) diff --git a/R/LearnerRegrRangerCustom.R b/R/LearnerRegrRangerCustom.R new file mode 100644 index 00000000..282bc8f6 --- /dev/null +++ b/R/LearnerRegrRangerCustom.R @@ -0,0 +1,205 @@ +#' @title Custom Ranger Regression Learner +#' +#' @name mlr_learners_regr.ranger_custom +#' +#' @description +#' Custom random regression forest. +#' Calls [ranger::ranger()] from package \CRANpkg{ranger}. +#' This custom random regression forest tries to mimic the behaviour of a Gaussian process and therefore may be more suitable for Bayesian optimization compared to a normal random regression forest. +#' +#' The `"extratrees"` splitting rule is used with `num.random.splits = 1`. +#' This results in a rather smooth prediction function. +#' No bootstrap or subsampling is used but rather all training data (`replace = FALSE` and `sample.fraction = 1`). +#' This may seem counter-intuitive at first but ensures that training data is almost perfectly interpolated (given `min.node.size = 1`). +#' Per default, `1000` trees are used. +#' Classical standard error estimation is no longer possible, but the `"extratrees"` splitting rule allows for estimating the variance based on the variance of the trees (simply taking the variance over the predictions of the trees). +#' This results in low variance close to training data points. +#' If `se.simple.spatial = TRUE` this point-wise variance is upwardly biased by adding the pointwise variance again scaled by the minimum Gower distance of the point to the training data (recall that a Gower distance of 0 means an identical point is present in the training data, whereas a distance of 1 means total dissimilarity to the training data). +#' This ensures that the variance is larger in regions dissimilar to the training data. +#' Summarizing, the standard error for a point is calculated as follows: +#' \deqn{\hat{\mathrm{SE}}(x_{i}) = \sqrt{(\hat{\mathrm{VAR}}(x_{i}) + \epsilon) + (\hat{\mathrm{VAR}}(x_{i}) + \epsilon) * \mathrm{GD}_{i})}} +#' Here, \eqn{\epsilon} is given by `se.simple.spatial.nugget` which can be set larger than 0 in the case of noisy function observations. +#' +#' @templateVar id regr.ranger_custom +#' @template learner +#' +#' @export +#' @template seealso_learner +#' @examples +#' library(data.table) +#' library(mlr3) +#' x = seq(-5, 5, length.out = 1001) +#' dat = data.table(x = c(-5, -2.5, -1, 0, 1, 2.5, 5), y = c(0, -0.1, 0.3, -1, 0.3, -0.1, 0)) +#' task = TaskRegr$new("example", backend = dat, target = "y") +#' learner = lrn("regr.ranger_custom") +#' learner$predict_type = "se" +#' learner$train(task) +#' predictions = learner$predict_newdata(data.table(x = x)) +#' if (requireNamespace("ggplot2")) { +#' library(ggplot2) +#' ggplot(aes(x = x, y = response), data = cbind(x, as.data.table(predictions))) + +#' geom_line() + +#' geom_ribbon(aes(min = response - se, max = response + se), alpha = 0.1) + +#' geom_point(aes(x = x, y = y), data = task$data()) + +#' labs(x = "x", y = "Prediction") + +#' theme_minimal() +#' } +LearnerRegrRangerCustom = R6Class("LearnerRegrRangerCustom", + inherit = mlr3::LearnerRegr, + + public = list( + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + initialize = function() { + ps = ps( + always.split.variables = p_uty(tags = "train"), + holdout = p_lgl(default = FALSE, tags = "train"), + importance = p_fct(c("none", "impurity", "impurity_corrected", "permutation"), tags = "train"), + keep.inbag = p_lgl(default = FALSE, tags = "train"), + local.importance = p_lgl(default = FALSE, tags = "train"), + max.depth = p_int(default = NULL, lower = 0L, special_vals = list(NULL), tags = "train"), + mtry = p_int(lower = 1L, special_vals = list(NULL), tags = "train"), + mtry.ratio = p_dbl(lower = 0, upper = 1, tags = "train"), + num.threads = p_int(1L, default = 1L, tags = c("train", "predict", "threads")), # changed + num.trees = p_int(1L, default = 1000L, tags = c("train", "predict", "hotstart")), # changed + oob.error = p_lgl(default = FALSE, tags = "train"), # changed + regularization.factor = p_uty(default = 1, tags = "train"), + regularization.usedepth = p_lgl(default = FALSE, tags = "train"), + respect.unordered.factors = p_fct(c("ignore", "order", "partition"), default = "ignore", tags = "train"), + save.memory = p_lgl(default = FALSE, tags = "train"), + scale.permutation.importance = p_lgl(default = FALSE, tags = "train"), + se.simple.spatial = p_lgl(default = TRUE, tags = "predict"), + se.simple.spatial.nugget = p_dbl(0, default = 0, tags = "predict"), + seed = p_int(default = NULL, special_vals = list(NULL), tags = c("train", "predict")), + split.select.weights = p_uty(default = NULL, tags = "train"), + verbose = p_lgl(default = TRUE, tags = c("train", "predict")), + write.forest = p_lgl(default = TRUE, tags = "train") + ) + ps$values = list(num.trees = 1000L, oob.error = FALSE, num.threads = 1L) + + # deps + ps$add_dep("se.simple.spatial.nugget", "se.simple.spatial", CondEqual$new(TRUE)) + + super$initialize( + id = "regr.ranger_custom", + param_set = ps, + predict_types = c("response", "se"), + feature_types = c("logical", "integer", "numeric", "character", "factor", "ordered"), + properties = c("weights", "importance", "oob_error", "hotstart_backward"), + packages = c("mlr3mbo", "ranger"), + man = "mlr3learners::mlr_learners_regr.ranger_custom" + ) + }, + + #' @description + #' The importance scores are extracted from the model slot `variable.importance`. + #' Parameter `importance.mode` must be set to `"impurity"`, `"impurity_corrected"`, or + #' `"permutation"` + #' + #' @return Named `numeric()`. + importance = function() { + if (is.null(self$model)) { + stopf("No model stored") + } + if (self$model$importance.mode == "none") { + stopf("No importance stored") + } + + sort(self$model$variable.importance, decreasing = TRUE) + }, + + #' @description + #' The out-of-bag error, extracted from model slot `prediction.error`. + #' + #' @return `numeric(1)`. + oob_error = function() { + if (is.null(self$model)) { + stopf("No model stored") + } + self$model$prediction.error + } + ), + + private = list( + .train = function(task) { + pv = self$param_set$get_values(tags = "train") + pv = convert_ratio_ranger(pv, "mtry", "mtry.ratio", length(task$feature_names)) + pv = insert_named(list(min.node.size = 1L, replace = FALSE, sample.fraction = 1L, splitrule = "extratrees", num.random.splits = 1L), pv) + + private$.train_task = task$clone(deep = TRUE) # required for se.method == "simple" and se.simple.spatial == TRUE + + invoke(ranger::ranger, dependent.variable.name = task$target_names, data = task$data(), case.weights = task$weights$weight, .args = pv) + }, + + .predict = function(task) { + pv = self$param_set$get_values(tags = "predict") + pv = insert_named(list(se.simple.spatial = TRUE, se.simple.spatial.nugget = 0), pv) + newdata = ordered_features(task, self) + + prediction = invoke(predict, self$model, data = newdata, type = "response", predict.all = TRUE, .args = pv) + response = apply(prediction$predictions, MARGIN = 1L, FUN = mean) + variance = apply(prediction$predictions, MARGIN = 1L, FUN = var) + if (pv$se.simple.spatial) { + gw_dists = get_gower_dist(fct_to_char(newdata), fct_to_char(ordered_features(private$.train_task, self))) # 0 if identical, 1 if maximally dissimilar + min_gw_dists = apply(gw_dists, MARGIN = 1L, FUN = min) # get the minium for each new point to the points used for training + #min_gw_dists = (min_gw_dists - min(min_gw_dists)) / (max(min_gw_dists) - min(min_gw_dists)) # scale to [0, 1] + #min_gw_dists[is.na(min_gw_dists)] = 0 + # upwardly bias variance by the mean variance scaled by the gower distance (and the potential nugget, useful for noisy) + #variance = variance + (mean(variance) * (min_gw_dists + (pv$se.simple.spatial.nugget %??% 0))) # 0 is default + variance = (variance + pv$se.simple.spatial.nugget) + (variance + pv$se.simple.spatial.nugget) * min_gw_dists + } + se = sqrt(variance) + list(response = response, se = if (self$predict_type == "se") se else NULL) + }, + + .hotstart = function(task) { + model = self$model + model$num.trees = self$param_set$values$num.trees + model + }, + + .train_task = NULL + ) +) + +#' @export +default_values.LearnerRegrRangerCustom = function(x, search_space, task, ...) { # nolint + special_defaults = list( + mtry = floor(sqrt(length(task$feature_names))), + mtry.ratio = floor(sqrt(length(task$feature_names))) / length(task$feature_names), + num.trees = 1000L, + min.node.size = 1L, + replace = FALSE, + sample.fraction = 1, + splitrule = "extratrees", + num.random.splits = 1L + ) + defaults = insert_named(default_values(x$param_set), special_defaults) + defaults[search_space$ids()] +} + +convert_ratio_ranger = function(pv, target, ratio, n) { + switch(to_decimal(c(target, ratio) %in% names(pv)) + 1L, + # !mtry && !mtry.ratio + pv, + + # !mtry && mtry.ratio + { + pv[[target]] = max(ceiling(pv[[ratio]] * n), 1) + remove_named(pv, ratio) + }, + + # mtry && !mtry.ratio + pv, + + # mtry && mtry.ratio + stopf("Hyperparameters '%s' and '%s' are mutually exclusive", target, ratio) + ) +} + +ordered_features = function(task, learner) { + cols = names(learner$state$data_prototype) + task$data(cols = intersect(cols, task$feature_names)) +} + diff --git a/R/helper.R b/R/helper.R index ca7bd8a1..71f474ea 100644 --- a/R/helper.R +++ b/R/helper.R @@ -31,9 +31,19 @@ archive_xy = function(archive) { char_to_fct = function(xydt) { # Convert character params to factors chr_cols = names(xydt)[map_chr(xydt, class) == "character"] - if (length(chr_cols)) + if (length(chr_cols)) { xydt[, (chr_cols) := map(.SD, as.factor), .SDcols = chr_cols] - return(xydt) + } + xydt +} + +fct_to_char = function(xydt) { + # Convert factor params to character + fct_cols = names(xydt)[map_chr(xydt, class) %in% c("factor", "ordered")] + if (length(fct_cols)) { + xydt[, (fct_cols) := map(.SD, as.character), .SDcols = fct_cols] + } + xydt } archive_x = function(archive) { diff --git a/R/zzz.R b/R/zzz.R index 9d98a50a..23186171 100644 --- a/R/zzz.R +++ b/R/zzz.R @@ -15,6 +15,10 @@ # nocov start backports::import(pkgname) + # add rregr.ranger_custom to learner dictionary + x = utils::getFromNamespace("mlr_learners", ns = "mlr3") + x$add("regr.ranger_custom", LearnerRegrRangerCustom) + # add mbo to tuner dictionary x = utils::getFromNamespace("mlr_tuners", ns = "mlr3tuning") x$add("mbo", TunerMbo) diff --git a/man-roxygen/learner.R b/man-roxygen/learner.R new file mode 100644 index 00000000..1fbfb4c0 --- /dev/null +++ b/man-roxygen/learner.R @@ -0,0 +1,14 @@ +#' @section Dictionary: +#' This [Learner] can be instantiated via the [dictionary][mlr3misc::Dictionary] [mlr_learners] or with the associated sugar function [lrn()]: +#' ``` +#' mlr_learners$get("<%= id %>") +#' lrn("<%= id %>") +#' ``` +#' +#' @section Meta Information: +#' `r mlr3misc::rd_info(mlr3::lrn("<%= id %>"))` +#' @md +#' +#' @section Parameters: +#' `r mlr3misc::rd_info(mlr3::lrn("<%= id %>")$param_set)` +#' @md diff --git a/man-roxygen/seealso_learner.R b/man-roxygen/seealso_learner.R new file mode 100644 index 00000000..ba1aefae --- /dev/null +++ b/man-roxygen/seealso_learner.R @@ -0,0 +1,17 @@ +#' @seealso +#' +#' * Chapter in the [mlr3book](https://mlr3book.mlr-org.com/): +#' \url{https://mlr3book.mlr-org.com/basics.html#learners} +#' * Package \CRANpkg{mlr3learners} for a solid collection of essential learners. +#' * Package [mlr3extralearners](https://github.com/mlr-org/mlr3extralearners) for more learners. +#' * [Dictionary][mlr3misc::Dictionary] of [Learners][Learner]: [mlr_learners] +#' * `as.data.table(mlr_learners)` for a table of available [Learners][Learner] in the running session (depending on the loaded packages). +#' * \CRANpkg{mlr3pipelines} to combine learners with pre- and postprocessing steps. +#' * Package \CRANpkg{mlr3viz} for some generic visualizations. +#' * Extension packages for additional task types: +#' * \CRANpkg{mlr3proba} for probabilistic supervised regression and survival analysis. +#' * \CRANpkg{mlr3cluster} for unsupervised clustering. +#' * \CRANpkg{mlr3tuning} for tuning of hyperparameters, \CRANpkg{mlr3tuningspaces} +#' for established default tuning spaces. +#' +#' @family Learner diff --git a/man/mlr_learners_regr.ranger_custom.Rd b/man/mlr_learners_regr.ranger_custom.Rd new file mode 100644 index 00000000..a161bb05 --- /dev/null +++ b/man/mlr_learners_regr.ranger_custom.Rd @@ -0,0 +1,200 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/LearnerRegrRangerCustom.R +\name{mlr_learners_regr.ranger_custom} +\alias{mlr_learners_regr.ranger_custom} +\alias{LearnerRegrRangerCustom} +\title{Custom Ranger Regression Learner} +\description{ +Custom random regression forest. +Calls \code{\link[ranger:ranger]{ranger::ranger()}} from package \CRANpkg{ranger}. +This custom random regression forest tries to mimic the behaviour of a Gaussian process and therefore may be more suitable for Bayesian optimization compared to a normal random regression forest. + +By default, the \code{"extratrees"} splitting rule is used with \code{num.random.splits = 1}. +This results in a rather smooth prediction function. +No bootstrap or subsampling is used but rather all training data (\code{replace = FALSE} and \code{sample.fraction = 1}). +This may seem counter-intuitive at first but ensures that training data is almost perfectly interpolated (given \code{min.node.size = 1}). +Per default, \code{1000} trees are used. +Classical standard error estimation is no longer possible but \code{se.method = "simple"} allows for estimating the variance based on the variance of the trees due to the \code{"extratrees"} splitting rule (simply taking the variance over the predictions of the trees). +This results in low variance close to training data points. +If \code{se.simple.spatial = TRUE} this point-wise variance is upwardly biased by adding the pointwise variance again scaled by the minimum Gower distance of the point to the training data (recall that a Gower distance of 0 means an identical point is present in the training data, whereas a distance of 1 means total dissimilarity to the training data). +This ensures that the variance is larger in regions dissimilar to the training data. +Summarizing, the standard error for a point is calculated as follows: +\deqn{\hat{\mathrm{SE}}(x_{i}) = \sqrt{\hat{\mathrm{VAR}}(x_{i}) + (\hat{\mathrm{VAR}}(x_{i}) + \epsilon) * \mathrm{GD}_{i})}} +Here, \eqn{\epsilon} is given by \code{se.simple.spatial.nugget} which can be set larger than 0 in the case of noisy function observations. +} +\section{Dictionary}{ + +This \link{Learner} can be instantiated via the \link[mlr3misc:Dictionary]{dictionary} \link{mlr_learners} or with the associated sugar function \code{\link[=lrn]{lrn()}}:\preformatted{mlr_learners$get("regr.ranger_custom") +lrn("regr.ranger_custom") +} +} + +\section{Meta Information}{ + +\itemize{ +\item Task type: \dQuote{regr} +\item Predict Types: \dQuote{response}, \dQuote{se} +\item Feature Types: \dQuote{logical}, \dQuote{integer}, \dQuote{numeric}, \dQuote{character}, \dQuote{factor}, \dQuote{ordered} +\item Required Packages: \CRANpkg{mlr3}, \CRANpkg{mlr3mbo}, \CRANpkg{ranger} +} +} + +\section{Parameters}{ +\tabular{lllll}{ + Id \tab Type \tab Default \tab Levels \tab Range \cr + alpha \tab numeric \tab 0.5 \tab \tab \eqn{(-\infty, \infty)}{(-Inf, Inf)} \cr + always.split.variables \tab untyped \tab - \tab \tab - \cr + holdout \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr + importance \tab character \tab - \tab none, impurity, impurity_corrected, permutation \tab - \cr + keep.inbag \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr + local.importance \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr + max.depth \tab integer \tab NULL \tab \tab \eqn{[0, \infty)}{[0, Inf)} \cr + min.node.size \tab integer \tab 1 \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr + min.prop \tab numeric \tab 0.1 \tab \tab \eqn{(-\infty, \infty)}{(-Inf, Inf)} \cr + minprop \tab numeric \tab 0.1 \tab \tab \eqn{(-\infty, \infty)}{(-Inf, Inf)} \cr + mtry \tab integer \tab - \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr + mtry.ratio \tab numeric \tab - \tab \tab \eqn{[0, 1]}{[0, 1]} \cr + num.random.splits \tab integer \tab 1 \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr + num.threads \tab integer \tab 1 \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr + num.trees \tab integer \tab 1000 \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr + oob.error \tab logical \tab TRUE \tab TRUE, FALSE \tab - \cr + regularization.factor \tab untyped \tab 1 \tab \tab - \cr + regularization.usedepth \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr + replace \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr + respect.unordered.factors \tab character \tab ignore \tab ignore, order, partition \tab - \cr + sample.fraction \tab numeric \tab 1 \tab \tab \eqn{[0, 1]}{[0, 1]} \cr + save.memory \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr + scale.permutation.importance \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr + se.method \tab character \tab simple \tab jack, infjack, simple \tab - \cr + se.simple.spatial \tab logical \tab TRUE \tab TRUE, FALSE \tab - \cr + se.simple.spatial.nugget \tab numeric \tab 0 \tab \tab \eqn{[0, 1]}{[0, 1]} \cr + seed \tab integer \tab NULL \tab \tab \eqn{(-\infty, \infty)}{(-Inf, Inf)} \cr + split.select.weights \tab untyped \tab \tab \tab - \cr + splitrule \tab character \tab extratrees \tab variance, extratrees, maxstat \tab - \cr + verbose \tab logical \tab TRUE \tab TRUE, FALSE \tab - \cr + write.forest \tab logical \tab TRUE \tab TRUE, FALSE \tab - \cr +} +} + +\examples{ +library(data.table) +library(mlr3) +x = seq(-5, 5, length.out = 1001) +dat = data.table(x = c(-5, -2.5, -1, 0, 1, 2.5, 5), y = c(0, -0.1, 0.3, -1, 0.3, -0.1, 0)) +task = TaskRegr$new("example", backend = dat, target = "y") +learner = lrn("regr.ranger_custom") +learner$predict_type = "se" +learner$train(task) +predictions = learner$predict_newdata(data.table(x = x)) +if (requireNamespace("ggplot2")) { + library(ggplot2) + ggplot(aes(x = x, y = response), data = cbind(x, as.data.table(predictions))) + + geom_line() + + geom_ribbon(aes(min = response - se, max = response + se), alpha = 0.1) + + geom_point(aes(x = x, y = y), data = task$data()) + + labs(x = "x", y = "Prediction") + + theme_minimal() +} +} +\seealso{ +\itemize{ +\item Chapter in the \href{https://mlr3book.mlr-org.com/}{mlr3book}: +\url{https://mlr3book.mlr-org.com/basics.html#learners} +\item Package \CRANpkg{mlr3learners} for a solid collection of essential learners. +\item Package \href{https://github.com/mlr-org/mlr3extralearners}{mlr3extralearners} for more learners. +\item \link[mlr3misc:Dictionary]{Dictionary} of \link[=Learner]{Learners}: \link{mlr_learners} +\item \code{as.data.table(mlr_learners)} for a table of available \link[=Learner]{Learners} in the running session (depending on the loaded packages). +\item \CRANpkg{mlr3pipelines} to combine learners with pre- and postprocessing steps. +\item Package \CRANpkg{mlr3viz} for some generic visualizations. +\item Extension packages for additional task types: +\itemize{ +\item \CRANpkg{mlr3proba} for probabilistic supervised regression and survival analysis. +\item \CRANpkg{mlr3cluster} for unsupervised clustering. +} +\item \CRANpkg{mlr3tuning} for tuning of hyperparameters, \CRANpkg{mlr3tuningspaces} +for established default tuning spaces. +} +} +\concept{Learner} +\section{Super classes}{ +\code{\link[mlr3:Learner]{mlr3::Learner}} -> \code{\link[mlr3:LearnerRegr]{mlr3::LearnerRegr}} -> \code{LearnerRegrRangerCustom} +} +\section{Methods}{ +\subsection{Public methods}{ +\itemize{ +\item \href{#method-new}{\code{LearnerRegrRangerCustom$new()}} +\item \href{#method-importance}{\code{LearnerRegrRangerCustom$importance()}} +\item \href{#method-oob_error}{\code{LearnerRegrRangerCustom$oob_error()}} +\item \href{#method-clone}{\code{LearnerRegrRangerCustom$clone()}} +} +} +\if{html}{ +\out{
Inherited methods} +\itemize{ +\item \out{}\href{../../mlr3/html/Learner.html#method-base_learner}{\code{mlr3::Learner$base_learner()}}\out{} +\item \out{}\href{../../mlr3/html/Learner.html#method-format}{\code{mlr3::Learner$format()}}\out{} +\item \out{}\href{../../mlr3/html/Learner.html#method-help}{\code{mlr3::Learner$help()}}\out{} +\item \out{}\href{../../mlr3/html/Learner.html#method-predict}{\code{mlr3::Learner$predict()}}\out{} +\item \out{}\href{../../mlr3/html/Learner.html#method-predict_newdata}{\code{mlr3::Learner$predict_newdata()}}\out{} +\item \out{}\href{../../mlr3/html/Learner.html#method-print}{\code{mlr3::Learner$print()}}\out{} +\item \out{}\href{../../mlr3/html/Learner.html#method-reset}{\code{mlr3::Learner$reset()}}\out{} +\item \out{}\href{../../mlr3/html/Learner.html#method-train}{\code{mlr3::Learner$train()}}\out{} +} +\out{
} +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-new}{}}} +\subsection{Method \code{new()}}{ +Creates a new instance of this \link[R6:R6Class]{R6} class. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{LearnerRegrRangerCustom$new()}\if{html}{\out{
}} +} + +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-importance}{}}} +\subsection{Method \code{importance()}}{ +The importance scores are extracted from the model slot \code{variable.importance}. +Parameter \code{importance.mode} must be set to \code{"impurity"}, \code{"impurity_corrected"}, or +\code{"permutation"} +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{LearnerRegrRangerCustom$importance()}\if{html}{\out{
}} +} + +\subsection{Returns}{ +Named \code{numeric()}. +} +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-oob_error}{}}} +\subsection{Method \code{oob_error()}}{ +The out-of-bag error, extracted from model slot \code{prediction.error}. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{LearnerRegrRangerCustom$oob_error()}\if{html}{\out{
}} +} + +\subsection{Returns}{ +\code{numeric(1)}. +} +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-clone}{}}} +\subsection{Method \code{clone()}}{ +The objects of this class are cloneable with this method. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{LearnerRegrRangerCustom$clone(deep = FALSE)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{deep}}{Whether to make a deep clone.} +} +\if{html}{\out{
}} +} +} +} From c360b4ac76fc4a38fcc6c6d1c4343efef6734a58 Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Thu, 25 Aug 2022 21:06:27 +0200 Subject: [PATCH 007/126] yahpo experiments --- attic/so_config/analyze.R | 15 +- attic/so_config/analyze_yahpo.R | 90 ++++++++ attic/so_config/helpers.R | 9 + attic/so_config/run_yahpo.R | 364 ++++++++++++++++++++++++++++++++ 4 files changed, 467 insertions(+), 11 deletions(-) create mode 100644 attic/so_config/analyze_yahpo.R create mode 100644 attic/so_config/helpers.R create mode 100644 attic/so_config/run_yahpo.R diff --git a/attic/so_config/analyze.R b/attic/so_config/analyze.R index 5218d764..3fe51370 100644 --- a/attic/so_config/analyze.R +++ b/attic/so_config/analyze.R @@ -9,7 +9,7 @@ library(GGally) x1 = readRDS("ac_instance_1.rds") x2 = readRDS("ac_instance_2.rds") x3 = readRDS("ac_instance_3.rds") -x4 = readRDS("ac_instance_4.rds") +x4 = readRDS("ac_instance_4.rds") # FIXME: currently only 166/230 evals x5 = readRDS("ac_instance_5.rds") data = rbind(x1$archive$data, x2$archive$data, x3$archive$data, x4$archive$data, x5$archive$data)[, c(x1$archive$cols_x, x1$archive$cols_y), with = FALSE] @@ -51,14 +51,7 @@ best[, acqf := as.factor(acqf)] best[, acqopt := as.factor(acqopt)] best[, fs_behavior := as.factor(fs_behavior)] -write.table(task$data(), "tmp.csv") - -data = read.table("tmp.csv") -task = TaskRegr$new("mbo", backend = data, target = "mean_perf") -task = ppl("robustify", impute_missings = TRUE, factors_to_numeric = TRUE)$train(task)[[1L]] -data = task$data() -data[, random_interleave := as.integer(random_interleave)] - -write.table(data[, - 1], "X.csv", sep = ",", row.names = FALSE, col.names = FALSE) -write.table(data[, 1], "Y.csv", sep = ",", row.names = FALSE, col.names = FALSE) +summary(best[random_interleave == FALSE & acqopt == "FS" & fs_behavior == "global"]) +# we go with +# init = lhs, init_size_factor = 6, random_interleave = FALSE, num.trees = 250, splitrule = extratrees, num.random.splits = 8, acqf = CB, lambda = 2.8, acqopt_iter_factor = 6, acqopt = FS, fs_behavior = global diff --git a/attic/so_config/analyze_yahpo.R b/attic/so_config/analyze_yahpo.R new file mode 100644 index 00000000..ed60b03e --- /dev/null +++ b/attic/so_config/analyze_yahpo.R @@ -0,0 +1,90 @@ +library(data.table) +library(ggplot2) +library(pammtools) +library(mlr3misc) + +#dat = rbind(readRDS("results_yahpo.rds"), readRDS("results_yahpo_own.rds"))[method != "mlrintermbo"] +dat = rbind(readRDS("results_yahpo.rds"), readRDS("results_yahpo_own.rds"))[method %in% c("mlr3mbo", "mlrintermbo", "mlr3mbo_default", "random", "smac4hpo")] +dat[, cumbudget := cumsum(budget), by = .(method, scenario, instance, repl)] +dat[, cumbudget_scaled := cumbudget / max(cumbudget), by = .(method, scenario, instance, repl)] +dat[, normalized_regret := (target - min(target)) / (max(target) - min(target)), by = .(scenario, instance)] +dat[, incumbent := cummin(normalized_regret), by = .(method, scenario, instance, repl)] + +get_incumbent_cumbudget = function(incumbent, cumbudget_scaled) { + budgets = seq(0, 1, length.out = 101) + map_dbl(budgets, function(budget) { + ind = which(cumbudget_scaled <= budget) + if (length(ind) == 0L) { + max(incumbent) + } else { + min(incumbent[ind]) + } + }) +} + +dat_budget = dat[, .(incumbent_budget = get_incumbent_cumbudget(incumbent, cumbudget_scaled), cumbudget_scaled = seq(0, 1, length.out = 101)), by = .(method, scenario, instance, repl)] + +agg_budget = dat_budget[, .(mean = mean(incumbent_budget), se = sd(incumbent_budget) / sqrt(.N)), by = .(cumbudget_scaled, method, scenario, instance)] +agg_budget[, method := factor(method, levels = c("random", "smac4hpo", "hb", "bohb", "dehb", "smac4mf", "optuna", "mlr3mbo", "mlr3mbo_default"), labels = c("Random", "SMAC", "HB", "BOHB", "DEHB", "SMAC-HB", "optuna", "mlr3mbo", "mlr3mbo_default"))] + +g = ggplot(aes(x = cumbudget_scaled, y = mean, colour = method, fill = method), data = agg_budget[cumbudget_scaled > 0.10]) + + scale_y_log10() + + geom_step() + + geom_stepribbon(aes(min = mean - se, max = mean + se), colour = NA, alpha = 0.3) + + labs(x = "Fraction of Budget Used", y = "Mean Normalized Regret", colour = "Optimizer", fill = "Optimizer") + + facet_wrap(~ scenario + instance, scales = "free", ncol = 4) + + theme_minimal() + + theme(legend.position = "bottom", legend.title = element_text(size = rel(0.75)), legend.text = element_text(size = rel(0.75))) +ggsave("anytime.pdf", plot = g, device = "pdf", width = 12, height = 15) + +overall_budget = agg_budget[, .(mean = mean(mean), se = sd(mean) / sqrt(.N)), by = .(method, cumbudget_scaled)] + +g = ggplot(aes(x = cumbudget_scaled, y = mean, colour = method, fill = method), data = overall_budget[cumbudget_scaled > 0.10]) + + scale_y_log10() + + geom_step() + + labs(x = "Fraction of Budget Used", y = "Mean Normalized Regret", colour = "Optimizer", fill = "Optimizer") + + theme_minimal() + + theme(legend.position = "bottom", legend.title = element_text(size = rel(0.75)), legend.text = element_text(size = rel(0.75))) +ggsave("anytime_average.pdf", plot = g, device = "pdf", width = 6, height = 4) + +methods = unique(agg_budget$method) +ranks = map_dtr(unique(agg_budget$scenario), function(scenario_) { + map_dtr(unique(agg_budget$instance), function(instance_) { + map_dtr(unique(agg_budget$cumbudget_scaled), function(cumbudget_scaled_) { + res = agg_budget[scenario == scenario_ & instance == instance_ & cumbudget_scaled == cumbudget_scaled_] + if (nrow(res) == 0L) { + return(data.table()) + } + setorderv(res, "mean") + data.table(rank = match(methods, res$method), method = methods, scenario = scenario_, instance = instance_, cumbudget_scaled = cumbudget_scaled_) + }) + }) +}) + +ranks_overall = ranks[, .(mean = mean(rank), se = sd(rank) / sqrt(.N)), by = .(method, cumbudget_scaled)] + +g = ggplot(aes(x = cumbudget_scaled, y = mean, colour = method, fill = method), data = ranks_overall[cumbudget_scaled > 0.10]) + + geom_line() + + geom_ribbon(aes(min = mean - se, max = mean + se), colour = NA, alpha = 0.3) + + labs(x = "Fraction of Budget Used", y = "Mean Rank", colour = "Optimizer", fill = "Optimizer") + + theme_minimal() + + theme(legend.position = "bottom", legend.title = element_text(size = rel(0.75)), legend.text = element_text(size = rel(0.75))) +ggsave("anytime_average_rank.pdf", plot = g, device = "pdf", width = 6, height = 4) + +library(scmamp) +best_agg = agg_budget[cumbudget_scaled == 0.25] +best_agg[, problem := paste0(scenario, "_", instance)] +tmp = - as.matrix(dcast(best_agg, problem ~ method, value.var = "mean")[, -1]) +friedmanTest(tmp) +pdf("plots/cd_025_mf.pdf", width = 6, height = 4, pointsize = 10) +plotCD(tmp, cex = 1) +dev.off() + +best_agg = agg_budget[cumbudget_scaled == 1] +best_agg[, problem := paste0(scenario, "_", instance)] +tmp = - as.matrix(dcast(best_agg, problem ~ method, value.var = "mean")[, -1]) +friedmanTest(tmp) +pdf("plots/cd_1_mf.pdf", width = 6, height = 4, pointsize = 10) +plotCD(tmp, cex = 1) +dev.off() + diff --git a/attic/so_config/helpers.R b/attic/so_config/helpers.R new file mode 100644 index 00000000..a66f06b4 --- /dev/null +++ b/attic/so_config/helpers.R @@ -0,0 +1,9 @@ +make_optim_instance = function(instance) { + benchmark = BenchmarkSet$new(as.character(instance$scenario), instance = as.character(instance$instance)) + benchmark$subset_codomain(instance$target) + objective = benchmark$get_objective(as.character(instance$instance), multifidelity = FALSE, check_values = FALSE) + n_evals = as.integer(ceiling(instance$budget / instance$max_budget)) # full budget + optim_instance = OptimInstanceSingleCrit$new(objective, search_space = benchmark$get_search_space(drop_fidelity_params = TRUE), terminator = trm("evals", n_evals = n_evals), check_values = FALSE) + optim_instance +} + diff --git a/attic/so_config/run_yahpo.R b/attic/so_config/run_yahpo.R new file mode 100644 index 00000000..86073132 --- /dev/null +++ b/attic/so_config/run_yahpo.R @@ -0,0 +1,364 @@ +library(batchtools) +library(data.table) +library(mlr3) +library(mlr3learners) +library(mlr3pipelines) +library(mlr3misc) +library(mlr3mbo) # @so_config +library(bbotk) +library(paradox) +reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) +library(reticulate) +yahpo_gym = import("yahpo_gym") + +packages = c("data.table", "mlr3", "mlr3learners", "mlr3pipelines", "mlr3misc", "mlr3mbo", "bbotk", "paradox", "mlrintermbo") + +RhpcBLASctl::blas_set_num_threads(1L) +RhpcBLASctl::omp_set_num_threads(1L) + +root = here::here() +experiments_dir = file.path(root) + +source_files = map_chr("helpers.R", function(x) file.path(experiments_dir, x)) +for (sf in source_files) { + source(sf) +} + +reg = makeExperimentRegistry(file.dir = "/gscratch/lschnei8/registry_mlr3mbo_so_config", packages = packages, source = source_files) +#reg = makeExperimentRegistry(file.dir = NA, conf.file = NA, packages = packages, source = source_files) # interactive session +saveRegistry(reg) + +# FIXME: also compare jack vs. infjack? +mlr3mbo_wrapper = function(job, data, instance, ...) { + reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + optim_instance = make_optim_instance(instance) + + xdt = list(init = "lhs", init_size_factor = 4L, random_interleave = FALSE, num.trees = 250L, splitrule = "extratrees", num.random.splits = 8, acqf = "CB", lambda = 2.8, acqopt_iter_factor = 6L, acqopt = "FS", fs_behavior = "global") + + d = optim_instance$search_space$length + init_design_size = d * xdt$init_size_factor + init_design = if (xdt$init == "random") generate_design_random(optim_instance$search_space, n = init_design_size)$data else if (xdt$init == "lhs") generate_design_lhs(optim_instance$search_space, n = init_design_size)$data + optim_instance$eval_batch(init_design) + + random_interleave_iter = if(xdt$random_interleave) xdt$random_interleave_iter else 0L + + learner = if (xdt$splitrule == "extratrees") { + lrn("regr.ranger", num.trees = xdt$num.trees, keep.inbag = TRUE, splitrule = xdt$splitrule, num.random.splits = xdt$num.random.splits) + } else if (xdt$splitrule == "variance") { + lrn("regr.ranger", num.trees = xdt$num.trees, keep.inbag = TRUE, splitrule = xdt$splitrule) + } + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% learner)) + + acq_function = if (xdt$acqf == "EI") { + AcqFunctionEI$new() + } else if (xdt$acqf == "CB") { + AcqFunctionCB$new(lambda = xdt$lambda) + } else if (xdt$acqf == "PI") { + AcqFunctionPI$new() + } + + acq_budget = 1000 * xdt$acqopt_iter_factor + + acq_optimizer = if (xdt$acqopt == "RS") { + AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = acq_budget)) + } else if (xdt$acqopt == "FS") { + if (xdt$fs_behavior == "global") { + n_repeats = 10L + maxit = 2L + batch_size = ceiling((acq_budget / n_repeats) / (1 + maxit)) + } else if (xdt$fs_behavior == "local") { + n_repeats = 2L + maxit = 10L + batch_size = ceiling((acq_budget / n_repeats) / (1 + maxit)) + } + AcqOptimizer$new(opt("focus_search", n_points = batch_size, maxit = maxit), terminator = trm("evals", n_evals = acq_budget)) + } + + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + optim_instance +} + +mlrintermbo_wrapper = function(job, data, instance, ...) { + reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + optim_instance = make_optim_instance(instance) + optimizer = opt("intermbo", on.surrogate.error = "stop") + learner = lrn("regr.ranger", se.method = "jack", keep.inbag = TRUE) + learner$predict_type = "se" + learner = GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% learner) + learner$predict_type = "se" + #learner = mlr::makeLearner("regr.randomForest", se.method = "jackknife", keep.inbag = TRUE) + #learner = mlr::makeImputeWrapper(learner, classes = list(numeric = mlr::imputeMax(2), factor = mlr::imputeConstant("__miss__"), logical = mlr::imputeUniform())) + #learner = mlr::setPredictType(learner, "se") + #optimizer$param_set$values$surrogate.learner = learner + optimizer$optimize(optim_instance) + optim_instance +} + +mlr3mbo_default_wrapper = function(job, data, instance, ...) { + reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + optim_instance = make_optim_instance(instance) + + xdt = list(init = "random", init_size_factor = 4L, random_interleave = FALSE, num.trees = 500L, splitrule = "variance", num.random.splits = NA_integer_, acqf = "EI", lambda = NA_real_, acqopt_iter_factor = 10L, acqopt = "RS", fs_behavior = NA_character_) + + d = optim_instance$search_space$length + init_design_size = d * xdt$init_size_factor + init_design = if (xdt$init == "random") generate_design_random(optim_instance$search_space, n = init_design_size)$data else if (xdt$init == "lhs") generate_design_lhs(optim_instance$search_space, n = init_design_size)$data + optim_instance$eval_batch(init_design) + + random_interleave_iter = if(xdt$random_interleave) xdt$random_interleave_iter else 0L + + learner = if (xdt$splitrule == "extratrees") { + lrn("regr.ranger", num.trees = xdt$num.trees, keep.inbag = TRUE, splitrule = xdt$splitrule, num.random.splits = xdt$num.random.splits) + } else if (xdt$splitrule == "variance") { + lrn("regr.ranger", num.trees = xdt$num.trees, keep.inbag = TRUE, splitrule = xdt$splitrule) + } + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% learner)) + + acq_function = if (xdt$acqf == "EI") { + AcqFunctionEI$new() + } else if (xdt$acqf == "CB") { + AcqFunctionCB$new(lambda = xdt$lambda) + } else if (xdt$acqf == "PI") { + AcqFunctionPI$new() + } + + acq_budget = 1000 * xdt$acqopt_iter_factor + + acq_optimizer = if (xdt$acqopt == "RS") { + AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = acq_budget)) + } else if (xdt$acqopt == "FS") { + if (xdt$fs_behavior == "global") { + n_repeats = 10L + maxit = 2L + batch_size = ceiling((acq_budget / n_repeats) / (1 + maxit)) + } else if (xdt$fs_behavior == "local") { + n_repeats = 2L + maxit = 10L + batch_size = ceiling((acq_budget / n_repeats) / (1 + maxit)) + } + AcqOptimizer$new(opt("focus_search", n_points = batch_size, maxit = maxit), terminator = trm("evals", n_evals = acq_budget)) + } + + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + optim_instance +} + +mlr3mbo_wrapper_custom = function(job, data, instance, ...) { + reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + optim_instance = make_optim_instance(instance) + + xdt = list(init = "lhs", init_size_factor = 1L, random_interleave = FALSE, num.trees = 250L, splitrule = "extratrees", num.random.splits = 10, acqf = "CB", lambda = 3, acqopt_iter_factor = 30L, acqopt = "FS", fs_behavior = "global") + + d = optim_instance$search_space$length + init_design_size = d * xdt$init_size_factor + init_design = if (xdt$init == "random") generate_design_random(optim_instance$search_space, n = init_design_size)$data else if (xdt$init == "lhs") generate_design_lhs(optim_instance$search_space, n = init_design_size)$data + optim_instance$eval_batch(init_design) + + random_interleave_iter = if(xdt$random_interleave) xdt$random_interleave_iter else 0L + + learner = if (xdt$splitrule == "extratrees") { + lrn("regr.ranger", num.trees = xdt$num.trees, keep.inbag = TRUE, splitrule = xdt$splitrule, num.random.splits = xdt$num.random.splits) + } else if (xdt$splitrule == "variance") { + lrn("regr.ranger", num.trees = xdt$num.trees, keep.inbag = TRUE, splitrule = xdt$splitrule) + } + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% learner)) + + acq_function = if (xdt$acqf == "EI") { + AcqFunctionEI$new() + } else if (xdt$acqf == "CB") { + AcqFunctionCB$new(lambda = xdt$lambda) + } else if (xdt$acqf == "PI") { + AcqFunctionPI$new() + } + + acq_budget = 1000 * xdt$acqopt_iter_factor + + acq_optimizer = if (xdt$acqopt == "RS") { + AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = acq_budget)) + } else if (xdt$acqopt == "FS") { + if (xdt$fs_behavior == "global") { + n_repeats = 5L + maxit = 5L + batch_size = ceiling((acq_budget / n_repeats) / (1 + maxit)) + } else if (xdt$fs_behavior == "local") { + n_repeats = 2L + maxit = 10L + batch_size = ceiling((acq_budget / n_repeats) / (1 + maxit)) + } + AcqOptimizer$new(opt("focus_search", n_points = batch_size, maxit = maxit), terminator = trm("evals", n_evals = acq_budget)) + } + + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + optim_instance +} + +mlr3mbo_wrapper_new_rf = function(job, data, instance, ...) { + reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + optim_instance = make_optim_instance(instance) + + d = optim_instance$search_space$length + init_design_size = max(c(3L, d * 1L)) + init_design = generate_design_lhs(optim_instance$search_space, n = init_design_size)$data + optim_instance$eval_batch(init_design) + + random_interleave_iter = 0L + + learner = lrn("regr.ranger_custom") + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% learner)) + + acq_function = AcqFunctionEI$new() + + acq_budget = 20000L + + acq_optimizer = { + n_repeats = 2L + maxit = 9L + batch_size = ceiling((acq_budget / n_repeats) / (1 + maxit)) + AcqOptimizer$new(opt("focus_search", n_points = batch_size, maxit = maxit), terminator = trm("evals", n_evals = acq_budget)) + } + + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + optim_instance +} + +# add algorithms +addAlgorithm("mlr3mbo", fun = mlr3mbo_wrapper) +addAlgorithm("mlrintermbo", fun = mlrintermbo_wrapper) +addAlgorithm("mlr3mbo_default", fun = mlr3mbo_default_wrapper) +addAlgorithm("mlr3mbo_custom", fun = mlr3mbo_wrapper_custom) +addAlgorithm("mlr3mbo_new_rf", fun = mlr3mbo_wrapper_new_rf) + +# setup scenarios and instances +get_nb301_setup = function(budget_factor = 40L) { + scenario = "nb301" + bench = yahpo_gym$benchmark_set$BenchmarkSet(scenario, instance = "CIFAR10") + fidelity_space = bench$get_fidelity_space() + fidelity_param_id = fidelity_space$get_hyperparameter_names()[1] + min_budget = fidelity_space$get_hyperparameter(fidelity_param_id)$lower + max_budget = fidelity_space$get_hyperparameter(fidelity_param_id)$upper + ndim = length(bench$config_space$get_hyperparameter_names()) - 1L # NOTE: instance is not part of + + instances = "CIFAR10" + target = "val_accuracy" + budget = ceiling(20L * max_budget + sqrt(ndim) * max_budget * budget_factor) + on_integer_scale = TRUE + minimize = bench$config$config$y_minimize[match(target, bench$config$config$y_names)] + setup = setDT(expand.grid(scenario = scenario, instance = instances, target = target, ndim = ndim, max_budget = max_budget, budget = budget, on_integer_scale = on_integer_scale, minimize = minimize, stringsAsFactors = FALSE)) + setup +} + +get_lcbench_setup = function(budget_factor = 40L) { + scenario = "lcbench" + bench = yahpo_gym$benchmark_set$BenchmarkSet(scenario, instance = "167168") + fidelity_space = bench$get_fidelity_space() + fidelity_param_id = fidelity_space$get_hyperparameter_names()[1] + min_budget = fidelity_space$get_hyperparameter(fidelity_param_id)$lower + max_budget = fidelity_space$get_hyperparameter(fidelity_param_id)$upper + ndim = length(bench$config_space$get_hyperparameter_names()) - 2L + + instances = c("167168", "189873", "189906") + target = "val_accuracy" + budget = ceiling(20L * max_budget + sqrt(ndim) * max_budget * budget_factor) + on_integer_scale = TRUE + minimize = bench$config$config$y_minimize[match(target, bench$config$config$y_names)] + setup = setDT(expand.grid(scenario = scenario, instance = instances, target = target, ndim = ndim, max_budget = max_budget, budget = budget, on_integer_scale = on_integer_scale, minimize = minimize, stringsAsFactors = FALSE)) + setup +} + +get_rbv2_setup = function(budget_factor = 40L) { + setup = map_dtr(c("rbv2_glmnet", "rbv2_rpart", "rbv2_ranger", "rbv2_xgboost", "rbv2_super"), function(scenario) { + bench = yahpo_gym$benchmark_set$BenchmarkSet(scenario, instance = "1040") + fidelity_space = bench$get_fidelity_space() + fidelity_param_id = "trainsize" + min_budget = fidelity_space$get_hyperparameter(fidelity_param_id)$lower + max_budget = fidelity_space$get_hyperparameter(fidelity_param_id)$upper + ndim = length(bench$config_space$get_hyperparameter_names()) - 3L # repl and trainsize and instance + + instances = switch(scenario, rbv2_glmnet = c("375", "458"), rbv2_rpart = c("14", "40499"), rbv2_ranger = c("16", "42"), rbv2_xgboost = c("12", "1501", "16", "40499"), rbv2_super = c("1053", "1457", "1063", "1479", "15", "1468")) + target = "acc" + budget = ceiling(20L * max_budget + sqrt(ndim) * max_budget * budget_factor) + on_integer_scale = FALSE + minimize = bench$config$config$y_minimize[match(target, bench$config$config$y_names)] + setup = setDT(expand.grid(scenario = scenario, instance = instances, target = target, ndim = ndim, max_budget = max_budget, budget = budget, on_integer_scale = on_integer_scale, minimize = minimize, stringsAsFactors = FALSE)) + }) +} + +setup = rbind(get_nb301_setup(), get_lcbench_setup(), get_rbv2_setup()) + +setup[, id := seq_len(.N)] + +# add problems +prob_designs = map(seq_len(nrow(setup)), function(i) { + prob_id = paste0(setup[i, ]$scenario, "_", setup[i, ]$instance, "_", setup[i, ]$target) + addProblem(prob_id, data = list(scenario = setup[i, ]$scenario, instance = setup[i, ]$instance, target = setup[i, ]$target, ndim = setup[i, ]$ndim, max_budget = setup[i, ]$max_budget, budget = setup[i, ]$budget, on_integer_scale = setup[i, ]$on_integer_scale, minimize = setup[i, ]$minimize)) + setNames(list(setup[i, ]), nm = prob_id) +}) +nn = sapply(prob_designs, names) +prob_designs = unlist(prob_designs, recursive = FALSE, use.names = FALSE) +names(prob_designs) = nn + +# add jobs for optimizers +optimizers = data.table(algorithm = c("mlr3mbo", "mlrintermbo", "mlr3mbo_default", "mlr3mbo_custom", "mlr3mbo_new_rf")) + +for (i in seq_len(nrow(optimizers))) { + algo_designs = setNames(list(optimizers[i, ]), nm = optimizers[i, ]$algorithm) + + ids = addExperiments( + prob.designs = prob_designs, + algo.designs = algo_designs, + repls = 30L + ) + addJobTags(ids, as.character(optimizers[i, ]$algorithm)) +} + +jobs = findJobs() +resources.default = list(walltime = 3600 * 5L, memory = 2048L, ntasks = 1L, ncpus = 1L, nodes = 1L, clusters = "teton", max.concurrent.jobs = 9999L) +submitJobs(jobs, resources = resources.default) + +done = findDone() +results = reduceResultsList(done, function(x, job) { + x = x$archive$data + budget_var = if (job$instance$scenario %in% c("lcbench", "nb301")) "epoch" else "trainsize" + target_var = job$instance$target + if (!job$instance$minimize) { + x[, (target_var) := - get(target_var)] + } + pars = job$pars + tmp = x[, target_var, with = FALSE] + tmp[, (budget_var) := job$instance$max_budget] + tmp[, method := pars$algo.pars$algorithm] + tmp[, scenario := pars$prob.pars$scenario] + tmp[, instance := pars$prob.pars$instance] + tmp[, repl := job$repl] + tmp[, iter := seq_len(.N)] + colnames(tmp) = c("target", "budget", "method", "scenario", "instance", "repl", "iter") + tmp +}) +results = rbindlist(results, fill = TRUE) +saveRDS(results, "results_yahpo_own.rds") + From 936d287d9d1304cf0452fe149e9f84c04cad3973 Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Mon, 29 Aug 2022 14:22:05 +0200 Subject: [PATCH 008/126] fix init_design_size for random_interleave, acqopt logging and warmstart --- R/AcqOptimizer.R | 18 +++++++++++++---- R/bayesopt_ego.R | 2 +- attic/so_config/analyze_yahpo.R | 2 +- attic/so_config/run_yahpo.R | 34 +++++++++++++++++++++++++++++++-- 4 files changed, 48 insertions(+), 8 deletions(-) diff --git a/R/AcqOptimizer.R b/R/AcqOptimizer.R index c64a649a..6e343305 100644 --- a/R/AcqOptimizer.R +++ b/R/AcqOptimizer.R @@ -28,14 +28,17 @@ AcqOptimizer = R6Class("AcqOptimizer", self$terminator = assert_r6(terminator, "Terminator") self$acq_function = assert_r6(acq_function, "AcqFunction", null.ok = TRUE) ps = ps( - warmstart = p_lgl(), - fix_distance = p_lgl(), + logging_level = p_fct(levels = c("fatal", "error", "warn", "info", "debug", "trace"), default = "warn"), + warmstart = p_lgl(default = FALSE), + warmstart_size = p_int(lower = 1L), + fix_distance = p_lgl(default = FALSE), dist_threshold = p_dbl(lower = 0, upper = 1) ) # FIXME: not having a dist_threshold may be problematic if only fix_distance is set to TRUE # FIXME: assert this - ps$values = list(warmstart = FALSE, fix_distance = FALSE) + ps$values = list(logging_level = "warn", warmstart = FALSE, fix_distance = FALSE) + ps$add_dep("warmstart_size", on = "warmstart", cond = CondEqual$new(TRUE)) ps$add_dep("dist_threshold", on = "fix_distance", cond = CondEqual$new(TRUE)) private$.param_set = ps }, @@ -53,6 +56,10 @@ AcqOptimizer = R6Class("AcqOptimizer", #' #' @return [data.table::data.table()] with 1 row per optimum and x as columns. optimize = function() { + logger = lgr::get_logger("bbotk") + old_threshold = logger$threshold + logger$set_threshold(self$param_set$values$logging_level) + instance = if (self$acq_function$codomain$length == 1L) { OptimInstanceSingleCrit$new(objective = self$acq_function, search_space = self$acq_function$domain, terminator = self$terminator, check_values = FALSE, keep_evals = "all") } else { @@ -64,7 +71,8 @@ AcqOptimizer = R6Class("AcqOptimizer", if (self$param_set$values$warmstart) { # NOTE: is this save if e.g. mu < nrow(best) in miesmuschel? - instance$eval_batch(self$acq_function$archive$best()[, instance$search_space$ids(), with = FALSE]) + n_select = min(nrow(self$acq_function$archive$data), self$param_set$values$warmstart_size) + instance$eval_batch(self$acq_function$archive$best(n_select = n_select)[, instance$search_space$ids(), with = FALSE]) } xdt = tryCatch(self$optimizer$optimize(instance), @@ -80,6 +88,8 @@ AcqOptimizer = R6Class("AcqOptimizer", xdt = fix_xdt_distance(xdt, previous_xdt = archive_x(self$acq_function$archive), search_space = self$acq_function$domain, dist_threshold = self$param_set$values$dist_threshold) } + logger$set_threshold(old_threshold) + xdt } ), diff --git a/R/bayesopt_ego.R b/R/bayesopt_ego.R index d6dd920c..fa38371b 100644 --- a/R/bayesopt_ego.R +++ b/R/bayesopt_ego.R @@ -109,7 +109,7 @@ bayesopt_ego = function( archive = instance$archive domain = instance$search_space d = domain$length - if (is.null(init_design_size) && instance$archive$n_evals == 0L) init_design_size = 4 * d + if (is.null(init_design_size) && instance$archive$n_evals == 0L) init_design_size = 4 * d else init_design_size = instance$archive$n_evals if (is.null(surrogate)) surrogate = default_surrogate(instance) if (is.null(acq_function)) acq_function = default_acqfun(instance) if (is.null(acq_optimizer)) acq_optimizer = default_acqopt(acq_function) diff --git a/attic/so_config/analyze_yahpo.R b/attic/so_config/analyze_yahpo.R index ed60b03e..8bb18fdf 100644 --- a/attic/so_config/analyze_yahpo.R +++ b/attic/so_config/analyze_yahpo.R @@ -4,7 +4,7 @@ library(pammtools) library(mlr3misc) #dat = rbind(readRDS("results_yahpo.rds"), readRDS("results_yahpo_own.rds"))[method != "mlrintermbo"] -dat = rbind(readRDS("results_yahpo.rds"), readRDS("results_yahpo_own.rds"))[method %in% c("mlr3mbo", "mlrintermbo", "mlr3mbo_default", "random", "smac4hpo")] +dat = rbind(readRDS("results_yahpo.rds"), readRDS("results_yahpo_own.rds"))[method %in% c("mlr3mbo", "mlrintermbo", "random", "smac4hpo", "smac4mf", "mlr3mbo_new_rf")] dat[, cumbudget := cumsum(budget), by = .(method, scenario, instance, repl)] dat[, cumbudget_scaled := cumbudget / max(cumbudget), by = .(method, scenario, instance, repl)] dat[, normalized_regret := (target - min(target)) / (max(target) - min(target)), by = .(scenario, instance)] diff --git a/attic/so_config/run_yahpo.R b/attic/so_config/run_yahpo.R index 86073132..53ceaf2f 100644 --- a/attic/so_config/run_yahpo.R +++ b/attic/so_config/run_yahpo.R @@ -246,12 +246,42 @@ mlr3mbo_wrapper_new_rf = function(job, data, instance, ...) { optim_instance } +mlr3mbo_wrapper_new_rf_ls = function(job, data, instance, ...) { + reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + optim_instance = make_optim_instance(instance) + + d = optim_instance$search_space$length + init_design_size = max(c(3L, d * 1L)) + init_design = generate_design_lhs(optim_instance$search_space, n = init_design_size)$data + optim_instance$eval_batch(init_design) + + random_interleave_iter = 3L + + learner = lrn("regr.ranger_custom") + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% learner)) + + acq_function = AcqFunctionEI$new() + + acq_optimizer = AcqOptimizer$new(opt("local_search", n_points = 100L), terminator = trm("evals", n_evals = 10000L)) + acq_optimizer$param_set$values$warmstart = TRUE + acq_optimizer$param_set$values$warmstart_size = 10L + + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + optim_instance +} + # add algorithms addAlgorithm("mlr3mbo", fun = mlr3mbo_wrapper) addAlgorithm("mlrintermbo", fun = mlrintermbo_wrapper) addAlgorithm("mlr3mbo_default", fun = mlr3mbo_default_wrapper) addAlgorithm("mlr3mbo_custom", fun = mlr3mbo_wrapper_custom) addAlgorithm("mlr3mbo_new_rf", fun = mlr3mbo_wrapper_new_rf) +addAlgorithm("mlr3mbo_new_rf_ls", fun = mlr3mbo_wrapper_new_rf_ls) # setup scenarios and instances get_nb301_setup = function(budget_factor = 40L) { @@ -323,7 +353,7 @@ prob_designs = unlist(prob_designs, recursive = FALSE, use.names = FALSE) names(prob_designs) = nn # add jobs for optimizers -optimizers = data.table(algorithm = c("mlr3mbo", "mlrintermbo", "mlr3mbo_default", "mlr3mbo_custom", "mlr3mbo_new_rf")) +optimizers = data.table(algorithm = c("mlr3mbo", "mlrintermbo", "mlr3mbo_default", "mlr3mbo_custom", "mlr3mbo_new_rf", "mlr3mbo_new_rf_ls")) for (i in seq_len(nrow(optimizers))) { algo_designs = setNames(list(optimizers[i, ]), nm = optimizers[i, ]$algorithm) @@ -337,7 +367,7 @@ for (i in seq_len(nrow(optimizers))) { } jobs = findJobs() -resources.default = list(walltime = 3600 * 5L, memory = 2048L, ntasks = 1L, ncpus = 1L, nodes = 1L, clusters = "teton", max.concurrent.jobs = 9999L) +resources.default = list(walltime = 3600 * 12L, memory = 2048L, ntasks = 1L, ncpus = 1L, nodes = 1L, clusters = "teton", max.concurrent.jobs = 9999L) submitJobs(jobs, resources = resources.default) done = findDone() From fefeb38cce49bf0c1e5ccb6c6024605643c0d425 Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Mon, 29 Aug 2022 22:36:24 +0200 Subject: [PATCH 009/126] tweaks in acq opt and warmstart --- R/AcqOptimizer.R | 32 ++++++++++++++++++++++++++++---- R/bayesopt_ego.R | 4 +++- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/R/AcqOptimizer.R b/R/AcqOptimizer.R index 6e343305..5c7d1c89 100644 --- a/R/AcqOptimizer.R +++ b/R/AcqOptimizer.R @@ -30,7 +30,7 @@ AcqOptimizer = R6Class("AcqOptimizer", ps = ps( logging_level = p_fct(levels = c("fatal", "error", "warn", "info", "debug", "trace"), default = "warn"), warmstart = p_lgl(default = FALSE), - warmstart_size = p_int(lower = 1L), + warmstart_size = p_int(lower = 1L, special_vals = list("all")), fix_distance = p_lgl(default = FALSE), dist_threshold = p_dbl(lower = 0, upper = 1) ) @@ -59,6 +59,7 @@ AcqOptimizer = R6Class("AcqOptimizer", logger = lgr::get_logger("bbotk") old_threshold = logger$threshold logger$set_threshold(self$param_set$values$logging_level) + on.exit(logger$set_threshold(old_threshold)) instance = if (self$acq_function$codomain$length == 1L) { OptimInstanceSingleCrit$new(objective = self$acq_function, search_space = self$acq_function$domain, terminator = self$terminator, check_values = FALSE, keep_evals = "all") @@ -70,11 +71,13 @@ AcqOptimizer = R6Class("AcqOptimizer", } if (self$param_set$values$warmstart) { + warmstart_size = if (self$param_set$values$warmstart_size == "all") Inf # NOTE: is this save if e.g. mu < nrow(best) in miesmuschel? - n_select = min(nrow(self$acq_function$archive$data), self$param_set$values$warmstart_size) + n_select = min(nrow(self$acq_function$archive$data), warmstart_size) instance$eval_batch(self$acq_function$archive$best(n_select = n_select)[, instance$search_space$ids(), with = FALSE]) } + # FIXME: how do we handle multic-rit here xdt = tryCatch(self$optimizer$optimize(instance), error = function(error_condition) { lg$info(error_condition$message) @@ -83,13 +86,21 @@ AcqOptimizer = R6Class("AcqOptimizer", classes = c("mbo_error", "acq_optimizer_error", "error", "condition"))) } ) + # check if candidate was already evaluated and if so get the best one not evaluated so far + if (nrow(self$acq_function$archive$data[xdt[, instance$archive$cols_x, with = FALSE], on = instance$archive$cols_x]) > 0L) { + xdt = tryCatch(get_best_not_evaluated(instance, evaluated = self$acq_function$archive$data), + error = function(error_condition) { + lg$info(error_condition$message) + stop(set_class(list(message = error_condition$message, call = NULL), + classes = c("mbo_error", "acq_optimizer_error", "error", "condition"))) + } + ) + } if (self$param_set$values$fix_distance) { xdt = fix_xdt_distance(xdt, previous_xdt = archive_x(self$acq_function$archive), search_space = self$acq_function$domain, dist_threshold = self$param_set$values$dist_threshold) } - logger$set_threshold(old_threshold) - xdt } ), @@ -121,3 +132,16 @@ AcqOptimizer = R6Class("AcqOptimizer", } ) ) + +# FIXME: document +get_best_not_evaluated = function(instance, evaluated) { + assert_r6(instance, classes = "OptimInstanceSingleCrit") + data = copy(instance$archive$data[, c(instance$archive$cols_x, instance$archive$cols_y), with = FALSE]) + evaluated = copy(evaluated) + data[, overlap := FALSE][evaluated, overlap := TRUE, on = instance$archive$cols_x] + candidates = data[overlap == FALSE] + candidates[[instance$archive$cols_y]] = candidates[[instance$archive$cols_y]] * instance$objective_multiplicator[instance$archive$cols_y] + xdt = setorderv(candidates, cols = instance$archive$cols_y)[1L, ] + xdt[[instance$archive$cols_y]] = xdt[[instance$archive$cols_y]] * instance$objective_multiplicator[instance$archive$cols_y] + xdt +} diff --git a/R/bayesopt_ego.R b/R/bayesopt_ego.R index fa38371b..f0b870e7 100644 --- a/R/bayesopt_ego.R +++ b/R/bayesopt_ego.R @@ -109,7 +109,7 @@ bayesopt_ego = function( archive = instance$archive domain = instance$search_space d = domain$length - if (is.null(init_design_size) && instance$archive$n_evals == 0L) init_design_size = 4 * d else init_design_size = instance$archive$n_evals + if (is.null(init_design_size) && instance$archive$n_evals == 0L) init_design_size = 4L * d if (is.null(surrogate)) surrogate = default_surrogate(instance) if (is.null(acq_function)) acq_function = default_acqfun(instance) if (is.null(acq_optimizer)) acq_optimizer = default_acqopt(acq_function) @@ -121,6 +121,8 @@ bayesopt_ego = function( if (isTRUE(init_design_size > 0L)) { design = generate_design_random(domain, n = init_design_size)$data instance$eval_batch(design) + } else { + init_design_size = instance$archive$n_evals } # loop From 62bb10a7a32b7db75d79361ca8f7761509916b68 Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Tue, 30 Aug 2022 15:59:52 +0200 Subject: [PATCH 010/126] some more infill optimization tweaks for so config --- DESCRIPTION | 2 +- attic/so_config/OptimizerChain.R | 83 ++++++++++++++++++++++++++ attic/so_config/analyze_yahpo.R | 3 +- attic/so_config/run_yahpo.R | 18 +++--- man/AcqFunction.Rd | 52 ++++++++-------- man/AcqOptimizer.Rd | 18 +++--- man/Surrogate.Rd | 24 ++++---- man/SurrogateLearner.Rd | 36 +++++------ man/SurrogateLearners.Rd | 36 +++++------ man/mlr_acqfunctions_aei.Rd | 50 ++++++++-------- man/mlr_acqfunctions_cb.Rd | 46 +++++++------- man/mlr_acqfunctions_ei.Rd | 50 ++++++++-------- man/mlr_acqfunctions_eips.Rd | 50 ++++++++-------- man/mlr_acqfunctions_pi.Rd | 50 ++++++++-------- man/mlr_acqfunctions_sms_ego.Rd | 44 +++++++------- man/mlr_learners_regr.ranger_custom.Rd | 81 ++++++++++++------------- man/mlr_optimizers_mbo.Rd | 36 +++++------ man/mlr_tuners_mbo.Rd | 36 +++++------ 18 files changed, 402 insertions(+), 313 deletions(-) create mode 100644 attic/so_config/OptimizerChain.R diff --git a/DESCRIPTION b/DESCRIPTION index 304c3a00..39ad1740 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -50,7 +50,7 @@ ByteCompile: no Encoding: UTF-8 NeedsCompilation: yes Roxygen: list(markdown = TRUE) -RoxygenNote: 7.1.2 +RoxygenNote: 7.2.1 Collate: 'mlr_acqfunctions.R' 'AcqFunction.R' diff --git a/attic/so_config/OptimizerChain.R b/attic/so_config/OptimizerChain.R new file mode 100644 index 00000000..4f04a373 --- /dev/null +++ b/attic/so_config/OptimizerChain.R @@ -0,0 +1,83 @@ +OptimizerChain = R6Class("OptimizerChain", inherit = bbotk::Optimizer, + public = list( + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + #' + #' @param optimizers (list of [Optimizer]s). + #' @param terminators (list of [Terminator]s | NULL). + initialize = function(optimizers, terminators = rep(list(NULL), length(optimizers))) { + assert_list(optimizers, types = "Optimizer", any.missing = FALSE) + assert_list(terminators, types = c("Terminator", "NULL"), len = length(optimizers)) + + param_sets = vector(mode = "list", length = length(optimizers)) + ids_taken = character(0L) + # for each optimizer check whether the id of the param_set + # (decuded from the optimizer class) is already taken; + # if necessary postfix the id + for (i_opt in seq_along(optimizers)) { + opt = optimizers[[i_opt]] + ps = opt$param_set$clone(deep = TRUE) + ps$set_id = class(opt)[[1L]] + try_postfix = 0L + while (ps$set_id %in% ids_taken) { + try_postfix = try_postfix + 1L + ps$set_id = paste0(class(opt)[[1L]], "_", try_postfix) + } + ids_taken[[i_opt]] = ps$set_id + param_sets[[i_opt]] = ps + } + private$.ids = map_chr(param_sets, "set_id") + super$initialize( + param_set = ParamSetCollection$new(param_sets), + param_classes = Reduce(intersect, mlr3misc::map(optimizers, "param_classes")), + properties = Reduce(intersect, mlr3misc::map(optimizers, "properties")), + packages = unique(unlist(mlr3misc::map(optimizers, "packages"))) + ) + private$.optimizers = optimizers + private$.terminators = terminators + } + ), + + private = list( + .optimizers = NULL, + .terminators = NULL, + .ids = NULL, + + .optimize = function(inst) { + terminator = inst$terminator + on.exit({inst$terminator = terminator}) + inner_inst = inst$clone(deep = TRUE) + + for (i_opt in seq_along(private$.optimizers)) { + inner_term = private$.terminators[[i_opt]] + if (!is.null(inner_term)) { + inner_inst$terminator = TerminatorCombo$new(list(inner_term, terminator)) + } else { + inner_inst$terminator = terminator + } + opt = private$.optimizers[[i_opt]] + opt$param_set$values = self$param_set$.__enclos_env__$private$.sets[[i_opt]]$values + opt$optimize(inner_inst) + inner_inst$archive$data$batch_nr = max(inst$archive$data$batch_nr, 0L) + + inner_inst$archive$data$batch_nr + inner_inst$archive$data$optimizer = private$.ids[i_opt] + inst$archive$data = rbind(inst$archive$data, inner_inst$archive$data, fill = TRUE) + inner_inst$archive$data = data.table() + if (terminator$is_terminated(inst$archive)) { + break + } + } + }, + + deep_clone = function(name, value) { + switch( + name, + .optimizers = mlr3misc::map(value, .f = function(x) x$clone(deep = TRUE)), + .terminators = mlr3misc::map(value, .f = function(x) if (!is.null(x)) x$clone(deep = TRUE)), + value + ) + } + ) +) + diff --git a/attic/so_config/analyze_yahpo.R b/attic/so_config/analyze_yahpo.R index 8bb18fdf..a6083844 100644 --- a/attic/so_config/analyze_yahpo.R +++ b/attic/so_config/analyze_yahpo.R @@ -4,7 +4,8 @@ library(pammtools) library(mlr3misc) #dat = rbind(readRDS("results_yahpo.rds"), readRDS("results_yahpo_own.rds"))[method != "mlrintermbo"] -dat = rbind(readRDS("results_yahpo.rds"), readRDS("results_yahpo_own.rds"))[method %in% c("mlr3mbo", "mlrintermbo", "random", "smac4hpo", "smac4mf", "mlr3mbo_new_rf")] +dat = rbind(readRDS("results_yahpo.rds"), readRDS("results_yahpo_own.rds"))[method %in% c("mlr3mbo", "mlrintermbo", "random", "smac4hpo", "smac4mf", "mlr3mbo_new_rf", "mlr3mbo_new_rf_ls")] +dat = dat[scenario %nin% c("nb301", "rbv2_super")] dat[, cumbudget := cumsum(budget), by = .(method, scenario, instance, repl)] dat[, cumbudget_scaled := cumbudget / max(cumbudget), by = .(method, scenario, instance, repl)] dat[, normalized_regret := (target - min(target)) / (max(target) - min(target)), by = .(scenario, instance)] diff --git a/attic/so_config/run_yahpo.R b/attic/so_config/run_yahpo.R index 53ceaf2f..c7970d06 100644 --- a/attic/so_config/run_yahpo.R +++ b/attic/so_config/run_yahpo.R @@ -19,7 +19,7 @@ RhpcBLASctl::omp_set_num_threads(1L) root = here::here() experiments_dir = file.path(root) -source_files = map_chr("helpers.R", function(x) file.path(experiments_dir, x)) +source_files = map_chr(c("helpers.R", "OptimizerChain.R"), function(x) file.path(experiments_dir, x)) for (sf in source_files) { source(sf) } @@ -222,7 +222,7 @@ mlr3mbo_wrapper_new_rf = function(job, data, instance, ...) { optim_instance = make_optim_instance(instance) d = optim_instance$search_space$length - init_design_size = max(c(3L, d * 1L)) + init_design_size = 4L * d init_design = generate_design_lhs(optim_instance$search_space, n = init_design_size)$data optim_instance$eval_batch(init_design) @@ -256,20 +256,22 @@ mlr3mbo_wrapper_new_rf_ls = function(job, data, instance, ...) { optim_instance = make_optim_instance(instance) d = optim_instance$search_space$length - init_design_size = max(c(3L, d * 1L)) + init_design_size = 4L * d init_design = generate_design_lhs(optim_instance$search_space, n = init_design_size)$data optim_instance$eval_batch(init_design) - random_interleave_iter = 3L + random_interleave_iter = 0L learner = lrn("regr.ranger_custom") surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% learner)) acq_function = AcqFunctionEI$new() - - acq_optimizer = AcqOptimizer$new(opt("local_search", n_points = 100L), terminator = trm("evals", n_evals = 10000L)) + + optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10000L), trm("evals", n_evals = 10000L))) + acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20000L)) + acq_optimizer$param_set$values$warmstart = TRUE - acq_optimizer$param_set$values$warmstart_size = 10L + acq_optimizer$param_set$values$warmstart_size = "all" bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) optim_instance @@ -367,7 +369,7 @@ for (i in seq_len(nrow(optimizers))) { } jobs = findJobs() -resources.default = list(walltime = 3600 * 12L, memory = 2048L, ntasks = 1L, ncpus = 1L, nodes = 1L, clusters = "teton", max.concurrent.jobs = 9999L) +resources.default = list(walltime = 3600 * 6L, memory = 2048L, ntasks = 1L, ncpus = 1L, nodes = 1L, clusters = "teton", max.concurrent.jobs = 9999L) submitJobs(jobs, resources = resources.default) done = findDone() diff --git a/man/AcqFunction.Rd b/man/AcqFunction.Rd index 2e33b52d..5b9e55fd 100644 --- a/man/AcqFunction.Rd +++ b/man/AcqFunction.Rd @@ -50,25 +50,25 @@ Objective function.} \section{Methods}{ \subsection{Public methods}{ \itemize{ -\item \href{#method-new}{\code{AcqFunction$new()}} -\item \href{#method-update}{\code{AcqFunction$update()}} -\item \href{#method-eval_many}{\code{AcqFunction$eval_many()}} -\item \href{#method-eval_dt}{\code{AcqFunction$eval_dt()}} -\item \href{#method-clone}{\code{AcqFunction$clone()}} -} -} -\if{html}{ -\out{
Inherited methods} -\itemize{ -\item \out{}\href{../../bbotk/html/Objective.html#method-eval}{\code{bbotk::Objective$eval()}}\out{} -\item \out{}\href{../../bbotk/html/Objective.html#method-format}{\code{bbotk::Objective$format()}}\out{} -\item \out{}\href{../../bbotk/html/Objective.html#method-print}{\code{bbotk::Objective$print()}}\out{} -} -\out{
} -} +\item \href{#method-AcqFunction-new}{\code{AcqFunction$new()}} +\item \href{#method-AcqFunction-update}{\code{AcqFunction$update()}} +\item \href{#method-AcqFunction-eval_many}{\code{AcqFunction$eval_many()}} +\item \href{#method-AcqFunction-eval_dt}{\code{AcqFunction$eval_dt()}} +\item \href{#method-AcqFunction-clone}{\code{AcqFunction$clone()}} +} +} +\if{html}{\out{ +
Inherited methods + +
+}} \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-new}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqFunction-new}{}}} \subsection{Method \code{new()}}{ Creates a new instance of this \link[R6:R6Class]{R6} class. @@ -92,8 +92,8 @@ Note that the surrogate can be initialized lazy and can later be set via the act } } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-update}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqFunction-update}{}}} \subsection{Method \code{update()}}{ Update the acquisition function. \subsection{Usage}{ @@ -102,8 +102,8 @@ Update the acquisition function. } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-eval_many}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqFunction-eval_many}{}}} \subsection{Method \code{eval_many()}}{ Evaluates multiple input values received as a list, converted to a \code{data.table()} on the objective function. Missing columns in xss are filled with \code{NA}s in \code{xdt}. @@ -127,8 +127,8 @@ and multiple y-columns for multi-criteria functions, e.g. } } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-eval_dt}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqFunction-eval_dt}{}}} \subsection{Method \code{eval_dt()}}{ Evaluates multiple input values on the objective function supplied by the user. \subsection{Usage}{ @@ -153,8 +153,8 @@ and multiple y-columns for multi-criteria functions, e.g. } } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-clone}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqFunction-clone}{}}} \subsection{Method \code{clone()}}{ The objects of this class are cloneable with this method. \subsection{Usage}{ diff --git a/man/AcqOptimizer.Rd b/man/AcqOptimizer.Rd index 52573d57..90701445 100644 --- a/man/AcqOptimizer.Rd +++ b/man/AcqOptimizer.Rd @@ -29,14 +29,14 @@ Set of hyperparameters.} \section{Methods}{ \subsection{Public methods}{ \itemize{ -\item \href{#method-new}{\code{AcqOptimizer$new()}} -\item \href{#method-optimize}{\code{AcqOptimizer$optimize()}} -\item \href{#method-clone}{\code{AcqOptimizer$clone()}} +\item \href{#method-AcqOptimizer-new}{\code{AcqOptimizer$new()}} +\item \href{#method-AcqOptimizer-optimize}{\code{AcqOptimizer$optimize()}} +\item \href{#method-AcqOptimizer-clone}{\code{AcqOptimizer$clone()}} } } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-new}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizer-new}{}}} \subsection{Method \code{new()}}{ Creates a new instance of this \link[R6:R6Class]{R6} class. \subsection{Usage}{ @@ -56,8 +56,8 @@ Creates a new instance of this \link[R6:R6Class]{R6} class. } } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-optimize}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizer-optimize}{}}} \subsection{Method \code{optimize()}}{ Optimize the acquisition function. @@ -77,8 +77,8 @@ the checks but most likely will. } } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-clone}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizer-clone}{}}} \subsection{Method \code{clone()}}{ The objects of this class are cloneable with this method. \subsection{Usage}{ diff --git a/man/Surrogate.Rd b/man/Surrogate.Rd index d93e49d8..f4a2b936 100644 --- a/man/Surrogate.Rd +++ b/man/Surrogate.Rd @@ -37,15 +37,15 @@ Asserts whether the current insample performance meets the performance threshold \section{Methods}{ \subsection{Public methods}{ \itemize{ -\item \href{#method-new}{\code{Surrogate$new()}} -\item \href{#method-update}{\code{Surrogate$update()}} -\item \href{#method-predict}{\code{Surrogate$predict()}} -\item \href{#method-clone}{\code{Surrogate$clone()}} +\item \href{#method-Surrogate-new}{\code{Surrogate$new()}} +\item \href{#method-Surrogate-update}{\code{Surrogate$update()}} +\item \href{#method-Surrogate-predict}{\code{Surrogate$predict()}} +\item \href{#method-Surrogate-clone}{\code{Surrogate$clone()}} } } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-new}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-Surrogate-new}{}}} \subsection{Method \code{new()}}{ Creates a new instance of this \link[R6:R6Class]{R6} class. \subsection{Usage}{ @@ -67,8 +67,8 @@ Creates a new instance of this \link[R6:R6Class]{R6} class. } } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-update}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-Surrogate-update}{}}} \subsection{Method \code{update()}}{ Train model with new data. \subsection{Usage}{ @@ -80,8 +80,8 @@ Train model with new data. } } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-predict}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-Surrogate-predict}{}}} \subsection{Method \code{predict()}}{ Predict mean response and standard error. \subsection{Usage}{ @@ -101,8 +101,8 @@ Arbitrary prediction object. } } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-clone}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-Surrogate-clone}{}}} \subsection{Method \code{clone()}}{ The objects of this class are cloneable with this method. \subsection{Usage}{ diff --git a/man/SurrogateLearner.Rd b/man/SurrogateLearner.Rd index 59200f02..5ac946d4 100644 --- a/man/SurrogateLearner.Rd +++ b/man/SurrogateLearner.Rd @@ -23,21 +23,21 @@ Asserts whether the current insample performance meets the performance threshold \section{Methods}{ \subsection{Public methods}{ \itemize{ -\item \href{#method-new}{\code{SurrogateLearner$new()}} -\item \href{#method-predict}{\code{SurrogateLearner$predict()}} -\item \href{#method-clone}{\code{SurrogateLearner$clone()}} -} -} -\if{html}{ -\out{
Inherited methods} -\itemize{ -\item \out{}\href{../../mlr3mbo/html/Surrogate.html#method-update}{\code{mlr3mbo::Surrogate$update()}}\out{} -} -\out{
} -} +\item \href{#method-SurrogateLearner-new}{\code{SurrogateLearner$new()}} +\item \href{#method-SurrogateLearner-predict}{\code{SurrogateLearner$predict()}} +\item \href{#method-SurrogateLearner-clone}{\code{SurrogateLearner$clone()}} +} +} +\if{html}{\out{ +
Inherited methods + +
+}} \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-new}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-SurrogateLearner-new}{}}} \subsection{Method \code{new()}}{ Creates a new instance of this \link[R6:R6Class]{R6} class. \subsection{Usage}{ @@ -57,8 +57,8 @@ Creates a new instance of this \link[R6:R6Class]{R6} class. } } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-predict}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-SurrogateLearner-predict}{}}} \subsection{Method \code{predict()}}{ Returns mean response and standard error. \subsection{Usage}{ @@ -78,8 +78,8 @@ New data.} } } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-clone}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-SurrogateLearner-clone}{}}} \subsection{Method \code{clone()}}{ The objects of this class are cloneable with this method. \subsection{Usage}{ diff --git a/man/SurrogateLearners.Rd b/man/SurrogateLearners.Rd index 07571645..d06b8e0c 100644 --- a/man/SurrogateLearners.Rd +++ b/man/SurrogateLearners.Rd @@ -25,21 +25,21 @@ Asserts whether the current insample performance meets the performance threshold \section{Methods}{ \subsection{Public methods}{ \itemize{ -\item \href{#method-new}{\code{SurrogateLearners$new()}} -\item \href{#method-predict}{\code{SurrogateLearners$predict()}} -\item \href{#method-clone}{\code{SurrogateLearners$clone()}} -} -} -\if{html}{ -\out{
Inherited methods} -\itemize{ -\item \out{}\href{../../mlr3mbo/html/Surrogate.html#method-update}{\code{mlr3mbo::Surrogate$update()}}\out{} -} -\out{
} -} +\item \href{#method-SurrogateLearners-new}{\code{SurrogateLearners$new()}} +\item \href{#method-SurrogateLearners-predict}{\code{SurrogateLearners$predict()}} +\item \href{#method-SurrogateLearners-clone}{\code{SurrogateLearners$clone()}} +} +} +\if{html}{\out{ +
Inherited methods + +
+}} \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-new}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-SurrogateLearners-new}{}}} \subsection{Method \code{new()}}{ Creates a new instance of this \link[R6:R6Class]{R6} class. \subsection{Usage}{ @@ -59,8 +59,8 @@ Creates a new instance of this \link[R6:R6Class]{R6} class. } } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-predict}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-SurrogateLearners-predict}{}}} \subsection{Method \code{predict()}}{ Returns a named list of data.tables. Each contains the mean response and standard error for one \code{y_col}. @@ -81,8 +81,8 @@ list of \code{\link[data.table:data.table]{data.table::data.table()}}s with the } } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-clone}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-SurrogateLearners-clone}{}}} \subsection{Method \code{clone()}}{ The objects of this class are cloneable with this method. \subsection{Usage}{ diff --git a/man/mlr_acqfunctions_aei.Rd b/man/mlr_acqfunctions_aei.Rd index 723c706b..7efe5b93 100644 --- a/man/mlr_acqfunctions_aei.Rd +++ b/man/mlr_acqfunctions_aei.Rd @@ -10,9 +10,11 @@ Augmented Expected Improvement. \section{Dictionary}{ This \link{AcqFunction} can be instantiated via the \link[mlr3misc:Dictionary]{dictionary} -\link{mlr_acqfunctions} or with the associated sugar function \code{\link[=acqf]{acqf()}}:\preformatted{mlr_acqfunctions$get("aei") +\link{mlr_acqfunctions} or with the associated sugar function \code{\link[=acqf]{acqf()}}: + +\if{html}{\out{
}}\preformatted{mlr_acqfunctions$get("aei") acqf("aei") -} +}\if{html}{\out{
}} } \section{Parameters}{ @@ -53,25 +55,25 @@ Other Acquisition Function: \section{Methods}{ \subsection{Public methods}{ \itemize{ -\item \href{#method-new}{\code{AcqFunctionAEI$new()}} -\item \href{#method-update}{\code{AcqFunctionAEI$update()}} -\item \href{#method-clone}{\code{AcqFunctionAEI$clone()}} -} -} -\if{html}{ -\out{
Inherited methods} -\itemize{ -\item \out{}\href{../../bbotk/html/Objective.html#method-eval}{\code{bbotk::Objective$eval()}}\out{} -\item \out{}\href{../../bbotk/html/Objective.html#method-format}{\code{bbotk::Objective$format()}}\out{} -\item \out{}\href{../../bbotk/html/Objective.html#method-print}{\code{bbotk::Objective$print()}}\out{} -\item \out{}\href{../../mlr3mbo/html/AcqFunction.html#method-eval_dt}{\code{mlr3mbo::AcqFunction$eval_dt()}}\out{} -\item \out{}\href{../../mlr3mbo/html/AcqFunction.html#method-eval_many}{\code{mlr3mbo::AcqFunction$eval_many()}}\out{} -} -\out{
} -} +\item \href{#method-AcqFunctionAEI-new}{\code{AcqFunctionAEI$new()}} +\item \href{#method-AcqFunctionAEI-update}{\code{AcqFunctionAEI$update()}} +\item \href{#method-AcqFunctionAEI-clone}{\code{AcqFunctionAEI$clone()}} +} +} +\if{html}{\out{ +
Inherited methods + +
+}} \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-new}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqFunctionAEI-new}{}}} \subsection{Method \code{new()}}{ Creates a new instance of this \link[R6:R6Class]{R6} class. \subsection{Usage}{ @@ -89,8 +91,8 @@ Creates a new instance of this \link[R6:R6Class]{R6} class. } } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-update}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqFunctionAEI-update}{}}} \subsection{Method \code{update()}}{ Updates acquisition function and sets \code{y_effective_best}. \subsection{Usage}{ @@ -99,8 +101,8 @@ Updates acquisition function and sets \code{y_effective_best}. } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-clone}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqFunctionAEI-clone}{}}} \subsection{Method \code{clone()}}{ The objects of this class are cloneable with this method. \subsection{Usage}{ diff --git a/man/mlr_acqfunctions_cb.Rd b/man/mlr_acqfunctions_cb.Rd index f2f15bd7..68d8806f 100644 --- a/man/mlr_acqfunctions_cb.Rd +++ b/man/mlr_acqfunctions_cb.Rd @@ -10,9 +10,11 @@ Confidence Bound. \section{Dictionary}{ This \link{AcqFunction} can be instantiated via the \link[mlr3misc:Dictionary]{dictionary} -\link{mlr_acqfunctions} or with the associated sugar function \code{\link[=acqf]{acqf()}}:\preformatted{mlr_acqfunctions$get("cb") +\link{mlr_acqfunctions} or with the associated sugar function \code{\link[=acqf]{acqf()}}: + +\if{html}{\out{
}}\preformatted{mlr_acqfunctions$get("cb") acqf("cb") -} +}\if{html}{\out{
}} } \section{Parameters}{ @@ -45,25 +47,25 @@ Other Acquisition Function: \section{Methods}{ \subsection{Public methods}{ \itemize{ -\item \href{#method-new}{\code{AcqFunctionCB$new()}} -\item \href{#method-clone}{\code{AcqFunctionCB$clone()}} -} -} -\if{html}{ -\out{
Inherited methods} -\itemize{ -\item \out{}\href{../../bbotk/html/Objective.html#method-eval}{\code{bbotk::Objective$eval()}}\out{} -\item \out{}\href{../../bbotk/html/Objective.html#method-format}{\code{bbotk::Objective$format()}}\out{} -\item \out{}\href{../../bbotk/html/Objective.html#method-print}{\code{bbotk::Objective$print()}}\out{} -\item \out{}\href{../../mlr3mbo/html/AcqFunction.html#method-eval_dt}{\code{mlr3mbo::AcqFunction$eval_dt()}}\out{} -\item \out{}\href{../../mlr3mbo/html/AcqFunction.html#method-eval_many}{\code{mlr3mbo::AcqFunction$eval_many()}}\out{} -\item \out{}\href{../../mlr3mbo/html/AcqFunction.html#method-update}{\code{mlr3mbo::AcqFunction$update()}}\out{} -} -\out{
} -} +\item \href{#method-AcqFunctionCB-new}{\code{AcqFunctionCB$new()}} +\item \href{#method-AcqFunctionCB-clone}{\code{AcqFunctionCB$clone()}} +} +} +\if{html}{\out{ +
Inherited methods + +
+}} \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-new}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqFunctionCB-new}{}}} \subsection{Method \code{new()}}{ Creates a new instance of this \link[R6:R6Class]{R6} class. \subsection{Usage}{ @@ -81,8 +83,8 @@ Creates a new instance of this \link[R6:R6Class]{R6} class. } } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-clone}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqFunctionCB-clone}{}}} \subsection{Method \code{clone()}}{ The objects of this class are cloneable with this method. \subsection{Usage}{ diff --git a/man/mlr_acqfunctions_ei.Rd b/man/mlr_acqfunctions_ei.Rd index bba1d3cb..f4f22b3e 100644 --- a/man/mlr_acqfunctions_ei.Rd +++ b/man/mlr_acqfunctions_ei.Rd @@ -10,9 +10,11 @@ Expected Improvement. \section{Dictionary}{ This \link{AcqFunction} can be instantiated via the \link[mlr3misc:Dictionary]{dictionary} -\link{mlr_acqfunctions} or with the associated sugar function \code{\link[=acqf]{acqf()}}:\preformatted{mlr_acqfunctions$get("ei") +\link{mlr_acqfunctions} or with the associated sugar function \code{\link[=acqf]{acqf()}}: + +\if{html}{\out{
}}\preformatted{mlr_acqfunctions$get("ei") acqf("ei") -} +}\if{html}{\out{
}} } \references{ @@ -43,25 +45,25 @@ Other Acquisition Function: \section{Methods}{ \subsection{Public methods}{ \itemize{ -\item \href{#method-new}{\code{AcqFunctionEI$new()}} -\item \href{#method-update}{\code{AcqFunctionEI$update()}} -\item \href{#method-clone}{\code{AcqFunctionEI$clone()}} -} -} -\if{html}{ -\out{
Inherited methods} -\itemize{ -\item \out{}\href{../../bbotk/html/Objective.html#method-eval}{\code{bbotk::Objective$eval()}}\out{} -\item \out{}\href{../../bbotk/html/Objective.html#method-format}{\code{bbotk::Objective$format()}}\out{} -\item \out{}\href{../../bbotk/html/Objective.html#method-print}{\code{bbotk::Objective$print()}}\out{} -\item \out{}\href{../../mlr3mbo/html/AcqFunction.html#method-eval_dt}{\code{mlr3mbo::AcqFunction$eval_dt()}}\out{} -\item \out{}\href{../../mlr3mbo/html/AcqFunction.html#method-eval_many}{\code{mlr3mbo::AcqFunction$eval_many()}}\out{} -} -\out{
} -} +\item \href{#method-AcqFunctionEI-new}{\code{AcqFunctionEI$new()}} +\item \href{#method-AcqFunctionEI-update}{\code{AcqFunctionEI$update()}} +\item \href{#method-AcqFunctionEI-clone}{\code{AcqFunctionEI$clone()}} +} +} +\if{html}{\out{ +
Inherited methods + +
+}} \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-new}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqFunctionEI-new}{}}} \subsection{Method \code{new()}}{ Creates a new instance of this \link[R6:R6Class]{R6} class. \subsection{Usage}{ @@ -77,8 +79,8 @@ Creates a new instance of this \link[R6:R6Class]{R6} class. } } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-update}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqFunctionEI-update}{}}} \subsection{Method \code{update()}}{ Updates acquisition function and sets \code{y_best}. \subsection{Usage}{ @@ -87,8 +89,8 @@ Updates acquisition function and sets \code{y_best}. } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-clone}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqFunctionEI-clone}{}}} \subsection{Method \code{clone()}}{ The objects of this class are cloneable with this method. \subsection{Usage}{ diff --git a/man/mlr_acqfunctions_eips.Rd b/man/mlr_acqfunctions_eips.Rd index 35ad4a82..12cd3bf9 100644 --- a/man/mlr_acqfunctions_eips.Rd +++ b/man/mlr_acqfunctions_eips.Rd @@ -17,9 +17,11 @@ surrogate. \section{Dictionary}{ This \link{AcqFunction} can be instantiated via the \link[mlr3misc:Dictionary]{dictionary} -\link{mlr_acqfunctions} or with the associated sugar function \code{\link[=acqf]{acqf()}}:\preformatted{mlr_acqfunctions$get("eips") +\link{mlr_acqfunctions} or with the associated sugar function \code{\link[=acqf]{acqf()}}: + +\if{html}{\out{
}}\preformatted{mlr_acqfunctions$get("eips") acqf("eips") -} +}\if{html}{\out{
}} } \references{ @@ -59,25 +61,25 @@ Other Acquisition Function: \section{Methods}{ \subsection{Public methods}{ \itemize{ -\item \href{#method-new}{\code{AcqFunctionEIPS$new()}} -\item \href{#method-update}{\code{AcqFunctionEIPS$update()}} -\item \href{#method-clone}{\code{AcqFunctionEIPS$clone()}} -} -} -\if{html}{ -\out{
Inherited methods} -\itemize{ -\item \out{}\href{../../bbotk/html/Objective.html#method-eval}{\code{bbotk::Objective$eval()}}\out{} -\item \out{}\href{../../bbotk/html/Objective.html#method-format}{\code{bbotk::Objective$format()}}\out{} -\item \out{}\href{../../bbotk/html/Objective.html#method-print}{\code{bbotk::Objective$print()}}\out{} -\item \out{}\href{../../mlr3mbo/html/AcqFunction.html#method-eval_dt}{\code{mlr3mbo::AcqFunction$eval_dt()}}\out{} -\item \out{}\href{../../mlr3mbo/html/AcqFunction.html#method-eval_many}{\code{mlr3mbo::AcqFunction$eval_many()}}\out{} -} -\out{
} -} +\item \href{#method-AcqFunctionEIPS-new}{\code{AcqFunctionEIPS$new()}} +\item \href{#method-AcqFunctionEIPS-update}{\code{AcqFunctionEIPS$update()}} +\item \href{#method-AcqFunctionEIPS-clone}{\code{AcqFunctionEIPS$clone()}} +} +} +\if{html}{\out{ +
Inherited methods + +
+}} \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-new}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqFunctionEIPS-new}{}}} \subsection{Method \code{new()}}{ Creates a new instance of this \link[R6:R6Class]{R6} class. \subsection{Usage}{ @@ -93,8 +95,8 @@ Creates a new instance of this \link[R6:R6Class]{R6} class. } } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-update}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqFunctionEIPS-update}{}}} \subsection{Method \code{update()}}{ Updates acquisition function and sets \code{y_best}. \subsection{Usage}{ @@ -103,8 +105,8 @@ Updates acquisition function and sets \code{y_best}. } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-clone}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqFunctionEIPS-clone}{}}} \subsection{Method \code{clone()}}{ The objects of this class are cloneable with this method. \subsection{Usage}{ diff --git a/man/mlr_acqfunctions_pi.Rd b/man/mlr_acqfunctions_pi.Rd index 85128254..95d9f87e 100644 --- a/man/mlr_acqfunctions_pi.Rd +++ b/man/mlr_acqfunctions_pi.Rd @@ -10,9 +10,11 @@ Probability of Improvement. \section{Dictionary}{ This \link{AcqFunction} can be instantiated via the \link[mlr3misc:Dictionary]{dictionary} -\link{mlr_acqfunctions} or with the associated sugar function \code{\link[=acqf]{acqf()}}:\preformatted{mlr_acqfunctions$get("pi") +\link{mlr_acqfunctions} or with the associated sugar function \code{\link[=acqf]{acqf()}}: + +\if{html}{\out{
}}\preformatted{mlr_acqfunctions$get("pi") acqf("pi") -} +}\if{html}{\out{
}} } \references{ @@ -43,25 +45,25 @@ Other Acquisition Function: \section{Methods}{ \subsection{Public methods}{ \itemize{ -\item \href{#method-new}{\code{AcqFunctionPI$new()}} -\item \href{#method-update}{\code{AcqFunctionPI$update()}} -\item \href{#method-clone}{\code{AcqFunctionPI$clone()}} -} -} -\if{html}{ -\out{
Inherited methods} -\itemize{ -\item \out{}\href{../../bbotk/html/Objective.html#method-eval}{\code{bbotk::Objective$eval()}}\out{} -\item \out{}\href{../../bbotk/html/Objective.html#method-format}{\code{bbotk::Objective$format()}}\out{} -\item \out{}\href{../../bbotk/html/Objective.html#method-print}{\code{bbotk::Objective$print()}}\out{} -\item \out{}\href{../../mlr3mbo/html/AcqFunction.html#method-eval_dt}{\code{mlr3mbo::AcqFunction$eval_dt()}}\out{} -\item \out{}\href{../../mlr3mbo/html/AcqFunction.html#method-eval_many}{\code{mlr3mbo::AcqFunction$eval_many()}}\out{} -} -\out{
} -} +\item \href{#method-AcqFunctionPI-new}{\code{AcqFunctionPI$new()}} +\item \href{#method-AcqFunctionPI-update}{\code{AcqFunctionPI$update()}} +\item \href{#method-AcqFunctionPI-clone}{\code{AcqFunctionPI$clone()}} +} +} +\if{html}{\out{ +
Inherited methods + +
+}} \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-new}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqFunctionPI-new}{}}} \subsection{Method \code{new()}}{ Creates a new instance of this \link[R6:R6Class]{R6} class. \subsection{Usage}{ @@ -77,8 +79,8 @@ Creates a new instance of this \link[R6:R6Class]{R6} class. } } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-update}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqFunctionPI-update}{}}} \subsection{Method \code{update()}}{ Updates acquisition function and sets \code{y_best}. \subsection{Usage}{ @@ -87,8 +89,8 @@ Updates acquisition function and sets \code{y_best}. } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-clone}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqFunctionPI-clone}{}}} \subsection{Method \code{clone()}}{ The objects of this class are cloneable with this method. \subsection{Usage}{ diff --git a/man/mlr_acqfunctions_sms_ego.Rd b/man/mlr_acqfunctions_sms_ego.Rd index 931d88c4..3fc68404 100644 --- a/man/mlr_acqfunctions_sms_ego.Rd +++ b/man/mlr_acqfunctions_sms_ego.Rd @@ -60,25 +60,25 @@ Other Acquisition Function: \section{Methods}{ \subsection{Public methods}{ \itemize{ -\item \href{#method-new}{\code{AcqFunctionSmsEgo$new()}} -\item \href{#method-update}{\code{AcqFunctionSmsEgo$update()}} -\item \href{#method-clone}{\code{AcqFunctionSmsEgo$clone()}} -} -} -\if{html}{ -\out{
Inherited methods} -\itemize{ -\item \out{}\href{../../bbotk/html/Objective.html#method-eval}{\code{bbotk::Objective$eval()}}\out{} -\item \out{}\href{../../bbotk/html/Objective.html#method-format}{\code{bbotk::Objective$format()}}\out{} -\item \out{}\href{../../bbotk/html/Objective.html#method-print}{\code{bbotk::Objective$print()}}\out{} -\item \out{}\href{../../mlr3mbo/html/AcqFunction.html#method-eval_dt}{\code{mlr3mbo::AcqFunction$eval_dt()}}\out{} -\item \out{}\href{../../mlr3mbo/html/AcqFunction.html#method-eval_many}{\code{mlr3mbo::AcqFunction$eval_many()}}\out{} -} -\out{
} -} +\item \href{#method-AcqFunctionSmsEgo-new}{\code{AcqFunctionSmsEgo$new()}} +\item \href{#method-AcqFunctionSmsEgo-update}{\code{AcqFunctionSmsEgo$update()}} +\item \href{#method-AcqFunctionSmsEgo-clone}{\code{AcqFunctionSmsEgo$clone()}} +} +} +\if{html}{\out{ +
Inherited methods + +
+}} \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-new}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqFunctionSmsEgo-new}{}}} \subsection{Method \code{new()}}{ Creates a new instance of this \link[R6:R6Class]{R6} class. \subsection{Usage}{ @@ -98,8 +98,8 @@ Creates a new instance of this \link[R6:R6Class]{R6} class. } } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-update}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqFunctionSmsEgo-update}{}}} \subsection{Method \code{update()}}{ Updates acquisition function and sets \code{ys_front}, \code{ref_point}, \code{epsilon}. \subsection{Usage}{ @@ -108,8 +108,8 @@ Updates acquisition function and sets \code{ys_front}, \code{ref_point}, \code{e } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-clone}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqFunctionSmsEgo-clone}{}}} \subsection{Method \code{clone()}}{ The objects of this class are cloneable with this method. \subsection{Usage}{ diff --git a/man/mlr_learners_regr.ranger_custom.Rd b/man/mlr_learners_regr.ranger_custom.Rd index a161bb05..5c674f8e 100644 --- a/man/mlr_learners_regr.ranger_custom.Rd +++ b/man/mlr_learners_regr.ranger_custom.Rd @@ -9,24 +9,26 @@ Custom random regression forest. Calls \code{\link[ranger:ranger]{ranger::ranger()}} from package \CRANpkg{ranger}. This custom random regression forest tries to mimic the behaviour of a Gaussian process and therefore may be more suitable for Bayesian optimization compared to a normal random regression forest. -By default, the \code{"extratrees"} splitting rule is used with \code{num.random.splits = 1}. +The \code{"extratrees"} splitting rule is used with \code{num.random.splits = 1}. This results in a rather smooth prediction function. No bootstrap or subsampling is used but rather all training data (\code{replace = FALSE} and \code{sample.fraction = 1}). This may seem counter-intuitive at first but ensures that training data is almost perfectly interpolated (given \code{min.node.size = 1}). Per default, \code{1000} trees are used. -Classical standard error estimation is no longer possible but \code{se.method = "simple"} allows for estimating the variance based on the variance of the trees due to the \code{"extratrees"} splitting rule (simply taking the variance over the predictions of the trees). +Classical standard error estimation is no longer possible, but the \code{"extratrees"} splitting rule allows for estimating the variance based on the variance of the trees (simply taking the variance over the predictions of the trees). This results in low variance close to training data points. If \code{se.simple.spatial = TRUE} this point-wise variance is upwardly biased by adding the pointwise variance again scaled by the minimum Gower distance of the point to the training data (recall that a Gower distance of 0 means an identical point is present in the training data, whereas a distance of 1 means total dissimilarity to the training data). This ensures that the variance is larger in regions dissimilar to the training data. Summarizing, the standard error for a point is calculated as follows: -\deqn{\hat{\mathrm{SE}}(x_{i}) = \sqrt{\hat{\mathrm{VAR}}(x_{i}) + (\hat{\mathrm{VAR}}(x_{i}) + \epsilon) * \mathrm{GD}_{i})}} +\deqn{\hat{\mathrm{SE}}(x_{i}) = \sqrt{(\hat{\mathrm{VAR}}(x_{i}) + \epsilon) + (\hat{\mathrm{VAR}}(x_{i}) + \epsilon) * \mathrm{GD}_{i})}} Here, \eqn{\epsilon} is given by \code{se.simple.spatial.nugget} which can be set larger than 0 in the case of noisy function observations. } \section{Dictionary}{ -This \link{Learner} can be instantiated via the \link[mlr3misc:Dictionary]{dictionary} \link{mlr_learners} or with the associated sugar function \code{\link[=lrn]{lrn()}}:\preformatted{mlr_learners$get("regr.ranger_custom") +This \link{Learner} can be instantiated via the \link[mlr3misc:Dictionary]{dictionary} \link{mlr_learners} or with the associated sugar function \code{\link[=lrn]{lrn()}}: + +\if{html}{\out{
}}\preformatted{mlr_learners$get("regr.ranger_custom") lrn("regr.ranger_custom") -} +}\if{html}{\out{
}} } \section{Meta Information}{ @@ -42,35 +44,26 @@ lrn("regr.ranger_custom") \section{Parameters}{ \tabular{lllll}{ Id \tab Type \tab Default \tab Levels \tab Range \cr - alpha \tab numeric \tab 0.5 \tab \tab \eqn{(-\infty, \infty)}{(-Inf, Inf)} \cr always.split.variables \tab untyped \tab - \tab \tab - \cr holdout \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr importance \tab character \tab - \tab none, impurity, impurity_corrected, permutation \tab - \cr keep.inbag \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr local.importance \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr max.depth \tab integer \tab NULL \tab \tab \eqn{[0, \infty)}{[0, Inf)} \cr - min.node.size \tab integer \tab 1 \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr - min.prop \tab numeric \tab 0.1 \tab \tab \eqn{(-\infty, \infty)}{(-Inf, Inf)} \cr - minprop \tab numeric \tab 0.1 \tab \tab \eqn{(-\infty, \infty)}{(-Inf, Inf)} \cr mtry \tab integer \tab - \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr mtry.ratio \tab numeric \tab - \tab \tab \eqn{[0, 1]}{[0, 1]} \cr - num.random.splits \tab integer \tab 1 \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr num.threads \tab integer \tab 1 \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr num.trees \tab integer \tab 1000 \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr - oob.error \tab logical \tab TRUE \tab TRUE, FALSE \tab - \cr + oob.error \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr regularization.factor \tab untyped \tab 1 \tab \tab - \cr regularization.usedepth \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr - replace \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr respect.unordered.factors \tab character \tab ignore \tab ignore, order, partition \tab - \cr - sample.fraction \tab numeric \tab 1 \tab \tab \eqn{[0, 1]}{[0, 1]} \cr save.memory \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr scale.permutation.importance \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr - se.method \tab character \tab simple \tab jack, infjack, simple \tab - \cr se.simple.spatial \tab logical \tab TRUE \tab TRUE, FALSE \tab - \cr - se.simple.spatial.nugget \tab numeric \tab 0 \tab \tab \eqn{[0, 1]}{[0, 1]} \cr + se.simple.spatial.nugget \tab numeric \tab 0 \tab \tab \eqn{[0, \infty)}{[0, Inf)} \cr seed \tab integer \tab NULL \tab \tab \eqn{(-\infty, \infty)}{(-Inf, Inf)} \cr split.select.weights \tab untyped \tab \tab \tab - \cr - splitrule \tab character \tab extratrees \tab variance, extratrees, maxstat \tab - \cr verbose \tab logical \tab TRUE \tab TRUE, FALSE \tab - \cr write.forest \tab logical \tab TRUE \tab TRUE, FALSE \tab - \cr } @@ -122,29 +115,29 @@ for established default tuning spaces. \section{Methods}{ \subsection{Public methods}{ \itemize{ -\item \href{#method-new}{\code{LearnerRegrRangerCustom$new()}} -\item \href{#method-importance}{\code{LearnerRegrRangerCustom$importance()}} -\item \href{#method-oob_error}{\code{LearnerRegrRangerCustom$oob_error()}} -\item \href{#method-clone}{\code{LearnerRegrRangerCustom$clone()}} -} -} -\if{html}{ -\out{
Inherited methods} -\itemize{ -\item \out{}\href{../../mlr3/html/Learner.html#method-base_learner}{\code{mlr3::Learner$base_learner()}}\out{} -\item \out{}\href{../../mlr3/html/Learner.html#method-format}{\code{mlr3::Learner$format()}}\out{} -\item \out{}\href{../../mlr3/html/Learner.html#method-help}{\code{mlr3::Learner$help()}}\out{} -\item \out{}\href{../../mlr3/html/Learner.html#method-predict}{\code{mlr3::Learner$predict()}}\out{} -\item \out{}\href{../../mlr3/html/Learner.html#method-predict_newdata}{\code{mlr3::Learner$predict_newdata()}}\out{} -\item \out{}\href{../../mlr3/html/Learner.html#method-print}{\code{mlr3::Learner$print()}}\out{} -\item \out{}\href{../../mlr3/html/Learner.html#method-reset}{\code{mlr3::Learner$reset()}}\out{} -\item \out{}\href{../../mlr3/html/Learner.html#method-train}{\code{mlr3::Learner$train()}}\out{} -} -\out{
} -} +\item \href{#method-LearnerRegrRangerCustom-new}{\code{LearnerRegrRangerCustom$new()}} +\item \href{#method-LearnerRegrRangerCustom-importance}{\code{LearnerRegrRangerCustom$importance()}} +\item \href{#method-LearnerRegrRangerCustom-oob_error}{\code{LearnerRegrRangerCustom$oob_error()}} +\item \href{#method-LearnerRegrRangerCustom-clone}{\code{LearnerRegrRangerCustom$clone()}} +} +} +\if{html}{\out{ +
Inherited methods + +
+}} \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-new}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-LearnerRegrRangerCustom-new}{}}} \subsection{Method \code{new()}}{ Creates a new instance of this \link[R6:R6Class]{R6} class. \subsection{Usage}{ @@ -153,8 +146,8 @@ Creates a new instance of this \link[R6:R6Class]{R6} class. } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-importance}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-LearnerRegrRangerCustom-importance}{}}} \subsection{Method \code{importance()}}{ The importance scores are extracted from the model slot \code{variable.importance}. Parameter \code{importance.mode} must be set to \code{"impurity"}, \code{"impurity_corrected"}, or @@ -168,8 +161,8 @@ Named \code{numeric()}. } } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-oob_error}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-LearnerRegrRangerCustom-oob_error}{}}} \subsection{Method \code{oob_error()}}{ The out-of-bag error, extracted from model slot \code{prediction.error}. \subsection{Usage}{ @@ -181,8 +174,8 @@ The out-of-bag error, extracted from model slot \code{prediction.error}. } } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-clone}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-LearnerRegrRangerCustom-clone}{}}} \subsection{Method \code{clone()}}{ The objects of this class are cloneable with this method. \subsection{Usage}{ diff --git a/man/mlr_optimizers_mbo.Rd b/man/mlr_optimizers_mbo.Rd index a81ce395..4f52a7ef 100644 --- a/man/mlr_optimizers_mbo.Rd +++ b/man/mlr_optimizers_mbo.Rd @@ -33,23 +33,23 @@ to be used, e.g., \link{bayesopt_ego} for sequential single objective MBO, a \li \section{Methods}{ \subsection{Public methods}{ \itemize{ -\item \href{#method-new}{\code{OptimizerMbo$new()}} -\item \href{#method-clone}{\code{OptimizerMbo$clone()}} -} -} -\if{html}{ -\out{
Inherited methods} -\itemize{ -\item \out{}\href{../../bbotk/html/Optimizer.html#method-format}{\code{bbotk::Optimizer$format()}}\out{} -\item \out{}\href{../../bbotk/html/Optimizer.html#method-help}{\code{bbotk::Optimizer$help()}}\out{} -\item \out{}\href{../../bbotk/html/Optimizer.html#method-optimize}{\code{bbotk::Optimizer$optimize()}}\out{} -\item \out{}\href{../../bbotk/html/Optimizer.html#method-print}{\code{bbotk::Optimizer$print()}}\out{} -} -\out{
} -} +\item \href{#method-OptimizerMbo-new}{\code{OptimizerMbo$new()}} +\item \href{#method-OptimizerMbo-clone}{\code{OptimizerMbo$clone()}} +} +} +\if{html}{\out{ +
Inherited methods + +
+}} \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-new}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-OptimizerMbo-new}{}}} \subsection{Method \code{new()}}{ Creates a new instance of this \link[R6:R6Class]{R6} class. For more information on default values for \code{loop_function}, \code{surrogate}, \code{acq_function} and \code{acq_optimizer}, see \code{mbo_defaults}. @@ -95,8 +95,8 @@ See for example \link{result_by_surrogate_design} which is used by default if th } } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-clone}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-OptimizerMbo-clone}{}}} \subsection{Method \code{clone()}}{ The objects of this class are cloneable with this method. \subsection{Usage}{ diff --git a/man/mlr_tuners_mbo.Rd b/man/mlr_tuners_mbo.Rd index 3ba55d7f..03c4f5a6 100644 --- a/man/mlr_tuners_mbo.Rd +++ b/man/mlr_tuners_mbo.Rd @@ -32,23 +32,23 @@ For additional information and documentation see \link{OptimizerMbo}. \section{Methods}{ \subsection{Public methods}{ \itemize{ -\item \href{#method-new}{\code{TunerMbo$new()}} -\item \href{#method-clone}{\code{TunerMbo$clone()}} -} -} -\if{html}{ -\out{
Inherited methods} -\itemize{ -\item \out{}\href{../../mlr3tuning/html/Tuner.html#method-format}{\code{mlr3tuning::Tuner$format()}}\out{} -\item \out{}\href{../../mlr3tuning/html/Tuner.html#method-help}{\code{mlr3tuning::Tuner$help()}}\out{} -\item \out{}\href{../../mlr3tuning/html/Tuner.html#method-print}{\code{mlr3tuning::Tuner$print()}}\out{} -\item \out{}\href{../../mlr3tuning/html/TunerFromOptimizer.html#method-optimize}{\code{mlr3tuning::TunerFromOptimizer$optimize()}}\out{} -} -\out{
} -} +\item \href{#method-TunerMbo-new}{\code{TunerMbo$new()}} +\item \href{#method-TunerMbo-clone}{\code{TunerMbo$clone()}} +} +} +\if{html}{\out{ +
Inherited methods + +
+}} \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-new}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-TunerMbo-new}{}}} \subsection{Method \code{new()}}{ Creates a new instance of this \link[R6:R6Class]{R6} class. For more information on default values for \code{loop_function}, \code{surrogate}, \code{acq_function} and \code{acq_optimizer}, see \code{mbo_defaults}. @@ -97,8 +97,8 @@ See for example \link{result_by_surrogate_design} which is used by default if th } } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-clone}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-TunerMbo-clone}{}}} \subsection{Method \code{clone()}}{ The objects of this class are cloneable with this method. \subsection{Usage}{ From 31022b4ffb672ef917d697e3a39a5b01dc6c1db2 Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Tue, 30 Aug 2022 17:22:44 +0200 Subject: [PATCH 011/126] .. --- attic/so_config/run_yahpo.R | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/attic/so_config/run_yahpo.R b/attic/so_config/run_yahpo.R index c7970d06..2f02bed2 100644 --- a/attic/so_config/run_yahpo.R +++ b/attic/so_config/run_yahpo.R @@ -7,11 +7,14 @@ library(mlr3misc) library(mlr3mbo) # @so_config library(bbotk) library(paradox) +library(R6) +library(checkmate) + reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) library(reticulate) yahpo_gym = import("yahpo_gym") -packages = c("data.table", "mlr3", "mlr3learners", "mlr3pipelines", "mlr3misc", "mlr3mbo", "bbotk", "paradox", "mlrintermbo") +packages = c("data.table", "mlr3", "mlr3learners", "mlr3pipelines", "mlr3misc", "mlr3mbo", "bbotk", "paradox", "mlrintermbo", "R6", "checkmate") RhpcBLASctl::blas_set_num_threads(1L) RhpcBLASctl::omp_set_num_threads(1L) @@ -272,6 +275,7 @@ mlr3mbo_wrapper_new_rf_ls = function(job, data, instance, ...) { acq_optimizer$param_set$values$warmstart = TRUE acq_optimizer$param_set$values$warmstart_size = "all" + acq_optimizer$param_set$values$logging_level = "info" bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) optim_instance From 0105890b7f14fae89d4fc533317286a8bbf249ac Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Fri, 2 Sep 2022 17:57:56 +0200 Subject: [PATCH 012/126] new ac setup --- R/AcqOptimizer.R | 4 +- attic/so_config/analyze_yahpo.R | 2 +- attic/so_config/get_filling_candidate.R | 8 ++ attic/so_config/run.R | 138 ++++++++++++++---------- attic/so_config/run_yahpo.R | 7 +- 5 files changed, 93 insertions(+), 66 deletions(-) create mode 100644 attic/so_config/get_filling_candidate.R diff --git a/R/AcqOptimizer.R b/R/AcqOptimizer.R index 5c7d1c89..0b515f95 100644 --- a/R/AcqOptimizer.R +++ b/R/AcqOptimizer.R @@ -138,8 +138,8 @@ get_best_not_evaluated = function(instance, evaluated) { assert_r6(instance, classes = "OptimInstanceSingleCrit") data = copy(instance$archive$data[, c(instance$archive$cols_x, instance$archive$cols_y), with = FALSE]) evaluated = copy(evaluated) - data[, overlap := FALSE][evaluated, overlap := TRUE, on = instance$archive$cols_x] - candidates = data[overlap == FALSE] + data[, .overlap := FALSE][evaluated, .overlap := TRUE, on = instance$archive$cols_x] + candidates = data[.overlap == FALSE] candidates[[instance$archive$cols_y]] = candidates[[instance$archive$cols_y]] * instance$objective_multiplicator[instance$archive$cols_y] xdt = setorderv(candidates, cols = instance$archive$cols_y)[1L, ] xdt[[instance$archive$cols_y]] = xdt[[instance$archive$cols_y]] * instance$objective_multiplicator[instance$archive$cols_y] diff --git a/attic/so_config/analyze_yahpo.R b/attic/so_config/analyze_yahpo.R index a6083844..3ab050c6 100644 --- a/attic/so_config/analyze_yahpo.R +++ b/attic/so_config/analyze_yahpo.R @@ -4,7 +4,7 @@ library(pammtools) library(mlr3misc) #dat = rbind(readRDS("results_yahpo.rds"), readRDS("results_yahpo_own.rds"))[method != "mlrintermbo"] -dat = rbind(readRDS("results_yahpo.rds"), readRDS("results_yahpo_own.rds"))[method %in% c("mlr3mbo", "mlrintermbo", "random", "smac4hpo", "smac4mf", "mlr3mbo_new_rf", "mlr3mbo_new_rf_ls")] +dat = rbind(readRDS("results_yahpo.rds"), readRDS("results_yahpo_own.rds"))[method %in% c("mlr3mbo", "mlrintermbo", "mlr3mbo_new_rf", "mlr3mbo_new_rf_ls", "smac4hpo", "random")] dat = dat[scenario %nin% c("nb301", "rbv2_super")] dat[, cumbudget := cumsum(budget), by = .(method, scenario, instance, repl)] dat[, cumbudget_scaled := cumbudget / max(cumbudget), by = .(method, scenario, instance, repl)] diff --git a/attic/so_config/get_filling_candidate.R b/attic/so_config/get_filling_candidate.R new file mode 100644 index 00000000..0e9f8ff5 --- /dev/null +++ b/attic/so_config/get_filling_candidate.R @@ -0,0 +1,8 @@ +get_filling_candidate = function(instance) { + newdata = generate_design_random(instance$search_space, n = 10000L)$data + gw_dists = get_gower_dist(fct_to_char(newdata), instance$archive$data[, instance$archive$cols_x, with = FALSE]) # 0 if identical, 1 if maximally dissimilar + min_gw_dists = apply(gw_dists, MARGIN = 1L, FUN = min) # get the minium for each new point to the points in the archive + which.max(min_gw_dists) + newdata[which.max(min_gw_dists), ] +} + diff --git a/attic/so_config/run.R b/attic/so_config/run.R index 327a5aa3..d6c7c60d 100755 --- a/attic/so_config/run.R +++ b/attic/so_config/run.R @@ -8,6 +8,8 @@ library(mlr3learners) library(mlr3pipelines) library(bbotk) library(paradox) +library(R6) +library(checkmate) library(mlr3mbo) # @so_config reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) library(reticulate) @@ -15,38 +17,51 @@ library(yahpogym) library("future") library("future.apply") -# FIXME: test stability stuff? +source("OptimizerChain.R") parser = ArgumentParser() parser$add_argument("-r", "--run", type = "integer", default = 1, help = "Id of run, should be within 1-5") args = parser$parse_args() run_id = args$run -stopifnot(run_id %in% 1:5) +stopifnot(run_id %in% 1:3) cat("run_id:", run_id, "\n") -seeds = c(2409, 2906, 0905, 1234, 1010) +seeds = c(2409, 2906, 0905) set.seed(seeds[run_id]) search_space = ps( - init = p_fct(c("random", "lhs")), - init_size_factor = p_int(lower = 1L, upper = 6L), + init = p_fct(c("random", "lhs", "sobol")), + init_size_factor = p_int(lower = 1L, upper = 4L), random_interleave = p_lgl(), - random_interleave_iter = p_int(lower = 2L, upper = 10L, depends = random_interleave == TRUE), - #surrogate = p_fct(c("GP", "RF")), # FIXME: BANANAS NN ensemble - #splitrule = p_fct(c("variance", "extratrees"), depends = surrogate == "RF"), - #num.random.splits = p_int(lower = 1L, upper = 10L, depends = surrogate == "RF" && splitrule == "extratrees"), - num.trees = p_int(lower = 100L, upper = 2000L), - splitrule = p_fct(c("variance", "extratrees")), - num.random.splits = p_int(lower = 1L, upper = 10L, depends = splitrule == "extratrees"), - acqf = p_fct(c("EI", "CB", "PI")), - lambda = p_dbl(lower = 0, upper = 3L, depends = acqf == "CB"), - acqopt_iter_factor = p_int(lower = 1L, upper = 20L), # lowest acqopt_iter is 1000 * 1 - acqopt = p_fct(c("RS", "FS")), # FIXME: miesmuschel - fs_behavior = p_fct(c("global", "local"), depends = acqopt == "FS") + random_interleave_iter = p_fct(c("2", "3", "5", "10"), depends = random_interleave == TRUE), + + surrogate = p_fct(c("ranger", "ranger_custom")), + mtry.ratio = p_fct(c("default", "1")), + num.trees = p_fct(c("10", "100", "1000")), + replace = p_lgl(depends = surrogate == "ranger"), + sample.fraction = p_fct(c("0.6", "0.8", "1"), depends = surrogate == "ranger"), + splitrule = p_fct(c("variance", "extratrees"), depends = surrogate == "ranger"), + num.random.splits = p_fct(c("1", "2", "10"), depends = surrogate == "ranger" && splitrule == "extratrees"), + min.node.size = p_fct(c("1", "5"), depends = surrogate == "ranger"), + se.method = p_fct(c("jack", "infjack"), depends = surrogate == "ranger"), + se.simple.spatial = p_lgl(depends = surrogate == "ranger_custom"), + + acqf = p_fct(c("EI", "CB")), + lambda = p_fct(c("1", "2", "3"), depends = acqf == "CB"), + acqopt = p_fct(c("RS_1000", "RS", "FS", "LS")) ) +search_space$trafo = function(x, param_set) { + x[["random_interleave_iter"]] = as.integer(x[["random_interleave_iter"]]) + x[["num.trees"]] = as.integer(x[["num.trees"]]) + x[["sample.fraction"]] = as.numeric(x[["sample.fraction"]]) + x[["num.random.splits"]] = as.integer(x[["num.random.splits"]]) + x[["min.node.size"]] = as.integer(x[["min.node.size"]]) + x[["lambda"]] = as.numeric(x[["lambda"]]) + x[map_lgl(x, function(y) length(y) == 0L)] = NA + x +} -# budget = ceiling(20 + sqrt(d) * 40) instances = readRDS("instances.rds") #instances = data.table(scenario = rep(paste0("rbv2_", c("aknn", "glmnet", "ranger", "rpart", "super", "svm", "xgboost")), each = 5L), # instance = c("40499", "1476", "6", "12", "41150", @@ -78,59 +93,59 @@ evaluate = function(xdt, instance) { d = optim_instance$search_space$length init_design_size = d * xdt$init_size_factor - init_design = if (xdt$init == "random") generate_design_random(optim_instance$search_space, n = init_design_size)$data else if (xdt$init == "lhs") generate_design_lhs(optim_instance$search_space, n = init_design_size)$data + init_design = if (xdt$init == "random") { + generate_design_random(optim_instance$search_space, n = init_design_size)$data + } else if (xdt$init == "lhs") { + generate_design_lhs(optim_instance$search_space, n = init_design_size)$data + } else if (xdt$init == "sobol") { + generate_design_sobol(optim_instance$search_space, n = init_design_size)$data + } + optim_instance$eval_batch(init_design) random_interleave_iter = if(xdt$random_interleave) xdt$random_interleave_iter else 0L - #surrogate = if(xdt$surrogate == "GP") { - # learner = lrn("regr.km", covtype = "matern3_2", optim.method = "gen", nugget.stability = 10^-8) - # SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% po("encodeimpact", affect_columns = selector_type(c("logical", "character", "factor", "ordered"))) %>>% learner)) - #} else if (xdt$surrogate == "RF") { - # learner = if (xdt$splitrule == "extratrees") { - # lrn("regr.ranger", num.trees = xdt$num.trees, keep.inbag = TRUE, splitrule = xdt$splitrule, num.random.splits = xdt$num.random.splits) - # } else if (xdt$splitrule == "variance") { - # lrn("regr.ranger", num.trees = xdt$num.trees, keep.inbag = TRUE, splitrule = xdt$splitrule) - # } - # SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% learner)) - #} - - learner = if (xdt$splitrule == "extratrees") { - lrn("regr.ranger", num.trees = xdt$num.trees, keep.inbag = TRUE, splitrule = xdt$splitrule, num.random.splits = xdt$num.random.splits) - } else if (xdt$splitrule == "variance") { - lrn("regr.ranger", num.trees = xdt$num.trees, keep.inbag = TRUE, splitrule = xdt$splitrule) + if (xdt$surrogate == "ranger") { + learner = lrn("regr.ranger", keep.inbag = TRUE) + values = as.list(xdt[, c("num.trees", "replace", "sample.fraction", "splitrule", "num.random.splits", "min.node.size", "se.method")]) + values = values[!map_lgl(values, function(x) is.na(x))] + } else if (xdt$surrogate == "ranger_custom") { + learner = lrn("regr.ranger_custom") + values = as.list(xdt[, c("num.trees", "se.simple.spatial")]) + values = values[!map_lgl(values, function(x) is.na(x))] + } + if (xdt$mtry.ratio == "1") { + values$mtry.ratio = 1 } + learner$param_set$values = insert_named(learner$param_set$values %??% list(), values) surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% learner)) acq_function = if (xdt$acqf == "EI") { AcqFunctionEI$new() } else if (xdt$acqf == "CB") { AcqFunctionCB$new(lambda = xdt$lambda) - } else if (xdt$acqf == "PI") { - AcqFunctionPI$new() } - - acq_budget = 1000 * xdt$acqopt_iter_factor - - acq_optimizer = if (xdt$acqopt == "RS") { - AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = acq_budget)) + + acq_optimizer = if (xdt$acqopt == "RS_1000") { + AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 1000L)) + } else if (xdt$acqopt == "RS") { + AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 20000L)) } else if (xdt$acqopt == "FS") { - if (xdt$fs_behavior == "global") { - n_repeats = 10L - maxit = 2L - batch_size = ceiling((acq_budget / n_repeats) / (1 + maxit)) - } else if (xdt$fs_behavior == "local") { n_repeats = 2L - maxit = 10L - batch_size = ceiling((acq_budget / n_repeats) / (1 + maxit)) - } - AcqOptimizer$new(opt("focus_search", n_points = batch_size, maxit = maxit), terminator = trm("evals", n_evals = acq_budget)) + maxit = 9L + batch_size = ceiling((20000L / n_repeats) / (1 + maxit)) # 1000L + AcqOptimizer$new(opt("focus_search", n_points = batch_size, maxit = maxit), terminator = trm("evals", n_evals = 20000L)) + } else if (xdt$acqopt == "LS") { + optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) + acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20020L)) + acq_optimizer$param_set$values$warmstart = TRUE + acq_optimizer$param_set$values$warmstart_size = "all" + acq_optimizer } bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) best = optim_instance$archive$best()[[instance$target]] - # (best - instance$mean) / instance$sd # normalize best w.r.t min_max.R empirical mean and sd ecdf_best = instance$ecdf(best) # evaluate the precomputed ecdf for the best value found; our target is effectively P(X <= best) cat("scenario:", instance$scenario, "instance:", instance$instance, "ECDF_best:", ecdf_best, "\n") ecdf_best @@ -142,24 +157,29 @@ objective = ObjectiveRFunDt$new( map_dtr(seq_len(nrow(xdt)), function(i) { plan("multicore") tmp = future_lapply(transpose_list(instances), function(instance) { - res_instance = tryCatch(evaluate(xdt[i, ], instance), error = function(error_condition) NA_real_) + res_instance = tryCatch(evaluate(xdt[i, ], instance), error = function(error_condition) 0) }, future.seed = TRUE) data.table(mean_perf = mean(unlist(tmp), na.rm = TRUE), raw_perfs = list(tmp), n_na = sum(is.na(unlist(tmp)))) }) }, domain = search_space, - codomain = ps(mean_perf = p_dbl(tags = "maximize")) + codomain = ps(mean_perf = p_dbl(tags = "maximize")), + check_values = FALSE ) ac_instance = OptimInstanceSingleCrit$new( objective = objective, - terminator = trm("evals", n_evals = 230L) # 30 init design + 200 + search_space = search_space, + terminator = trm("evals", n_evals = 200L) # 100 init design + 100 ) -surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% lrn("regr.ranger", num.trees = 2000L, keep.inbag = TRUE))) -acq_function = AcqFunctionEI$new() -acq_optimizer = AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 10000L)) -design = generate_design_lhs(ac_instance$search_space, n = 30L)$data +surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% lrn("regr.ranger", num.trees = 1000L, se.method = "jack", keep.inbag = TRUE))) +acq_function = AcqFunctionCB$new(lambda = 3) +optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) +acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20020L)) +acq_optimizer$param_set$values$warmstart = TRUE +acq_optimizer$param_set$values$warmstart_size = "all" +design = generate_design_sobol(ac_instance$search_space, n = 100L)$data ac_instance$eval_batch(design) saveRDS(ac_instance, paste0("ac_instance_", run_id, ".rds")) bayesopt_ego(ac_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = 5L) diff --git a/attic/so_config/run_yahpo.R b/attic/so_config/run_yahpo.R index 2f02bed2..1503dfaa 100644 --- a/attic/so_config/run_yahpo.R +++ b/attic/so_config/run_yahpo.R @@ -270,12 +270,11 @@ mlr3mbo_wrapper_new_rf_ls = function(job, data, instance, ...) { acq_function = AcqFunctionEI$new() - optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10000L), trm("evals", n_evals = 10000L))) - acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20000L)) + optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) + acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20010L)) acq_optimizer$param_set$values$warmstart = TRUE acq_optimizer$param_set$values$warmstart_size = "all" - acq_optimizer$param_set$values$logging_level = "info" bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) optim_instance @@ -373,7 +372,7 @@ for (i in seq_len(nrow(optimizers))) { } jobs = findJobs() -resources.default = list(walltime = 3600 * 6L, memory = 2048L, ntasks = 1L, ncpus = 1L, nodes = 1L, clusters = "teton", max.concurrent.jobs = 9999L) +resources.default = list(walltime = 3600 * 9L, memory = 2048L, ntasks = 1L, ncpus = 1L, nodes = 1L, clusters = "teton", max.concurrent.jobs = 9999L) submitJobs(jobs, resources = resources.default) done = findDone() From 630a330220d0cbdff2790575ac7ecba67e48a51f Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Fri, 2 Sep 2022 18:23:54 +0200 Subject: [PATCH 013/126] .. --- attic/so_config/run.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/attic/so_config/run.R b/attic/so_config/run.R index d6c7c60d..28d9b93f 100755 --- a/attic/so_config/run.R +++ b/attic/so_config/run.R @@ -23,10 +23,10 @@ parser = ArgumentParser() parser$add_argument("-r", "--run", type = "integer", default = 1, help = "Id of run, should be within 1-5") args = parser$parse_args() run_id = args$run -stopifnot(run_id %in% 1:3) +stopifnot(run_id %in% 1:5) cat("run_id:", run_id, "\n") -seeds = c(2409, 2906, 0905) +seeds = c(2409, 2906, 0905, 2412, 3112) set.seed(seeds[run_id]) From 3dbe05ab2dbdbd06732b0662e2d95c98607377fa Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Tue, 6 Sep 2022 01:00:35 +0200 Subject: [PATCH 014/126] .. --- attic/so_config/run_yahpo.R | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/attic/so_config/run_yahpo.R b/attic/so_config/run_yahpo.R index 1503dfaa..1c8a1abf 100644 --- a/attic/so_config/run_yahpo.R +++ b/attic/so_config/run_yahpo.R @@ -280,6 +280,37 @@ mlr3mbo_wrapper_new_rf_ls = function(job, data, instance, ...) { optim_instance } +mlr3mbo_wrapper_xxx_ls = function(job, data, instance, ...) { + reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + optim_instance = make_optim_instance(instance) + + d = optim_instance$search_space$length + init_design_size = 4L * d + init_design = generate_design_lhs(optim_instance$search_space, n = init_design_size)$data + optim_instance$eval_batch(init_design) + + random_interleave_iter = 4L + + learner = lrn("regr.ranger", splitrule = "extratrees", num.random.splits = 1, min.node.size = 1, num.trees = 10, mtry.ratio = 1, replace = TRUE, sample.fraction = 1, se.method = "jack") + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% learner)) + + acq_function = AcqFunctionEI$new() + + optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) + acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20010L)) + + acq_optimizer$param_set$values$warmstart = TRUE + acq_optimizer$param_set$values$warmstart_size = "all" + + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + optim_instance +} + # add algorithms addAlgorithm("mlr3mbo", fun = mlr3mbo_wrapper) addAlgorithm("mlrintermbo", fun = mlrintermbo_wrapper) @@ -287,6 +318,7 @@ addAlgorithm("mlr3mbo_default", fun = mlr3mbo_default_wrapper) addAlgorithm("mlr3mbo_custom", fun = mlr3mbo_wrapper_custom) addAlgorithm("mlr3mbo_new_rf", fun = mlr3mbo_wrapper_new_rf) addAlgorithm("mlr3mbo_new_rf_ls", fun = mlr3mbo_wrapper_new_rf_ls) +addAlgorithm("mlr3mbo_xxx_ls", fun = mlr3mbo_wrapper_xxx_ls) # setup scenarios and instances get_nb301_setup = function(budget_factor = 40L) { @@ -358,7 +390,7 @@ prob_designs = unlist(prob_designs, recursive = FALSE, use.names = FALSE) names(prob_designs) = nn # add jobs for optimizers -optimizers = data.table(algorithm = c("mlr3mbo", "mlrintermbo", "mlr3mbo_default", "mlr3mbo_custom", "mlr3mbo_new_rf", "mlr3mbo_new_rf_ls")) +optimizers = data.table(algorithm = c("mlr3mbo", "mlrintermbo", "mlr3mbo_default", "mlr3mbo_custom", "mlr3mbo_new_rf", "mlr3mbo_new_rf_ls", "mlr3mbo_xxx_ls")) for (i in seq_len(nrow(optimizers))) { algo_designs = setNames(list(optimizers[i, ]), nm = optimizers[i, ]$algorithm) @@ -372,7 +404,7 @@ for (i in seq_len(nrow(optimizers))) { } jobs = findJobs() -resources.default = list(walltime = 3600 * 9L, memory = 2048L, ntasks = 1L, ncpus = 1L, nodes = 1L, clusters = "teton", max.concurrent.jobs = 9999L) +resources.default = list(walltime = 3600 * 12L, memory = 2048L, ntasks = 1L, ncpus = 1L, nodes = 1L, clusters = "teton", max.concurrent.jobs = 9999L) submitJobs(jobs, resources = resources.default) done = findDone() From 0fac55d7813804ae0ed861cdb8cfc658c0fba91b Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Tue, 6 Sep 2022 11:42:17 +0200 Subject: [PATCH 015/126] .. --- attic/so_config/analyze_yahpo.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/attic/so_config/analyze_yahpo.R b/attic/so_config/analyze_yahpo.R index 3ab050c6..8cdb39dd 100644 --- a/attic/so_config/analyze_yahpo.R +++ b/attic/so_config/analyze_yahpo.R @@ -4,7 +4,7 @@ library(pammtools) library(mlr3misc) #dat = rbind(readRDS("results_yahpo.rds"), readRDS("results_yahpo_own.rds"))[method != "mlrintermbo"] -dat = rbind(readRDS("results_yahpo.rds"), readRDS("results_yahpo_own.rds"))[method %in% c("mlr3mbo", "mlrintermbo", "mlr3mbo_new_rf", "mlr3mbo_new_rf_ls", "smac4hpo", "random")] +dat = rbind(readRDS("results_yahpo.rds"), readRDS("results_yahpo_own.rds"))[method %in% c("mlr3mbo", "mlrintermbo", "mlr3mbo_new_rf", "mlr3mbo_new_rf_ls", "mlr3mbo_xxx_ls", "smac4hpo", "random")] dat = dat[scenario %nin% c("nb301", "rbv2_super")] dat[, cumbudget := cumsum(budget), by = .(method, scenario, instance, repl)] dat[, cumbudget_scaled := cumbudget / max(cumbudget), by = .(method, scenario, instance, repl)] @@ -35,7 +35,7 @@ g = ggplot(aes(x = cumbudget_scaled, y = mean, colour = method, fill = method), labs(x = "Fraction of Budget Used", y = "Mean Normalized Regret", colour = "Optimizer", fill = "Optimizer") + facet_wrap(~ scenario + instance, scales = "free", ncol = 4) + theme_minimal() + - theme(legend.position = "bottom", legend.title = element_text(size = rel(0.75)), legend.text = element_text(size = rel(0.75))) + theme(legend.position = "bottom", legend.title = element_text(size = rel(0.75)), legend.text = element_text(size = rel(0.5))) ggsave("anytime.pdf", plot = g, device = "pdf", width = 12, height = 15) overall_budget = agg_budget[, .(mean = mean(mean), se = sd(mean) / sqrt(.N)), by = .(method, cumbudget_scaled)] From 93a438c766a2acbe2b26869f21bf562aadfea6ae Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Wed, 7 Sep 2022 14:59:31 +0200 Subject: [PATCH 016/126] .. --- R/AcqOptimizer.R | 12 ------------ R/helper.R | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/R/AcqOptimizer.R b/R/AcqOptimizer.R index 0b515f95..cb933182 100644 --- a/R/AcqOptimizer.R +++ b/R/AcqOptimizer.R @@ -133,15 +133,3 @@ AcqOptimizer = R6Class("AcqOptimizer", ) ) -# FIXME: document -get_best_not_evaluated = function(instance, evaluated) { - assert_r6(instance, classes = "OptimInstanceSingleCrit") - data = copy(instance$archive$data[, c(instance$archive$cols_x, instance$archive$cols_y), with = FALSE]) - evaluated = copy(evaluated) - data[, .overlap := FALSE][evaluated, .overlap := TRUE, on = instance$archive$cols_x] - candidates = data[.overlap == FALSE] - candidates[[instance$archive$cols_y]] = candidates[[instance$archive$cols_y]] * instance$objective_multiplicator[instance$archive$cols_y] - xdt = setorderv(candidates, cols = instance$archive$cols_y)[1L, ] - xdt[[instance$archive$cols_y]] = xdt[[instance$archive$cols_y]] * instance$objective_multiplicator[instance$archive$cols_y] - xdt -} diff --git a/R/helper.R b/R/helper.R index 71f474ea..ec81f6c9 100644 --- a/R/helper.R +++ b/R/helper.R @@ -182,3 +182,17 @@ surrogate_mult_max_to_min = function(codomain, y_cols) { mult_max_to_min = function(codomain) { ifelse(map_lgl(codomain$tags, has_element, "minimize"), 1, -1) } + +# FIXME: document +# used in AcqOptimizer +get_best_not_evaluated = function(instance, evaluated) { + assert_r6(instance, classes = "OptimInstanceSingleCrit") + data = copy(instance$archive$data[, c(instance$archive$cols_x, instance$archive$cols_y), with = FALSE]) + evaluated = copy(evaluated) + data[, .overlap := FALSE][evaluated, .overlap := TRUE, on = instance$archive$cols_x] + candidates = data[.overlap == FALSE] + candidates[[instance$archive$cols_y]] = candidates[[instance$archive$cols_y]] * instance$objective_multiplicator[instance$archive$cols_y] + xdt = setorderv(candidates, cols = instance$archive$cols_y)[1L, ] + xdt[[instance$archive$cols_y]] = xdt[[instance$archive$cols_y]] * instance$objective_multiplicator[instance$archive$cols_y] + xdt +} From 0bb4b988cb286816a30050bfbf94ddd110bac209 Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Thu, 8 Sep 2022 01:28:08 +0200 Subject: [PATCH 017/126] .. --- R/LearnerRegrRangerCustom.R | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/R/LearnerRegrRangerCustom.R b/R/LearnerRegrRangerCustom.R index 282bc8f6..4fc8e4fd 100644 --- a/R/LearnerRegrRangerCustom.R +++ b/R/LearnerRegrRangerCustom.R @@ -12,6 +12,7 @@ #' No bootstrap or subsampling is used but rather all training data (`replace = FALSE` and `sample.fraction = 1`). #' This may seem counter-intuitive at first but ensures that training data is almost perfectly interpolated (given `min.node.size = 1`). #' Per default, `1000` trees are used. +#' # FIXME: #' Classical standard error estimation is no longer possible, but the `"extratrees"` splitting rule allows for estimating the variance based on the variance of the trees (simply taking the variance over the predictions of the trees). #' This results in low variance close to training data points. #' If `se.simple.spatial = TRUE` this point-wise variance is upwardly biased by adding the pointwise variance again scaled by the minimum Gower distance of the point to the training data (recall that a Gower distance of 0 means an identical point is present in the training data, whereas a distance of 1 means total dissimilarity to the training data). @@ -55,22 +56,22 @@ LearnerRegrRangerCustom = R6Class("LearnerRegrRangerCustom", ps = ps( always.split.variables = p_uty(tags = "train"), holdout = p_lgl(default = FALSE, tags = "train"), - importance = p_fct(c("none", "impurity", "impurity_corrected", "permutation"), tags = "train"), + importance = p_fct(levels = c("none", "impurity", "impurity_corrected", "permutation"), tags = "train"), keep.inbag = p_lgl(default = FALSE, tags = "train"), local.importance = p_lgl(default = FALSE, tags = "train"), max.depth = p_int(default = NULL, lower = 0L, special_vals = list(NULL), tags = "train"), mtry = p_int(lower = 1L, special_vals = list(NULL), tags = "train"), mtry.ratio = p_dbl(lower = 0, upper = 1, tags = "train"), - num.threads = p_int(1L, default = 1L, tags = c("train", "predict", "threads")), # changed - num.trees = p_int(1L, default = 1000L, tags = c("train", "predict", "hotstart")), # changed + num.threads = p_int(lower = 1L, default = 1L, tags = c("train", "predict", "threads")), # changed + num.trees = p_int(lower = 1L, default = 1000L, tags = c("train", "predict", "hotstart")), # changed oob.error = p_lgl(default = FALSE, tags = "train"), # changed regularization.factor = p_uty(default = 1, tags = "train"), regularization.usedepth = p_lgl(default = FALSE, tags = "train"), - respect.unordered.factors = p_fct(c("ignore", "order", "partition"), default = "ignore", tags = "train"), + respect.unordered.factors = p_fct(levels = c("ignore", "order", "partition"), default = "ignore", tags = "train"), save.memory = p_lgl(default = FALSE, tags = "train"), scale.permutation.importance = p_lgl(default = FALSE, tags = "train"), se.simple.spatial = p_lgl(default = TRUE, tags = "predict"), - se.simple.spatial.nugget = p_dbl(0, default = 0, tags = "predict"), + se.simple.spatial.nugget = p_dbl(lower = 0, default = 0, tags = "predict"), seed = p_int(default = NULL, special_vals = list(NULL), tags = c("train", "predict")), split.select.weights = p_uty(default = NULL, tags = "train"), verbose = p_lgl(default = TRUE, tags = c("train", "predict")), @@ -136,21 +137,27 @@ LearnerRegrRangerCustom = R6Class("LearnerRegrRangerCustom", pv = self$param_set$get_values(tags = "predict") pv = insert_named(list(se.simple.spatial = TRUE, se.simple.spatial.nugget = 0), pv) newdata = ordered_features(task, self) - - prediction = invoke(predict, self$model, data = newdata, type = "response", predict.all = TRUE, .args = pv) + traindata = ordered_features(private$.train_task, self) + data = rbind(traindata, newdata) + n_traindata = nrow(traindata) + n_newdata = nrow(newdata) + traindata_ids = seq_len(n_traindata) + newdata_ids = (n_traindata + 1L):nrow(data) + + prediction = invoke(predict, self$model, data = data, type = "response", predict.all = TRUE, .args = pv) response = apply(prediction$predictions, MARGIN = 1L, FUN = mean) - variance = apply(prediction$predictions, MARGIN = 1L, FUN = var) + + variance = numeric(n_newdata) + for (i in newdata_ids) { + variance[i - n_traindata] = var(prediction$predictions[i, ]) + var(response[c(traindata_ids, i)]) + } if (pv$se.simple.spatial) { - gw_dists = get_gower_dist(fct_to_char(newdata), fct_to_char(ordered_features(private$.train_task, self))) # 0 if identical, 1 if maximally dissimilar + gw_dists = get_gower_dist(fct_to_char(newdata), fct_to_char(traindata)) # 0 if identical, 1 if maximally dissimilar min_gw_dists = apply(gw_dists, MARGIN = 1L, FUN = min) # get the minium for each new point to the points used for training - #min_gw_dists = (min_gw_dists - min(min_gw_dists)) / (max(min_gw_dists) - min(min_gw_dists)) # scale to [0, 1] - #min_gw_dists[is.na(min_gw_dists)] = 0 - # upwardly bias variance by the mean variance scaled by the gower distance (and the potential nugget, useful for noisy) - #variance = variance + (mean(variance) * (min_gw_dists + (pv$se.simple.spatial.nugget %??% 0))) # 0 is default - variance = (variance + pv$se.simple.spatial.nugget) + (variance + pv$se.simple.spatial.nugget) * min_gw_dists + variance = (variance * pv$se.simple.spatial.nugget) + (variance * min_gw_dists) } se = sqrt(variance) - list(response = response, se = if (self$predict_type == "se") se else NULL) + list(response = response[newdata_ids], se = if (self$predict_type == "se") se else NULL) }, .hotstart = function(task) { From 44ba83de9869e6c1cf3915637203a1bf92d9e592 Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Thu, 8 Sep 2022 01:37:12 +0200 Subject: [PATCH 018/126] .. --- attic/so_config/run_yahpo.R | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/attic/so_config/run_yahpo.R b/attic/so_config/run_yahpo.R index 1c8a1abf..4fcd191c 100644 --- a/attic/so_config/run_yahpo.R +++ b/attic/so_config/run_yahpo.R @@ -291,13 +291,13 @@ mlr3mbo_wrapper_xxx_ls = function(job, data, instance, ...) { d = optim_instance$search_space$length init_design_size = 4L * d - init_design = generate_design_lhs(optim_instance$search_space, n = init_design_size)$data + init_design = generate_design_sobol(optim_instance$search_space, n = init_design_size)$data optim_instance$eval_batch(init_design) random_interleave_iter = 4L - - learner = lrn("regr.ranger", splitrule = "extratrees", num.random.splits = 1, min.node.size = 1, num.trees = 10, mtry.ratio = 1, replace = TRUE, sample.fraction = 1, se.method = "jack") - surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% learner)) + + learner = lrn("regr.ranger_custom") + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor", multiplier = 2) %>>% learner)) acq_function = AcqFunctionEI$new() From 0f221a4737865c7461f12af5d54170cf115a50b4 Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Sat, 10 Sep 2022 00:56:01 +0200 Subject: [PATCH 019/126] .. --- R/AcqFunctionMean.R | 46 +++++++++ attic/so_config/analyze_glmnet_458.R | 140 +++++++++++++++++++++++++++ attic/so_config/analyze_yahpo.R | 11 ++- attic/so_config/run_yahpo.R | 35 +++++++ 4 files changed, 227 insertions(+), 5 deletions(-) create mode 100644 R/AcqFunctionMean.R create mode 100644 attic/so_config/analyze_glmnet_458.R diff --git a/R/AcqFunctionMean.R b/R/AcqFunctionMean.R new file mode 100644 index 00000000..753a3765 --- /dev/null +++ b/R/AcqFunctionMean.R @@ -0,0 +1,46 @@ +#' @title Acquisition Function Mean +#' +#' @include AcqFunction.R +#' @name mlr_acqfunctions_mean +#' +#' @templateVar id mean +#' @template section_dictionary_acqfunctions +#' +#' @description +#' Mean. +#' +#' @family Acquisition Function +#' @export +AcqFunctionMean = R6Class("AcqFunctionMean", + inherit = AcqFunction, + public = list( + + #' @field y_best (`numeric(1)`). + y_best = NULL, + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + #' + #' @param surrogate (`NULL` | [SurrogateLearner]). + initialize = function(surrogate = NULL) { + assert_r6(surrogate, "SurrogateLearner", null.ok = TRUE) + super$initialize("acq_mean", surrogate = surrogate, direction = "maximize") + }, + + #' @description + #' Updates acquisition function and sets `y_best`. + update = function() { + self$y_best = min(self$surrogate_max_to_min * self$archive$data[[self$surrogate$y_cols]]) + } + ), + + private = list( + .fun = function(xdt) { + p = self$surrogate$predict(xdt) + mu = p$mean + data.table(acq_mean = mu) + } + ) +) + +mlr_acqfunctions$add("mean", AcqFunctionMean) diff --git a/attic/so_config/analyze_glmnet_458.R b/attic/so_config/analyze_glmnet_458.R new file mode 100644 index 00000000..397792ec --- /dev/null +++ b/attic/so_config/analyze_glmnet_458.R @@ -0,0 +1,140 @@ +glmnet_458_smac = readRDS("rbv2_glmnet_458_smac.rds") +glmnet_458_smac[, method := "smac4hpo"] +glmnet_458_smac[, s := log(s)] + +glmnet_458_mlr3mbo = readRDS("rbv2_glmnet_458_mlr3mbo_new_rf.rds") +glmnet_458_mlr3mbo[, method := "mlr3mbo"] +glmnet_458 = rbind(glmnet_458_smac, glmnet_458_mlr3mbo) + +glmnet_458_best = map_dtr(1:30, function(r) { + rbind(glmnet_458[method == "smac4hpo" & repl == r, ][which.max(acc), ], + glmnet_458[method == "mlr3mbo" & repl == r, ][which.max(acc), ]) +}) + +reticulate::use_virtualenv("/home/lps/.local/share/virtualenvs/yahpo_gym-4ygV7ggv/", required = TRUE) +source("helpers.R") +library(yahpogym) +library(bbotk) +instance = make_optim_instance(data.table(scenario = "rbv2_glmnet", instance = "458", target = "acc", ndi = 3L, max_budget = 1, budget = 10^6, on_integer_scale = FALSE, minimize = FALSE)) +xdt = generate_design_grid(instance$search_space, resolution = 100)$data +instance$eval_batch(xdt) +ref = instance$archive$data +breaks = quantile(ref$acc, c(0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 0.99, 0.999, 1)) + +r = 2 + +g = ggplot(data = ref, aes(x = alpha, y = s, z = acc)) + + geom_point(aes(x = alpha, y = s, colour = iter, shape = method), data = glmnet_458[repl == r], size = 3) + + geom_contour(breaks = breaks) + + geom_contour_filled(breaks = breaks, alpha = 0.1) + + facet_grid(method ~ num.impute.selected.cpo) + + theme(legend.position = "bottom") + +instance_x = instance$clone(deep = TRUE) +instance$archive$data = glmnet_458[method == "mlr3mbo" & repl == 1] + +learner = lrn("regr.ranger_custom") +surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% learner)) + +acq_function = AcqFunctionEI$new() +surrogate$archive = instance$archive +acq_function$surrogate = surrogate +acq_function$surrogate$update() +acq_function$update() + +acq_instance = OptimInstanceSingleCrit$new(objective = acq_function, search_space = acq_function$domain, terminator = trm("none"), check_values = FALSE, keep_evals = "all") +acq_instance$eval_batch(xdt) + +acq_data = copy(acq_instance$archive$data) +acq_data = cbind(acq_data, sp) +breaks = quantile(acq_data$mean, c(0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 0.99, 0.999, 1)) + +acq_instance$archive$clear() + +acq_budget = 20000L + +acq_optimizer = { + n_repeats = 2L + maxit = 9L + batch_size = ceiling((acq_budget / n_repeats) / (1 + maxit)) + AcqOptimizer$new(opt("focus_search", n_points = batch_size, maxit = maxit), terminator = trm("evals", n_evals = acq_budget)) +} + +acq_optimizer$acq_function = acq_function +candidate = acq_instance$archive$best() + +g3 = ggplot(data = acq_data, aes(x = alpha, y = s, z = mean)) + + geom_contour(breaks = breaks) + + geom_contour_filled(breaks = breaks) + + facet_wrap(~ num.impute.selected.cpo) + + theme(legend.position = "bottom") + +saveRDS(instance$archive$data[, c(instance$archive$cols_x, instance$archive$cols_y), with = FALSE], "glmnet_458_mbo_data.rds") + +library(mlr3) +library(mlr3learners) +library(mlr3pipelines) +library(mlr3viz) + + +tdat = readRDS("glmnet_458_mbo_data.rds") + +tdat[, num.impute.selected.cpo := as.factor(num.impute.selected.cpo)] +task = TaskRegr$new("test", backend = tdat, target = "acc") +resampling = rsmp("repeated_cv", folds = 10L)$instantiate(task) +gp = as_learner(po("encode") %>>% po("fixfactors") %>>% lrn("regr.km", covtype = "matern3_2", optim.method = "gen")) +gp2 = as_learner(po("encodeimpact") %>>% po("fixfactors") %>>% lrn("regr.km", covtype = "matern3_2", optim.method = "gen")) +rp = as_learner(po("fixfactors") %>>% lrn("regr.rpart")) +rf = as_learner(po("fixfactors") %>>% lrn("regr.ranger")) +lm = as_learner(po("fixfactors") %>>% lrn("regr.lm")) +kknn = as_learner(po("fixfactors") %>>% lrn("regr.kknn")) +kknn1 = as_learner(po("fixfactors") %>>% lrn("regr.kknn", k = 1)) +kknn1$id = "kknn1" +mars = as_learner(po("encode") %>>% po("fixfactors") %>>% lrn("regr.mars")) +gam = as_learner(po("encode") %>>% po("fixfactors") %>>% lrn("regr.gam")) +lm = as_learner(po("fixfactors") %>>% lrn("regr.lm")) +learners = list(learner, gp, gp2, rp, rf, kknn, mars, gam, lm) +bg = benchmark_grid(task, learners, resampling) +b = benchmark(bg) +scores = b$aggregate(msrs(c("regr.mse", "regr.mae", "regr.rsq", "regr.ktau"))) +scores_long = melt(scores, id.vars = "learner_id", measure.vars = c("regr.mse", "regr.mae", "regr.rsq", "regr.ktau")) + +ggplot(aes(x = learner_id, y = value), data = scores_long) + + geom_point(size = 3) + + facet_wrap(~ variable, scales = "free") + + theme_minimal() + + theme(axis.text.x = element_text(angle = 45, vjust = 0.5, hjust = 1)) + +breaks = quantile(task$data()$acc, c(seq(0, 1, by = 0.1))) +weights = as.numeric(cut(tdat$acc, breaks)) +weights[is.na(weights)] = 1 +task2 = TaskRegr$new("test", backend = cbind(tdat, weights), target = "acc") +task2$col_roles$weight = "weights" +task2$col_roles$feature = setdiff(task2$col_roles$feature, "weights") +learners = list(rf, kknn, kknn1, lm, gp) + +map(learners, function(l) { + l$train(task2) +}) + +diffs = map_dtr(1:100, function(r) { + print(r) + map_dtr(c(2^(5:13)), function(k) { + xdt_sample = ref[sample(.N, size = k, replace = FALSE), ] + best = xdt_sample[which.max(acc), ] + preds = map_dbl(learners, function(l) { + which.max(l$predict_newdata(xdt_sample)$response) + }) + xdt_learner_p_best = xdt_sample[preds, ] + data.table(diff = best$acc - xdt_learner_p_best$acc, k = k, learner = map_chr(learners, "id"), repl = r) + }) +}) + +mean_diffs = diffs[, .(mean_diff = mean(diff), se_diff = sd(diff) / sqrt(.N)), by = .(k, learner)] + +ggplot(aes(x = k, y = mean_diff, colour = learner), data = mean_diffs) + + geom_point() + + geom_errorbar(aes(ymin = mean_diff - se_diff, ymax = mean_diff + se_diff)) + + geom_line() + + theme_minimal() + diff --git a/attic/so_config/analyze_yahpo.R b/attic/so_config/analyze_yahpo.R index 8cdb39dd..a4f7bbef 100644 --- a/attic/so_config/analyze_yahpo.R +++ b/attic/so_config/analyze_yahpo.R @@ -4,7 +4,7 @@ library(pammtools) library(mlr3misc) #dat = rbind(readRDS("results_yahpo.rds"), readRDS("results_yahpo_own.rds"))[method != "mlrintermbo"] -dat = rbind(readRDS("results_yahpo.rds"), readRDS("results_yahpo_own.rds"))[method %in% c("mlr3mbo", "mlrintermbo", "mlr3mbo_new_rf", "mlr3mbo_new_rf_ls", "mlr3mbo_xxx_ls", "smac4hpo", "random")] +dat = rbind(readRDS("results_yahpo.rds"), readRDS("results_yahpo_own.rds"))[method %in% c("mlr3mbo", "mlrintermbo", "mlr3mbo_new_rf", "mlr3mbo_new_rf_ls", "smac4hpo", "random")] dat = dat[scenario %nin% c("nb301", "rbv2_super")] dat[, cumbudget := cumsum(budget), by = .(method, scenario, instance, repl)] dat[, cumbudget_scaled := cumbudget / max(cumbudget), by = .(method, scenario, instance, repl)] @@ -33,16 +33,17 @@ g = ggplot(aes(x = cumbudget_scaled, y = mean, colour = method, fill = method), geom_step() + geom_stepribbon(aes(min = mean - se, max = mean + se), colour = NA, alpha = 0.3) + labs(x = "Fraction of Budget Used", y = "Mean Normalized Regret", colour = "Optimizer", fill = "Optimizer") + - facet_wrap(~ scenario + instance, scales = "free", ncol = 4) + + facet_wrap(~ scenario + instance, scales = "free", ncol = 5) + theme_minimal() + theme(legend.position = "bottom", legend.title = element_text(size = rel(0.75)), legend.text = element_text(size = rel(0.5))) -ggsave("anytime.pdf", plot = g, device = "pdf", width = 12, height = 15) +ggsave("anytime.pdf", plot = g, device = "pdf", width = 15, height = 10) overall_budget = agg_budget[, .(mean = mean(mean), se = sd(mean) / sqrt(.N)), by = .(method, cumbudget_scaled)] g = ggplot(aes(x = cumbudget_scaled, y = mean, colour = method, fill = method), data = overall_budget[cumbudget_scaled > 0.10]) + scale_y_log10() + geom_step() + + geom_stepribbon(aes(min = mean - se, max = mean + se), colour = NA, alpha = 0.1) + labs(x = "Fraction of Budget Used", y = "Mean Normalized Regret", colour = "Optimizer", fill = "Optimizer") + theme_minimal() + theme(legend.position = "bottom", legend.title = element_text(size = rel(0.75)), legend.text = element_text(size = rel(0.75))) @@ -77,7 +78,7 @@ best_agg = agg_budget[cumbudget_scaled == 0.25] best_agg[, problem := paste0(scenario, "_", instance)] tmp = - as.matrix(dcast(best_agg, problem ~ method, value.var = "mean")[, -1]) friedmanTest(tmp) -pdf("plots/cd_025_mf.pdf", width = 6, height = 4, pointsize = 10) +pdf("cd_025_mf.pdf", width = 6, height = 4, pointsize = 10) plotCD(tmp, cex = 1) dev.off() @@ -85,7 +86,7 @@ best_agg = agg_budget[cumbudget_scaled == 1] best_agg[, problem := paste0(scenario, "_", instance)] tmp = - as.matrix(dcast(best_agg, problem ~ method, value.var = "mean")[, -1]) friedmanTest(tmp) -pdf("plots/cd_1_mf.pdf", width = 6, height = 4, pointsize = 10) +pdf("cd_1_mf.pdf", width = 6, height = 4, pointsize = 10) plotCD(tmp, cex = 1) dev.off() diff --git a/attic/so_config/run_yahpo.R b/attic/so_config/run_yahpo.R index 4fcd191c..ebf7d30a 100644 --- a/attic/so_config/run_yahpo.R +++ b/attic/so_config/run_yahpo.R @@ -311,6 +311,40 @@ mlr3mbo_wrapper_xxx_ls = function(job, data, instance, ...) { optim_instance } +mlr3mbo_wrapper_kknn_ls = function(job, data, instance, ...) { + reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + optim_instance = make_optim_instance(instance) + + d = optim_instance$search_space$length + init_design_size = 4L * d + init_design = generate_design_sobol(optim_instance$search_space, n = init_design_size)$data + optim_instance$eval_batch(init_design) + + random_interleave_iter = 4L + + learner = lrn("regr.kknn") + learner = as_learner(po("imputeoor", multiplier = 2) %>>% po("fixfactors") %>>% po("imputesample") %>>% learner) + learner$predict_types = "response" + surrogate = SurrogateLearner$new(learner) + + acq_function = AcqFunctionMean$new() + + optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) + acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20010L)) + + acq_optimizer$param_set$values$logging_level = "info" + acq_optimizer$param_set$values$warmstart = TRUE + acq_optimizer$param_set$values$warmstart_size = "all" + + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + optim_instance +} + # add algorithms addAlgorithm("mlr3mbo", fun = mlr3mbo_wrapper) addAlgorithm("mlrintermbo", fun = mlrintermbo_wrapper) @@ -319,6 +353,7 @@ addAlgorithm("mlr3mbo_custom", fun = mlr3mbo_wrapper_custom) addAlgorithm("mlr3mbo_new_rf", fun = mlr3mbo_wrapper_new_rf) addAlgorithm("mlr3mbo_new_rf_ls", fun = mlr3mbo_wrapper_new_rf_ls) addAlgorithm("mlr3mbo_xxx_ls", fun = mlr3mbo_wrapper_xxx_ls) +addAlgorithm("mlr3mbo_kknn_ls", fun = mlr3mbo_wrapper_kknn_ls) # setup scenarios and instances get_nb301_setup = function(budget_factor = 40L) { From 6e1b09376d3ff680ad323e8518cc9be86766705e Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Sat, 10 Sep 2022 00:57:56 +0200 Subject: [PATCH 020/126] .. --- DESCRIPTION | 1 + NAMESPACE | 1 + man/AcqFunction.Rd | 1 + man/mlr_acqfunctions_aei.Rd | 1 + man/mlr_acqfunctions_cb.Rd | 1 + man/mlr_acqfunctions_ei.Rd | 1 + man/mlr_acqfunctions_eips.Rd | 1 + man/mlr_acqfunctions_mean.Rd | 104 +++++++++++++++++++++++++ man/mlr_acqfunctions_pi.Rd | 1 + man/mlr_acqfunctions_sms_ego.Rd | 1 + man/mlr_learners_regr.ranger_custom.Rd | 3 + 11 files changed, 116 insertions(+) create mode 100644 man/mlr_acqfunctions_mean.Rd diff --git a/DESCRIPTION b/DESCRIPTION index 39ad1740..327e9228 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -58,6 +58,7 @@ Collate: 'AcqFunctionCB.R' 'AcqFunctionEI.R' 'AcqFunctionEIPS.R' + 'AcqFunctionMean.R' 'AcqFunctionPI.R' 'AcqFunctionSmsEgo.R' 'AcqOptimizer.R' diff --git a/NAMESPACE b/NAMESPACE index ee0aa93f..3e5636ca 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -6,6 +6,7 @@ export(AcqFunctionAEI) export(AcqFunctionCB) export(AcqFunctionEI) export(AcqFunctionEIPS) +export(AcqFunctionMean) export(AcqFunctionPI) export(AcqFunctionSmsEgo) export(AcqOptimizer) diff --git a/man/AcqFunction.Rd b/man/AcqFunction.Rd index 5b9e55fd..e1e39e9e 100644 --- a/man/AcqFunction.Rd +++ b/man/AcqFunction.Rd @@ -13,6 +13,7 @@ Other Acquisition Function: \code{\link{mlr_acqfunctions_cb}}, \code{\link{mlr_acqfunctions_eips}}, \code{\link{mlr_acqfunctions_ei}}, +\code{\link{mlr_acqfunctions_mean}}, \code{\link{mlr_acqfunctions_pi}}, \code{\link{mlr_acqfunctions_sms_ego}} } diff --git a/man/mlr_acqfunctions_aei.Rd b/man/mlr_acqfunctions_aei.Rd index 7efe5b93..84f9de26 100644 --- a/man/mlr_acqfunctions_aei.Rd +++ b/man/mlr_acqfunctions_aei.Rd @@ -36,6 +36,7 @@ Other Acquisition Function: \code{\link{mlr_acqfunctions_cb}}, \code{\link{mlr_acqfunctions_eips}}, \code{\link{mlr_acqfunctions_ei}}, +\code{\link{mlr_acqfunctions_mean}}, \code{\link{mlr_acqfunctions_pi}}, \code{\link{mlr_acqfunctions_sms_ego}} } diff --git a/man/mlr_acqfunctions_cb.Rd b/man/mlr_acqfunctions_cb.Rd index 68d8806f..628ff0b5 100644 --- a/man/mlr_acqfunctions_cb.Rd +++ b/man/mlr_acqfunctions_cb.Rd @@ -37,6 +37,7 @@ Other Acquisition Function: \code{\link{mlr_acqfunctions_aei}}, \code{\link{mlr_acqfunctions_eips}}, \code{\link{mlr_acqfunctions_ei}}, +\code{\link{mlr_acqfunctions_mean}}, \code{\link{mlr_acqfunctions_pi}}, \code{\link{mlr_acqfunctions_sms_ego}} } diff --git a/man/mlr_acqfunctions_ei.Rd b/man/mlr_acqfunctions_ei.Rd index f4f22b3e..143217ac 100644 --- a/man/mlr_acqfunctions_ei.Rd +++ b/man/mlr_acqfunctions_ei.Rd @@ -28,6 +28,7 @@ Other Acquisition Function: \code{\link{mlr_acqfunctions_aei}}, \code{\link{mlr_acqfunctions_cb}}, \code{\link{mlr_acqfunctions_eips}}, +\code{\link{mlr_acqfunctions_mean}}, \code{\link{mlr_acqfunctions_pi}}, \code{\link{mlr_acqfunctions_sms_ego}} } diff --git a/man/mlr_acqfunctions_eips.Rd b/man/mlr_acqfunctions_eips.Rd index 12cd3bf9..26fc444f 100644 --- a/man/mlr_acqfunctions_eips.Rd +++ b/man/mlr_acqfunctions_eips.Rd @@ -35,6 +35,7 @@ Other Acquisition Function: \code{\link{mlr_acqfunctions_aei}}, \code{\link{mlr_acqfunctions_cb}}, \code{\link{mlr_acqfunctions_ei}}, +\code{\link{mlr_acqfunctions_mean}}, \code{\link{mlr_acqfunctions_pi}}, \code{\link{mlr_acqfunctions_sms_ego}} } diff --git a/man/mlr_acqfunctions_mean.Rd b/man/mlr_acqfunctions_mean.Rd new file mode 100644 index 00000000..ef5926ec --- /dev/null +++ b/man/mlr_acqfunctions_mean.Rd @@ -0,0 +1,104 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/AcqFunctionMean.R +\name{mlr_acqfunctions_mean} +\alias{mlr_acqfunctions_mean} +\alias{AcqFunctionMean} +\title{Acquisition Function Mean} +\description{ +Mean. +} +\section{Dictionary}{ + +This \link{AcqFunction} can be instantiated via the \link[mlr3misc:Dictionary]{dictionary} +\link{mlr_acqfunctions} or with the associated sugar function \code{\link[=acqf]{acqf()}}: + +\if{html}{\out{
}}\preformatted{mlr_acqfunctions$get("mean") +acqf("mean") +}\if{html}{\out{
}} +} + +\seealso{ +Other Acquisition Function: +\code{\link{AcqFunction}}, +\code{\link{mlr_acqfunctions_aei}}, +\code{\link{mlr_acqfunctions_cb}}, +\code{\link{mlr_acqfunctions_eips}}, +\code{\link{mlr_acqfunctions_ei}}, +\code{\link{mlr_acqfunctions_pi}}, +\code{\link{mlr_acqfunctions_sms_ego}} +} +\concept{Acquisition Function} +\section{Super classes}{ +\code{\link[bbotk:Objective]{bbotk::Objective}} -> \code{\link[mlr3mbo:AcqFunction]{mlr3mbo::AcqFunction}} -> \code{AcqFunctionMean} +} +\section{Public fields}{ +\if{html}{\out{
}} +\describe{ +\item{\code{y_best}}{(\code{numeric(1)}).} +} +\if{html}{\out{
}} +} +\section{Methods}{ +\subsection{Public methods}{ +\itemize{ +\item \href{#method-AcqFunctionMean-new}{\code{AcqFunctionMean$new()}} +\item \href{#method-AcqFunctionMean-update}{\code{AcqFunctionMean$update()}} +\item \href{#method-AcqFunctionMean-clone}{\code{AcqFunctionMean$clone()}} +} +} +\if{html}{\out{ +
Inherited methods + +
+}} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqFunctionMean-new}{}}} +\subsection{Method \code{new()}}{ +Creates a new instance of this \link[R6:R6Class]{R6} class. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{AcqFunctionMean$new(surrogate = NULL)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{surrogate}}{(\code{NULL} | \link{SurrogateLearner}).} +} +\if{html}{\out{
}} +} +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqFunctionMean-update}{}}} +\subsection{Method \code{update()}}{ +Updates acquisition function and sets \code{y_best}. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{AcqFunctionMean$update()}\if{html}{\out{
}} +} + +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqFunctionMean-clone}{}}} +\subsection{Method \code{clone()}}{ +The objects of this class are cloneable with this method. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{AcqFunctionMean$clone(deep = FALSE)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{deep}}{Whether to make a deep clone.} +} +\if{html}{\out{
}} +} +} +} diff --git a/man/mlr_acqfunctions_pi.Rd b/man/mlr_acqfunctions_pi.Rd index 95d9f87e..3e425a05 100644 --- a/man/mlr_acqfunctions_pi.Rd +++ b/man/mlr_acqfunctions_pi.Rd @@ -29,6 +29,7 @@ Other Acquisition Function: \code{\link{mlr_acqfunctions_cb}}, \code{\link{mlr_acqfunctions_eips}}, \code{\link{mlr_acqfunctions_ei}}, +\code{\link{mlr_acqfunctions_mean}}, \code{\link{mlr_acqfunctions_sms_ego}} } \concept{Acquisition Function} diff --git a/man/mlr_acqfunctions_sms_ego.Rd b/man/mlr_acqfunctions_sms_ego.Rd index 3fc68404..3e19f1c9 100644 --- a/man/mlr_acqfunctions_sms_ego.Rd +++ b/man/mlr_acqfunctions_sms_ego.Rd @@ -38,6 +38,7 @@ Other Acquisition Function: \code{\link{mlr_acqfunctions_cb}}, \code{\link{mlr_acqfunctions_eips}}, \code{\link{mlr_acqfunctions_ei}}, +\code{\link{mlr_acqfunctions_mean}}, \code{\link{mlr_acqfunctions_pi}} } \concept{Acquisition Function} diff --git a/man/mlr_learners_regr.ranger_custom.Rd b/man/mlr_learners_regr.ranger_custom.Rd index 5c674f8e..0e265ac8 100644 --- a/man/mlr_learners_regr.ranger_custom.Rd +++ b/man/mlr_learners_regr.ranger_custom.Rd @@ -14,6 +14,8 @@ This results in a rather smooth prediction function. No bootstrap or subsampling is used but rather all training data (\code{replace = FALSE} and \code{sample.fraction = 1}). This may seem counter-intuitive at first but ensures that training data is almost perfectly interpolated (given \code{min.node.size = 1}). Per default, \code{1000} trees are used. +} +\section{FIXME:}{ Classical standard error estimation is no longer possible, but the \code{"extratrees"} splitting rule allows for estimating the variance based on the variance of the trees (simply taking the variance over the predictions of the trees). This results in low variance close to training data points. If \code{se.simple.spatial = TRUE} this point-wise variance is upwardly biased by adding the pointwise variance again scaled by the minimum Gower distance of the point to the training data (recall that a Gower distance of 0 means an identical point is present in the training data, whereas a distance of 1 means total dissimilarity to the training data). @@ -22,6 +24,7 @@ Summarizing, the standard error for a point is calculated as follows: \deqn{\hat{\mathrm{SE}}(x_{i}) = \sqrt{(\hat{\mathrm{VAR}}(x_{i}) + \epsilon) + (\hat{\mathrm{VAR}}(x_{i}) + \epsilon) * \mathrm{GD}_{i})}} Here, \eqn{\epsilon} is given by \code{se.simple.spatial.nugget} which can be set larger than 0 in the case of noisy function observations. } + \section{Dictionary}{ This \link{Learner} can be instantiated via the \link[mlr3misc:Dictionary]{dictionary} \link{mlr_learners} or with the associated sugar function \code{\link[=lrn]{lrn()}}: From 8ca00934da965252b758d3f6ab6f847e1bc09d8a Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Sat, 10 Sep 2022 01:08:26 +0200 Subject: [PATCH 021/126] .. --- attic/so_config/run_yahpo.R | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/attic/so_config/run_yahpo.R b/attic/so_config/run_yahpo.R index ebf7d30a..d40a3217 100644 --- a/attic/so_config/run_yahpo.R +++ b/attic/so_config/run_yahpo.R @@ -337,7 +337,6 @@ mlr3mbo_wrapper_kknn_ls = function(job, data, instance, ...) { optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20010L)) - acq_optimizer$param_set$values$logging_level = "info" acq_optimizer$param_set$values$warmstart = TRUE acq_optimizer$param_set$values$warmstart_size = "all" @@ -425,7 +424,7 @@ prob_designs = unlist(prob_designs, recursive = FALSE, use.names = FALSE) names(prob_designs) = nn # add jobs for optimizers -optimizers = data.table(algorithm = c("mlr3mbo", "mlrintermbo", "mlr3mbo_default", "mlr3mbo_custom", "mlr3mbo_new_rf", "mlr3mbo_new_rf_ls", "mlr3mbo_xxx_ls")) +optimizers = data.table(algorithm = c("mlr3mbo", "mlrintermbo", "mlr3mbo_default", "mlr3mbo_custom", "mlr3mbo_new_rf", "mlr3mbo_new_rf_ls", "mlr3mbo_xxx_ls", "mlr3mbo_kknn_ls")) for (i in seq_len(nrow(optimizers))) { algo_designs = setNames(list(optimizers[i, ]), nm = optimizers[i, ]$algorithm) From 9cfb3e10cfdfbcfaa70d156b5cdbe1236e53090d Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Tue, 4 Oct 2022 21:38:33 +0200 Subject: [PATCH 022/126] play around with log scaling --- R/AcqFunctionMean.R | 13 +- R/bayesopt_ego_log.R | 142 +++++++++++++++ attic/so_config/analyze_glmnet_458.R | 260 +++++++++++++++++---------- attic/so_config/analyze_yahpo.R | 4 +- 4 files changed, 308 insertions(+), 111 deletions(-) create mode 100644 R/bayesopt_ego_log.R diff --git a/R/AcqFunctionMean.R b/R/AcqFunctionMean.R index 753a3765..247f5954 100644 --- a/R/AcqFunctionMean.R +++ b/R/AcqFunctionMean.R @@ -15,29 +15,20 @@ AcqFunctionMean = R6Class("AcqFunctionMean", inherit = AcqFunction, public = list( - #' @field y_best (`numeric(1)`). - y_best = NULL, - #' @description #' Creates a new instance of this [R6][R6::R6Class] class. #' #' @param surrogate (`NULL` | [SurrogateLearner]). initialize = function(surrogate = NULL) { assert_r6(surrogate, "SurrogateLearner", null.ok = TRUE) - super$initialize("acq_mean", surrogate = surrogate, direction = "maximize") - }, - - #' @description - #' Updates acquisition function and sets `y_best`. - update = function() { - self$y_best = min(self$surrogate_max_to_min * self$archive$data[[self$surrogate$y_cols]]) + super$initialize("acq_mean", surrogate = surrogate, direction = "minimize") } ), private = list( .fun = function(xdt) { p = self$surrogate$predict(xdt) - mu = p$mean + mu = self$surrogate_max_to_min * p$mean data.table(acq_mean = mu) } ) diff --git a/R/bayesopt_ego_log.R b/R/bayesopt_ego_log.R new file mode 100644 index 00000000..49c05eab --- /dev/null +++ b/R/bayesopt_ego_log.R @@ -0,0 +1,142 @@ +#' @title Sequential Singlecriteria Bayesian Optimization +#' +#' @description +#' FIXME: document +#' MBO loop function for sequential singlecriteria Bayesian optimization. +#' Normally used inside an [OptimizerMbo]. +#' +#' @param instance ([bbotk::OptimInstanceSingleCrit])\cr +#' The [bbotk::OptimInstanceSingleCrit] to be optimized. +#' @param init_design_size (`NULL` | `integer(1)`)\cr +#' Size of the initial design. +#' If `NULL` \code{4 * d} is used with \code{d} being the dimensionality of the search space. +#' Points are drawn uniformly at random. +#' @param surrogate (`NULL` | [Surrogate])\cr +#' [Surrogate] to be used as a surrogate. +#' Typically a [SurrogateLearner]. +#' If `NULL` \code{default_surrogate(instance)} is used. +#' @param acq_function (`NULL` | [AcqFunction]).\cr +#' [AcqFunction] to be used as acquisition function. +#' If `NULL` \code{default_acqfun(instance)} is used. +#' @param acq_optimizer ([AcqOptimizer])\cr +#' [AcqOptimizer] to be used as acquisition function optimizer. +#' If `NULL` \code{default_acqopt(acqfun)} is used. +#' @param random_interleave_iter (`integer(1)`)\cr +#' Every "random_interleave_iter" iteration (starting after the initial design), a point is +#' sampled uniformly at random and evaluated (instead of a model based proposal). +#' For example, if `random_interleave_iter = 2`, random interleaving is performed in the second, +#' fourth, sixth, ... iteration. +#' Default is `0`, i.e., no random interleaving is performed at all. +#' +#' @note +#' * If `surrogate` is `NULL` but `acq_function` is given and contains a `$surrogate`, this +#' [Surrogate] is used. +#' * You can pass a `surrogate` that was not given the [bbotk::Archive] of the +#' `instance` during initialization. +#' In this case, the [bbotk::Archive] of the given `instance` is set during execution. +#' * Similarly, you can pass an `acq_function` that was not given the `surrogate` during initialization +#' and an `acq_optimizer` that was not given the `acq_function`, i.e., delayed initialization is +#' handled automatically. +#' +#' @return invisible(instance)\cr +#' The original instance is modified in-place and returned invisible. +#' +#' @references +#' * `r format_bib("jones_1998")` +#' * `r format_bib("snoek_2012")` +#' @family Loop Function +#' @export +#' @examples +#' library(bbotk) +#' library(paradox) +#' library(mlr3learners) +#' +#' # expected improvement +#' objective = ObjectiveRFun$new( +#' fun = function(xs) list(y = xs$x ^ 2), +#' domain = ps(x = p_dbl(lower = -5, upper = 5)), +#' codomain = ps(y = p_dbl(tags = "minimize")) +#' ) +#' +#' terminator = trm("evals", n_evals = 5) +#' +#' instance = OptimInstanceSingleCrit$new( +#' objective = objective, +#' terminator = terminator +#' ) +#' +#' bayesopt_ego_log(instance) +bayesopt_ego_log = function( + instance, + init_design_size = NULL, + surrogate = NULL, + acq_function = NULL, + acq_optimizer = NULL, + random_interleave_iter = 0L, + epsilon = 1e-3 + ) { + + # assertions and defaults + assert_r6(instance, "OptimInstanceSingleCrit") + assert_int(init_design_size, lower = 1L, null.ok = TRUE) + assert_r6(surrogate, classes = "SurrogateLearner", null.ok = TRUE) + assert_r6(acq_function, classes = "AcqFunction", null.ok = TRUE) + assert_r6(acq_optimizer, classes = "AcqOptimizer", null.ok = TRUE) + assert_int(random_interleave_iter, lower = 0L) + assert_number(epsilon, lower = 0, upper = 1) + + surrogate = surrogate %??% acq_function$surrogate + + archive = instance$archive + domain = instance$search_space + d = domain$length + if (is.null(init_design_size) && instance$archive$n_evals == 0L) init_design_size = 4L * d + if (is.null(surrogate)) surrogate = default_surrogate(instance) + if (is.null(acq_function)) acq_function = AcqFunctionEI$new() + if (is.null(acq_optimizer)) acq_optimizer = default_acqopt(acq_function) + surrogate$archive = archive + acq_function$surrogate = surrogate + acq_optimizer$acq_function = acq_function + + surrogate$y_cols = "y_trafo" + acq_function$surrogate_max_to_min = 1 + + # initial design + if (isTRUE(init_design_size > 0L)) { + design = generate_design_random(domain, n = init_design_size)$data + instance$eval_batch(design) + } else { + init_design_size = instance$archive$n_evals + } + + # loop + repeat { + + y = instance$archive$data[[instance$archive$cols_y]] * instance$objective_multiplicator + min_y = min(y) - epsilon * diff(range(y)) + max_y = max(y) + y_log = log((y - min_y) / (max_y - min_y)) + instance$archive$data[, y_trafo := y_log] + + xdt = tryCatch({ + # random interleaving is handled here + if (isTRUE((instance$archive$n_evals - init_design_size + 1L) %% random_interleave_iter == 0)) { + stop(set_class(list(message = "Random interleaving", call = NULL), classes = c("mbo_error", "random_interleave", "error", "condition"))) + } + acq_function$surrogate$update() + if (test_r6(acq_function, "AcqFunctionEI")) { + acq_function$y_best = min(y_log) # manual update + } + acq_optimizer$optimize() + }, mbo_error = function(mbo_error_condition) { + lg$info("Proposing a randomly sampled point") + SamplerUnif$new(domain)$sample(1L)$data + }) + + instance$eval_batch(xdt) + if (instance$is_terminated) break + } + + return(invisible(instance)) +} + diff --git a/attic/so_config/analyze_glmnet_458.R b/attic/so_config/analyze_glmnet_458.R index 397792ec..008e6bf4 100644 --- a/attic/so_config/analyze_glmnet_458.R +++ b/attic/so_config/analyze_glmnet_458.R @@ -1,140 +1,204 @@ -glmnet_458_smac = readRDS("rbv2_glmnet_458_smac.rds") -glmnet_458_smac[, method := "smac4hpo"] -glmnet_458_smac[, s := log(s)] - -glmnet_458_mlr3mbo = readRDS("rbv2_glmnet_458_mlr3mbo_new_rf.rds") -glmnet_458_mlr3mbo[, method := "mlr3mbo"] -glmnet_458 = rbind(glmnet_458_smac, glmnet_458_mlr3mbo) +library(mlr3) +library(mlr3learners) +library(mlr3pipelines) +library(mlr3mbo) -glmnet_458_best = map_dtr(1:30, function(r) { - rbind(glmnet_458[method == "smac4hpo" & repl == r, ][which.max(acc), ], - glmnet_458[method == "mlr3mbo" & repl == r, ][which.max(acc), ]) -}) +set.seed(1) reticulate::use_virtualenv("/home/lps/.local/share/virtualenvs/yahpo_gym-4ygV7ggv/", required = TRUE) source("helpers.R") +source("OptimizerChain.R") library(yahpogym) library(bbotk) instance = make_optim_instance(data.table(scenario = "rbv2_glmnet", instance = "458", target = "acc", ndi = 3L, max_budget = 1, budget = 10^6, on_integer_scale = FALSE, minimize = FALSE)) xdt = generate_design_grid(instance$search_space, resolution = 100)$data instance$eval_batch(xdt) -ref = instance$archive$data +ref = copy(instance$archive$data) breaks = quantile(ref$acc, c(0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 0.99, 0.999, 1)) -r = 2 +instance = make_optim_instance(data.table(scenario = "rbv2_glmnet", instance = "458", target = "acc", ndi = 3L, max_budget = 1, budget = 90L, on_integer_scale = FALSE, minimize = FALSE)) +d = instance$search_space$length +init_design_size = 4L * d +init_design = generate_design_lhs(instance$search_space, n = init_design_size)$data +instance$eval_batch(init_design) + +random_interleave_iter = 0L + +learner = lrn("regr.ranger_custom", num.trees = 10L, mtry.ratio = 1) +surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor", multiplier = 10) %>>% learner)) + +acq_function = AcqFunctionEI$new() -g = ggplot(data = ref, aes(x = alpha, y = s, z = acc)) + - geom_point(aes(x = alpha, y = s, colour = iter, shape = method), data = glmnet_458[repl == r], size = 3) + +optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) +acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20010L)) + +acq_optimizer$param_set$values$warmstart = TRUE +acq_optimizer$param_set$values$warmstart_size = "all" + +bayesopt_ego(instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + +mbo = copy(instance$archive$data)[, c("alpha", "num.impute.selected.cpo", "s", "acc")] +mbo[, method := "mbo"] +mbo[, repl := 1] +mbo[, iter := seq_len(.N)] + +instance$archive$clear() +instance$eval_batch(init_design) + +bayesopt_ego_log(instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + +mbo_log = copy(instance$archive$data)[, c("alpha", "num.impute.selected.cpo", "s", "acc")] +mbo_log[, method := "mbo_log"] +mbo_log[, repl := 1] +mbo_log[, iter := seq_len(.N)] + +glmnet_458_smac = readRDS("rbv2_glmnet_458_smac.rds")[repl == 2] +glmnet_458_smac[, method := "smac4hpo"] +glmnet_458_smac[, s := log(s)] + +glmnet_458_mbo = readRDS("rbv2_glmnet_458_mlr3mbo_new_rf.rds")[repl == 1] +glmnet_458_mbo[, method := "mlr3mbo_new_rf"] + +glmnet_458 = rbind(glmnet_458_smac, glmnet_458_mbo, mbo, mbo_log, fill = TRUE) + +glmnet_458_best = rbind(glmnet_458[method == "smac4hpo"][which.max(acc), ], + glmnet_458[method == "mlr3mbo_new_rf"][which.max(acc), ], + glmnet_458[method == "mbo"][which.max(acc), ], + glmnet_458[method == "mbo_log"][which.max(acc), ]) + +g1 = ggplot(data = ref, aes(x = alpha, y = s, z = acc)) + + geom_point(aes(x = alpha, y = s, colour = iter, shape = method), data = glmnet_458[method %in% c("smac4hpo", "mbo", "mbo_log")], size = 3) + geom_contour(breaks = breaks) + geom_contour_filled(breaks = breaks, alpha = 0.1) + facet_grid(method ~ num.impute.selected.cpo) + theme(legend.position = "bottom") +g2 = ggplot(data = glmnet_458[method %in% c("smac4hpo", "mbo", "mbo_log") & iter > 12], aes(x = iter, y = cummax(acc), colour = method)) + + geom_step() + instance_x = instance$clone(deep = TRUE) -instance$archive$data = glmnet_458[method == "mlr3mbo" & repl == 1] +instance$archive$data = glmnet_458[method == "mbo_log"] -learner = lrn("regr.ranger_custom") +learner = lrn("regr.ranger", mtry.ratio = 1, num.trees = 10L, se.method = "jack", min.node.size = 1L, splitrule = "extratrees", num.random.splits = 1L) surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% learner)) - acq_function = AcqFunctionEI$new() surrogate$archive = instance$archive acq_function$surrogate = surrogate + +surrogate$y_cols = "y_trafo" +acq_function$surrogate_max_to_min = 1 + +y = instance$archive$data[[instance$archive$cols_y]] * instance$objective_multiplicator +min_y = min(y) - 0.01 * diff(range(y)) +max_y = max(y) +y_log = log((y - min_y) / (max_y - min_y)) +instance$archive$data[, y_trafo := y_log] acq_function$surrogate$update() -acq_function$update() +acq_function$y_best = min(y_log) # manual update + acq_instance = OptimInstanceSingleCrit$new(objective = acq_function, search_space = acq_function$domain, terminator = trm("none"), check_values = FALSE, keep_evals = "all") acq_instance$eval_batch(xdt) +sp = surrogate$predict(xdt) acq_data = copy(acq_instance$archive$data) acq_data = cbind(acq_data, sp) -breaks = quantile(acq_data$mean, c(0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 0.99, 0.999, 1)) +acq_data[which.max(acq_ei), ] acq_instance$archive$clear() -acq_budget = 20000L - -acq_optimizer = { - n_repeats = 2L - maxit = 9L - batch_size = ceiling((acq_budget / n_repeats) / (1 + maxit)) - AcqOptimizer$new(opt("focus_search", n_points = batch_size, maxit = maxit), terminator = trm("evals", n_evals = acq_budget)) -} +optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) +acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20010L)) acq_optimizer$acq_function = acq_function -candidate = acq_instance$archive$best() +candidate = acq_optimizer$optimize() +acq_best = acq_data[which.max(acq_ei), ] g3 = ggplot(data = acq_data, aes(x = alpha, y = s, z = mean)) + - geom_contour(breaks = breaks) + - geom_contour_filled(breaks = breaks) + + geom_contour() + + geom_contour_filled() + facet_wrap(~ num.impute.selected.cpo) + theme(legend.position = "bottom") -saveRDS(instance$archive$data[, c(instance$archive$cols_x, instance$archive$cols_y), with = FALSE], "glmnet_458_mbo_data.rds") +g4 = ggplot(data = acq_data, aes(x = alpha, y = s, z = se)) + + geom_contour() + + geom_contour_filled() + + facet_wrap(~ num.impute.selected.cpo) + + theme(legend.position = "bottom") -library(mlr3) -library(mlr3learners) -library(mlr3pipelines) -library(mlr3viz) - - -tdat = readRDS("glmnet_458_mbo_data.rds") - -tdat[, num.impute.selected.cpo := as.factor(num.impute.selected.cpo)] -task = TaskRegr$new("test", backend = tdat, target = "acc") -resampling = rsmp("repeated_cv", folds = 10L)$instantiate(task) -gp = as_learner(po("encode") %>>% po("fixfactors") %>>% lrn("regr.km", covtype = "matern3_2", optim.method = "gen")) -gp2 = as_learner(po("encodeimpact") %>>% po("fixfactors") %>>% lrn("regr.km", covtype = "matern3_2", optim.method = "gen")) -rp = as_learner(po("fixfactors") %>>% lrn("regr.rpart")) -rf = as_learner(po("fixfactors") %>>% lrn("regr.ranger")) -lm = as_learner(po("fixfactors") %>>% lrn("regr.lm")) -kknn = as_learner(po("fixfactors") %>>% lrn("regr.kknn")) -kknn1 = as_learner(po("fixfactors") %>>% lrn("regr.kknn", k = 1)) -kknn1$id = "kknn1" -mars = as_learner(po("encode") %>>% po("fixfactors") %>>% lrn("regr.mars")) -gam = as_learner(po("encode") %>>% po("fixfactors") %>>% lrn("regr.gam")) -lm = as_learner(po("fixfactors") %>>% lrn("regr.lm")) -learners = list(learner, gp, gp2, rp, rf, kknn, mars, gam, lm) -bg = benchmark_grid(task, learners, resampling) -b = benchmark(bg) -scores = b$aggregate(msrs(c("regr.mse", "regr.mae", "regr.rsq", "regr.ktau"))) -scores_long = melt(scores, id.vars = "learner_id", measure.vars = c("regr.mse", "regr.mae", "regr.rsq", "regr.ktau")) - -ggplot(aes(x = learner_id, y = value), data = scores_long) + - geom_point(size = 3) + - facet_wrap(~ variable, scales = "free") + - theme_minimal() + - theme(axis.text.x = element_text(angle = 45, vjust = 0.5, hjust = 1)) - -breaks = quantile(task$data()$acc, c(seq(0, 1, by = 0.1))) -weights = as.numeric(cut(tdat$acc, breaks)) -weights[is.na(weights)] = 1 -task2 = TaskRegr$new("test", backend = cbind(tdat, weights), target = "acc") -task2$col_roles$weight = "weights" -task2$col_roles$feature = setdiff(task2$col_roles$feature, "weights") -learners = list(rf, kknn, kknn1, lm, gp) - -map(learners, function(l) { - l$train(task2) -}) - -diffs = map_dtr(1:100, function(r) { - print(r) - map_dtr(c(2^(5:13)), function(k) { - xdt_sample = ref[sample(.N, size = k, replace = FALSE), ] - best = xdt_sample[which.max(acc), ] - preds = map_dbl(learners, function(l) { - which.max(l$predict_newdata(xdt_sample)$response) - }) - xdt_learner_p_best = xdt_sample[preds, ] - data.table(diff = best$acc - xdt_learner_p_best$acc, k = k, learner = map_chr(learners, "id"), repl = r) - }) -}) - -mean_diffs = diffs[, .(mean_diff = mean(diff), se_diff = sd(diff) / sqrt(.N)), by = .(k, learner)] - -ggplot(aes(x = k, y = mean_diff, colour = learner), data = mean_diffs) + - geom_point() + - geom_errorbar(aes(ymin = mean_diff - se_diff, ymax = mean_diff + se_diff)) + - geom_line() + - theme_minimal() +g5 = ggplot(data = acq_data, aes(x = alpha, y = s, z = acq_ei)) + + geom_contour() + + geom_contour_filled() + + geom_point(data = candidate, colour = "red", size = 2) + + geom_point(data = acq_best, colour = "white", size = 2) + + facet_wrap(~ num.impute.selected.cpo) + + theme(legend.position = "bottom") + +ggsave("/tmp/lol.png", plot = g5, width = 12, height = 3) + +#library(mlr3) +#library(mlr3learners) +#library(mlr3pipelines) +#library(mlr3viz) +# +#tdat = readRDS("glmnet_458_mbo_data.rds") +# +#tdat[, num.impute.selected.cpo := as.factor(num.impute.selected.cpo)] +#task = TaskRegr$new("test", backend = tdat, target = "acc") +#resampling = rsmp("repeated_cv", folds = 10L)$instantiate(task) +#gp = as_learner(po("encode") %>>% po("fixfactors") %>>% lrn("regr.km", covtype = "matern3_2", optim.method = "gen")) +#gp2 = as_learner(po("encodeimpact") %>>% po("fixfactors") %>>% lrn("regr.km", covtype = "matern3_2", optim.method = "gen")) +#rp = as_learner(po("fixfactors") %>>% lrn("regr.rpart")) +#rf = as_learner(po("fixfactors") %>>% lrn("regr.ranger")) +#lm = as_learner(po("fixfactors") %>>% lrn("regr.lm")) +#kknn = as_learner(po("fixfactors") %>>% lrn("regr.kknn")) +#kknn1 = as_learner(po("fixfactors") %>>% lrn("regr.kknn", k = 1)) +#kknn1$id = "kknn1" +#mars = as_learner(po("encode") %>>% po("fixfactors") %>>% lrn("regr.mars")) +#gam = as_learner(po("encode") %>>% po("fixfactors") %>>% lrn("regr.gam")) +#lm = as_learner(po("fixfactors") %>>% lrn("regr.lm")) +#learners = list(learner, gp, gp2, rp, rf, kknn, mars, gam, lm) +#bg = benchmark_grid(task, learners, resampling) +#b = benchmark(bg) +#scores = b$aggregate(msrs(c("regr.mse", "regr.mae", "regr.rsq", "regr.ktau"))) +#scores_long = melt(scores, id.vars = "learner_id", measure.vars = c("regr.mse", "regr.mae", "regr.rsq", "regr.ktau")) +# +#ggplot(aes(x = learner_id, y = value), data = scores_long) + +# geom_point(size = 3) + +# facet_wrap(~ variable, scales = "free") + +# theme_minimal() + +# theme(axis.text.x = element_text(angle = 45, vjust = 0.5, hjust = 1)) +# +#breaks = quantile(task$data()$acc, c(seq(0, 1, by = 0.1))) +#weights = as.numeric(cut(tdat$acc, breaks)) +#weights[is.na(weights)] = 1 +#task2 = TaskRegr$new("test", backend = cbind(tdat, weights), target = "acc") +#task2$col_roles$weight = "weights" +#task2$col_roles$feature = setdiff(task2$col_roles$feature, "weights") +#learners = list(rf, kknn, kknn1, lm, gp) +# +#map(learners, function(l) { +# l$train(task2) +#}) +# +#diffs = map_dtr(1:100, function(r) { +# print(r) +# map_dtr(c(2^(5:13)), function(k) { +# xdt_sample = ref[sample(.N, size = k, replace = FALSE), ] +# best = xdt_sample[which.max(acc), ] +# preds = map_dbl(learners, function(l) { +# which.max(l$predict_newdata(xdt_sample)$response) +# }) +# xdt_learner_p_best = xdt_sample[preds, ] +# data.table(diff = best$acc - xdt_learner_p_best$acc, k = k, learner = map_chr(learners, "id"), repl = r) +# }) +#}) +# +#mean_diffs = diffs[, .(mean_diff = mean(diff), se_diff = sd(diff) / sqrt(.N)), by = .(k, learner)] +# +#ggplot(aes(x = k, y = mean_diff, colour = learner), data = mean_diffs) + +# geom_point() + +# geom_errorbar(aes(ymin = mean_diff - se_diff, ymax = mean_diff + se_diff)) + +# geom_line() + +# theme_minimal() diff --git a/attic/so_config/analyze_yahpo.R b/attic/so_config/analyze_yahpo.R index a4f7bbef..4c8b7b8f 100644 --- a/attic/so_config/analyze_yahpo.R +++ b/attic/so_config/analyze_yahpo.R @@ -4,7 +4,7 @@ library(pammtools) library(mlr3misc) #dat = rbind(readRDS("results_yahpo.rds"), readRDS("results_yahpo_own.rds"))[method != "mlrintermbo"] -dat = rbind(readRDS("results_yahpo.rds"), readRDS("results_yahpo_own.rds"))[method %in% c("mlr3mbo", "mlrintermbo", "mlr3mbo_new_rf", "mlr3mbo_new_rf_ls", "smac4hpo", "random")] +dat = rbind(readRDS("results_yahpo.rds"), readRDS("results_yahpo_own.rds"))[method %in% c("mlr3mbo", "mlrintermbo", "mlr3mbo_new_rf", "mlr3mbo_new_rf_ls", "mlr3mbo_kknn_ls", "smac4hpo", "random")] dat = dat[scenario %nin% c("nb301", "rbv2_super")] dat[, cumbudget := cumsum(budget), by = .(method, scenario, instance, repl)] dat[, cumbudget_scaled := cumbudget / max(cumbudget), by = .(method, scenario, instance, repl)] @@ -26,7 +26,7 @@ get_incumbent_cumbudget = function(incumbent, cumbudget_scaled) { dat_budget = dat[, .(incumbent_budget = get_incumbent_cumbudget(incumbent, cumbudget_scaled), cumbudget_scaled = seq(0, 1, length.out = 101)), by = .(method, scenario, instance, repl)] agg_budget = dat_budget[, .(mean = mean(incumbent_budget), se = sd(incumbent_budget) / sqrt(.N)), by = .(cumbudget_scaled, method, scenario, instance)] -agg_budget[, method := factor(method, levels = c("random", "smac4hpo", "hb", "bohb", "dehb", "smac4mf", "optuna", "mlr3mbo", "mlr3mbo_default"), labels = c("Random", "SMAC", "HB", "BOHB", "DEHB", "SMAC-HB", "optuna", "mlr3mbo", "mlr3mbo_default"))] +#agg_budget[, method := factor(method, levels = c("random", "smac4hpo", "hb", "bohb", "dehb", "smac4mf", "optuna", "mlr3mbo", "mlr3mbo_default"), labels = c("Random", "SMAC", "HB", "BOHB", "DEHB", "SMAC-HB", "optuna", "mlr3mbo", "mlr3mbo_default"))] g = ggplot(aes(x = cumbudget_scaled, y = mean, colour = method, fill = method), data = agg_budget[cumbudget_scaled > 0.10]) + scale_y_log10() + From 73f216b4f6f7fa5bdd9f83ec8b0e97572c86949d Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Wed, 23 Nov 2022 12:00:02 +0100 Subject: [PATCH 023/126] chore: AcqFunctionCB cleanup --- R/AcqFunctionCB.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/AcqFunctionCB.R b/R/AcqFunctionCB.R index a3497455..c0bcd282 100644 --- a/R/AcqFunctionCB.R +++ b/R/AcqFunctionCB.R @@ -80,7 +80,7 @@ AcqFunctionCB = R6Class("AcqFunctionCB", constants = list(...) lambda = constants$lambda p = self$surrogate$predict(xdt) - res = p$mean - self$surrogate_max_to_min * self$constants$values$lambda * p$se + res = p$mean - self$surrogate_max_to_min * lambda * p$se data.table(acq_cb = res) } ) From 538d384cc54c7c64eb908a66d8bd813a038360d6 Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Wed, 23 Nov 2022 12:01:02 +0100 Subject: [PATCH 024/126] merge acqf_ttei from acqf_ttei branch for now --- R/AcqFunctionTTEI.R | 166 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) create mode 100644 R/AcqFunctionTTEI.R diff --git a/R/AcqFunctionTTEI.R b/R/AcqFunctionTTEI.R new file mode 100644 index 00000000..a2d72887 --- /dev/null +++ b/R/AcqFunctionTTEI.R @@ -0,0 +1,166 @@ +#' @title Acquisition Function Top-Two Expected Improvement +#' +#' @include AcqFunction.R +#' @name mlr_acqfunctions_ttei +#' +#' @templateVar id ttei +#' @template section_dictionary_acqfunctions +#' +#' @description +#' Top-Two Expected Improvement. +#' +#' With probability `beta` simply the Expected Improvement. +#' With probability `1 - beta` the Expected Improvement over the current Expected Improvement. +#' This requires to first obtain the current argmax of the Expected Improvement to then be able to +#' formulate the Expected Improvement with respect to the posterior prediction of that argmax. +#' +#' @references +#' * `r format_bib("qin_2017")` +#' +#' @family Acquisition Function +#' @export +#' @examples +#' if (requireNamespace("mlr3learners") & +#' requireNamespace("DiceKriging") & +#' requireNamespace("rgenoud")) { +#' library(bbotk) +#' library(paradox) +#' library(mlr3learners) +#' library(data.table) +#' +#' fun = function(xs) { +#' list(y = xs$x ^ 2) +#' } +#' domain = ps(x = p_dbl(lower = -10, upper = 10)) +#' codomain = ps(y = p_dbl(tags = "minimize")) +#' objective = ObjectiveRFun$new(fun = fun, domain = domain, codomain = codomain) +#' +#' instance = OptimInstanceSingleCrit$new( +#' objective = objective, +#' terminator = trm("evals", n_evals = 5)) +#' +#' instance$eval_batch(data.table(x = c(-6, -5, 3, 9))) +#' +#' learner = lrn("regr.km", +#' covtype = "matern3_2", +#' optim.method = "gen", +#' nugget.stability = 10^-8, +#' control = list(trace = FALSE)) +#' +#' surrogate = srlrn(learner, archive = instance$archive) +#' +#' acq_function = acqf("ttei", +#' surrogate = surrogate, +#' toplvl_acq_optimizer = acqo(opt("random_search"), terminator = trm("evals", n_evals = 100))) +#' +#' acq_function$surrogate$update() +#' acq_function$update() +#' acq_function$eval_dt(data.table(x = c(-1, 0, 1))) +#' } +AcqFunctionTTEI = R6Class("AcqFunctionTTEI", + inherit = AcqFunction, + + public = list( + #' @field y_best (`numeric(1)`)\cr + #' Best objective function value observed so far. + y_best = NULL, + + #' @field ei_argmax_mean_se ([data.table::data.table()])\cr + #' data.table::data.table() containing one row with the surrogate mean and standard error + #' prediction for the current argmax of the Expected Improvement. + ei_argmax_mean_se = NULL, # FIXME: assert type and format? + + #' @field is_ei (`logical(1)`)\cr + #' Whether in the next iteration simply the Expected Improvement is to be optimized or the + #' Expected Improvement over the Expected Improvement. + is_ei = NULL, + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + #' + #' @param surrogate (`NULL` | [SurrogateLearner]). + #' @param beta (`numeric(1)`)\cr + #' With probability `beta` the toplevel Expected Improvement acquisition function is to be + #' optimized and with `1 - beta` the Expected Improvement over the Expected Improvement. + #' Default is `0.5`. + #' @param toplvl_acq_optimizer (`NULL` | [AcqOptimizer])\cr + #' Acquisition function optimizer for the toplevel Expected Improvement acquisition function + #' used to find the current argmax. + #' If `NULL` will be initialized to + #' `acqo(opt("random_search", batch_size = 1000), terminator = trm("evals", n_evals = 10000))`. + initialize = function(surrogate = NULL, beta = 0.5, toplvl_acq_optimizer = NULL) { + assert_r6(surrogate, "SurrogateLearner", null.ok = TRUE) + assert_number(beta, lower = 0, upper = 1) + assert_r6(toplvl_acq_optimizer, classes = "AcqOptimizer", null.ok = TRUE) + if (is.null(toplvl_acq_optimizer)) toplvl_acq_optimizer = acqo(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 10000L)) + constants = ps(beta = p_dbl(lower = 0, upper = 1, default = 0.5)) + constants$values$beta = beta + private$.toplvl_acq_function_ei = AcqFunctionEI$new(surrogate = surrogate) # FIXME: AB and read-only? + private$.toplvl_acq_optimizer = toplvl_acq_optimizer$clone(deep = TRUE) # FIXME: AB and read-only? + super$initialize("acq_ttei", constants = constants, surrogate = surrogate, direction = "maximize", label = "Top-Two Expected Improvement", man = "mlr3mbo::mlr_acqfunctions_ttei") + }, + + #' @description + #' Updates acquisition function and performs the following steps: + #' * sets `y_best` + #' * updates the internal toplevel Expected Improvement acquisition function + #' * samples whether simply the Expected Improvement is to be optimized or the Expected + #' Improvement over the Expected Improvement (depending on `beta`) and sets `is_ei` accordingly + #' * if `is_ei = FALSE`, proceeds to optimize the toplevel Expected Improvement acquisition function to + #' find the current argmax and sets `ei_argmax_mean_se` + update = function() { + self$y_best = min(self$surrogate_max_to_min * self$archive$data[[self$surrogate$y_cols]]) + private$.toplvl_acq_function_ei$surrogate = self$surrogate + private$.toplvl_acq_function_ei$update() + private$.toplvl_acq_optimizer$acq_function = private$.toplvl_acq_function_ei + if (runif(1, min = 0, max = 1) <= self$constants$values$beta) { + self$is_ei = TRUE + } else { + self$is_ei = FALSE + ei_argmax = private$.toplvl_acq_optimizer$optimize() + p_ei_argmax = self$surrogate$predict(ei_argmax) + self$ei_argmax_mean_se = cbind(ei_argmax, data.table(mean = p_ei_argmax$mean, se = p_ei_argmax$se)) + } + } + ), + + private = list( + .fun = function(xdt, ...) { + constants = list(...) + if (is.null(self$y_best)) { + stop("$y_best is not set. Missed to call $update()?") + } + if (is.null(self$is_ei)) { + stop("$is_ei is not set. Missed to call $update()?") + } + # can xdt_ei_argmax be NULL? + if (self$is_ei) { + p = self$surrogate$predict(xdt) + mu = p$mean + se = p$se + d = self$y_best - self$surrogate_max_to_min * mu + d_norm = d / se + ei = d * pnorm(d_norm) + se * dnorm(d_norm) + ei = ifelse(se < 1e-20, 0, ei) + data.table(acq_ttei = ei, acq_ttei_is_ei = self$is_ei) + } else { + p = self$surrogate$predict(xdt) + mu = p$mean + se = p$se + se_overall = sqrt((self$ei_argmax_mean_se$se ^ 2) + (se ^ 2)) + d = (self$surrogate_max_to_min * self$ei_argmax_mean_se$mean - self$surrogate_max_to_min * mu) + d_norm = d / se_overall + ttei = se_overall * (d_norm * pnorm(d_norm) + dnorm(d_norm)) + ttei = ifelse(se < 1e-20, 0, ttei) # FIXME: what about se_overall? + data.table(acq_ttei = ttei, acq_ttei_is_ei = self$is_ei) + } + }, + + .toplvl_acq_function_ei = NULL, + + .toplvl_acq_optimizer = NULL + ) +) + +mlr_acqfunctions$add("ttei", AcqFunctionTTEI) + From 9ecc7a7c83efdb1e975222afe6120dd4d73f0b5a Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Wed, 23 Nov 2022 18:57:29 +0100 Subject: [PATCH 025/126] .. --- R/bibentries.R | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/R/bibentries.R b/R/bibentries.R index 08fe8017..d9967ab5 100644 --- a/R/bibentries.R +++ b/R/bibentries.R @@ -93,7 +93,7 @@ bibentries = c( pages = "50--66" ), - horn_2015 = bibentry("inproceedings", + horn_2015 = bibentry("inproceedings", title = "Model-Based Multi-objective Optimization: Taxonomy, Multi-Point Proposal, Toolbox and Benchmark", author = "Horn, Daniel and Wagner, Tobias and Biermann, Dirk and Weihs, Claus and Bischl, Bernd", year = "2015", @@ -102,9 +102,17 @@ bibentries = c( ), ginsbourger_2008 = bibentry("misc", - title = "A Multi-Points Criterion for Deterministic Parallel Global Optimization Based on Gaussian processes", - author = "Ginsbourger, David and Le Riche, Rodolphe and Carraro, Laurent", - year = "2008" + title = "A Multi-Points Criterion for Deterministic Parallel Global Optimization Based on Gaussian Processes", + author = "Ginsbourger, David and Le Riche, Rodolphe and Carraro, Laurent", + year = "2008" + ), + + qin_2017 = bibentry("article", + title = "Improving the Expected Improvement Algorithm", + author = "Qin, Chao and Klabjan, Diego and Russo, Daniel", + journal = "Advances in Neural Information Processing Systems", + volume = "30", + year = "2017" ) ) From 5e11422eba980029204367122b46b2e7c26a0876 Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Wed, 23 Nov 2022 19:05:56 +0100 Subject: [PATCH 026/126] prepare new so_config --- .gitignore | 3 + DESCRIPTION | 1 + NAMESPACE | 1 + R/bayesopt_ego_log.R | 8 +- attic/so_config/LearnerRegrRangerCustom.R | 159 ++++++ attic/so_config/run.R | 122 +++-- attic/so_config_old/OptimizerChain.R | 83 ++++ attic/{so_config => so_config_old}/analyze.R | 0 .../analyze_glmnet_458.R | 0 .../analyze_yahpo.R | 0 .../get_filling_candidate.R | 0 attic/{so_config => so_config_old}/helpers.R | 0 attic/so_config_old/min_max.R | 40 ++ attic/so_config_old/run.R | 187 +++++++ attic/so_config_old/run_yahpo.R | 465 ++++++++++++++++++ attic/so_config_old/submit.sh | 20 + man/AcqFunction.Rd | 1 + man/mlr_acqfunctions.Rd | 3 +- man/mlr_acqfunctions_cb.Rd | 1 + man/mlr_acqfunctions_ei.Rd | 1 + man/mlr_acqfunctions_eips.Rd | 1 + man/mlr_acqfunctions_mean.Rd | 1 + man/mlr_acqfunctions_pi.Rd | 1 + man/mlr_acqfunctions_smsego.Rd | 1 + man/mlr_acqfunctions_ttei.Rd | 184 +++++++ man/mlr_loop_functions_mpcl.Rd | 2 +- 26 files changed, 1229 insertions(+), 56 deletions(-) create mode 100644 attic/so_config/LearnerRegrRangerCustom.R create mode 100644 attic/so_config_old/OptimizerChain.R rename attic/{so_config => so_config_old}/analyze.R (100%) rename attic/{so_config => so_config_old}/analyze_glmnet_458.R (100%) rename attic/{so_config => so_config_old}/analyze_yahpo.R (100%) rename attic/{so_config => so_config_old}/get_filling_candidate.R (100%) rename attic/{so_config => so_config_old}/helpers.R (100%) create mode 100644 attic/so_config_old/min_max.R create mode 100755 attic/so_config_old/run.R create mode 100644 attic/so_config_old/run_yahpo.R create mode 100644 attic/so_config_old/submit.sh create mode 100644 man/mlr_acqfunctions_ttei.Rd diff --git a/.gitignore b/.gitignore index 0774b2e0..711b7064 100644 --- a/.gitignore +++ b/.gitignore @@ -39,5 +39,8 @@ Meta attic/so_config/fanova.py attic/so_config/Pipfile attic/so_config/*.rds +attic/so_config_old/fanova.py +attic/so_config_old/Pipfile +attic/so_config_old/*.rds /doc/ /Meta/ diff --git a/DESCRIPTION b/DESCRIPTION index b23e0c47..eb5c24a5 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -80,6 +80,7 @@ Collate: 'AcqFunctionMean.R' 'AcqFunctionPI.R' 'AcqFunctionSmsEgo.R' + 'AcqFunctionTTEI.R' 'AcqOptimizer.R' 'OptimizerMbo.R' 'Surrogate.R' diff --git a/NAMESPACE b/NAMESPACE index 59aea600..37587989 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -10,6 +10,7 @@ export(AcqFunctionEIPS) export(AcqFunctionMean) export(AcqFunctionPI) export(AcqFunctionSmsEgo) +export(AcqFunctionTTEI) export(AcqOptimizer) export(OptimizerMbo) export(Surrogate) diff --git a/R/bayesopt_ego_log.R b/R/bayesopt_ego_log.R index 28f581fd..e2873730 100644 --- a/R/bayesopt_ego_log.R +++ b/R/bayesopt_ego_log.R @@ -67,7 +67,7 @@ #' list(y = xs$x ^ 2) #' } #' domain = ps(x = p_dbl(lower = -10, upper = 10)) -#' codomain = ps(y = p_dbl(tags = "minimize")) +#' codomain = ps(y = p_dbl(tags = "maximize")) #' objective = ObjectiveRFun$new(fun = fun, domain = domain, codomain = codomain) #' #' instance = OptimInstanceSingleCrit$new( @@ -105,7 +105,7 @@ bayesopt_ego_log = function( assert_r6(instance, "OptimInstanceSingleCrit") assert_int(init_design_size, lower = 1L, null.ok = TRUE) assert_r6(surrogate, classes = "Surrogate") # cannot be SurrogateLearner due to EIPS - assert_r6(acq_function, classes = "AcqFunction") + assert_r6(acq_function, classes = "AcqFunction") # FIXME: should explicityly assert acqfs and make sure that codomain tag is handled assert_r6(acq_optimizer, classes = "AcqOptimizer") assert_int(random_interleave_iter, lower = 0L) assert_number(epsilon, lower = 0, upper = 1) @@ -122,6 +122,10 @@ bayesopt_ego_log = function( surrogate$y_cols = "y_trafo" acq_function$surrogate_max_to_min = 1 + if (test_r6(acq_function, classes = "AcqFunctionCB") | test_r6(acq_function, classes = "AcqFunctionMean")) { + acq_function$codomain$params[[acq_function$id]]$tags = "minimize" + } + # initial design if (isTRUE(init_design_size > 0L)) { design = generate_design_random(domain, n = init_design_size)$data diff --git a/attic/so_config/LearnerRegrRangerCustom.R b/attic/so_config/LearnerRegrRangerCustom.R new file mode 100644 index 00000000..78633ec5 --- /dev/null +++ b/attic/so_config/LearnerRegrRangerCustom.R @@ -0,0 +1,159 @@ +#' @title Ranger Regression Learner Custom +#' +#' @name mlr_learners_regr.ranger_custom +#' +#' @description +#' Random regression forest. +#' Calls [ranger::ranger()] from package \CRANpkg{ranger}. +#' +#' @inheritSection mlr_learners_classif.ranger Custom mlr3 parameters +#' @inheritSection mlr_learners_classif.ranger Initial parameter values +#' +#' @templateVar id regr.ranger_custom +#' @template learner +#' +#' @references +#' `r format_bib("wright_2017", "breiman_2001")` +#' +#' @export +#' @template seealso_learner +#' @template example +LearnerRegrRangerCustom = R6Class("LearnerRegrRangerCustom", + inherit = LearnerRegr, + + public = list( + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + initialize = function() { + ps = ps( + alpha = p_dbl(default = 0.5, tags = "train"), + always.split.variables = p_uty(tags = "train"), + holdout = p_lgl(default = FALSE, tags = "train"), + importance = p_fct(c("none", "impurity", "impurity_corrected", "permutation"), tags = "train"), + keep.inbag = p_lgl(default = FALSE, tags = "train"), + max.depth = p_int(default = NULL, lower = 0L, special_vals = list(NULL), tags = "train"), + min.node.size = p_int(1L, default = 5L, special_vals = list(NULL), tags = "train"), + min.prop = p_dbl(default = 0.1, tags = "train"), + minprop = p_dbl(default = 0.1, tags = "train"), + mtry = p_int(lower = 1L, special_vals = list(NULL), tags = "train"), + mtry.ratio = p_dbl(lower = 0, upper = 1, tags = "train"), + num.random.splits = p_int(1L, default = 1L, tags = "train"), + num.threads = p_int(1L, default = 1L, tags = c("train", "predict", "threads")), + num.trees = p_int(1L, default = 500L, tags = c("train", "predict", "hotstart")), + oob.error = p_lgl(default = TRUE, tags = "train"), + quantreg = p_lgl(default = FALSE, tags = "train"), + regularization.factor = p_uty(default = 1, tags = "train"), + regularization.usedepth = p_lgl(default = FALSE, tags = "train"), + replace = p_lgl(default = TRUE, tags = "train"), + respect.unordered.factors = p_fct(c("ignore", "order", "partition"), default = "ignore", tags = "train"), + sample.fraction = p_dbl(0L, 1L, tags = "train"), + save.memory = p_lgl(default = FALSE, tags = "train"), + scale.permutation.importance = p_lgl(default = FALSE, tags = "train"), + se.method = p_fct(c("jack", "infjack", "simple"), default = "infjack", tags = "predict"), # FIXME: only works if predict_type == "se". How to set dependency? + seed = p_int(default = NULL, special_vals = list(NULL), tags = c("train", "predict")), + split.select.weights = p_uty(default = NULL, tags = "train"), + splitrule = p_fct(c("variance", "extratrees", "maxstat"), default = "variance", tags = "train"), + verbose = p_lgl(default = TRUE, tags = c("train", "predict")), + write.forest = p_lgl(default = TRUE, tags = "train") + ) + + ps$values = list(num.threads = 1L) + + # deps + ps$add_dep("num.random.splits", "splitrule", CondEqual$new("extratrees")) + ps$add_dep("alpha", "splitrule", CondEqual$new("maxstat")) + ps$add_dep("minprop", "splitrule", CondEqual$new("maxstat")) + ps$add_dep("scale.permutation.importance", "importance", CondEqual$new("permutation")) + + + super$initialize( + id = "regr.ranger_custom", + param_set = ps, + predict_types = c("response", "se"), + feature_types = c("logical", "integer", "numeric", "character", "factor", "ordered"), + properties = c("weights", "importance", "oob_error", "hotstart_backward"), + packages = c("mlr3learners", "ranger"), + man = "mlr3learners::mlr_learners_regr.ranger_custom" + ) + }, + + #' @description + #' The importance scores are extracted from the model slot `variable.importance`. + #' Parameter `importance.mode` must be set to `"impurity"`, `"impurity_corrected"`, or + #' `"permutation"` + #' + #' @return Named `numeric()`. + importance = function() { + if (is.null(self$model)) { + stopf("No model stored") + } + if (self$model$importance.mode == "none") { + stopf("No importance stored") + } + + sort(self$model$variable.importance, decreasing = TRUE) + }, + + #' @description + #' The out-of-bag error, extracted from model slot `prediction.error`. + #' + #' @return `numeric(1)`. + oob_error = function() { + if (is.null(self$model)) { + stopf("No model stored") + } + self$model$prediction.error + } + ), + + private = list( + .train = function(task) { + pv = self$param_set$get_values(tags = "train") + pv = mlr3learners:::convert_ratio(pv, "mtry", "mtry.ratio", length(task$feature_names)) + + if (self$predict_type == "se") { + pv$keep.inbag = TRUE # nolint + } + + mlr3misc::invoke(ranger::ranger, + dependent.variable.name = task$target_names, + data = task$data(), + case.weights = task$weights$weight, + .args = pv + ) + }, + + .predict = function(task) { + pv = self$param_set$get_values(tags = "predict") + newdata = mlr3learners:::ordered_features(task, self) + + if (isTRUE(pv$se.method == "simple")) { + prediction = mlr3misc::invoke(predict, self$model, data = newdata, type = "response", .args = pv[setdiff(names(pv), "se.method")], predict.all = TRUE) + response = rowMeans(prediction$predictions) + variance = apply(prediction$predictions, MARGIN = 1L, var) + list(response = response, se = sqrt(variance)) + } else { + prediction = mlr3misc::invoke(predict, self$model, data = newdata, type = self$predict_type, .args = pv) + list(response = prediction$predictions, se = prediction$se) + } + }, + + .hotstart = function(task) { + model = self$model + model$num.trees = self$param_set$values$num.trees + model + } + ) +) + +#' @export +default_values.LearnerRegrRangerCustom = function(x, search_space, task, ...) { # nolint + special_defaults = list( + mtry = floor(sqrt(length(task$feature_names))), + mtry.ratio = floor(sqrt(length(task$feature_names))) / length(task$feature_names), + sample.fraction = 1 + ) + defaults = insert_named(default_values(x$param_set), special_defaults) + defaults[search_space$ids()] +} diff --git a/attic/so_config/run.R b/attic/so_config/run.R index 28d9b93f..75667735 100755 --- a/attic/so_config/run.R +++ b/attic/so_config/run.R @@ -6,7 +6,7 @@ library(mlr3) library(mlr3misc) library(mlr3learners) library(mlr3pipelines) -library(bbotk) +library(bbotk) # @localsearch library(paradox) library(R6) library(checkmate) @@ -18,6 +18,7 @@ library("future") library("future.apply") source("OptimizerChain.R") +source("LearnerRegrRangerCustom.R") parser = ArgumentParser() parser$add_argument("-r", "--run", type = "integer", default = 1, help = "Id of run, should be within 1-5") @@ -31,36 +32,18 @@ seeds = c(2409, 2906, 0905, 2412, 3112) set.seed(seeds[run_id]) search_space = ps( + loop_function = p_fct(c("ego", "ego_log")), init = p_fct(c("random", "lhs", "sobol")), init_size_factor = p_int(lower = 1L, upper = 4L), random_interleave = p_lgl(), - random_interleave_iter = p_fct(c("2", "3", "5", "10"), depends = random_interleave == TRUE), - - surrogate = p_fct(c("ranger", "ranger_custom")), - mtry.ratio = p_fct(c("default", "1")), - num.trees = p_fct(c("10", "100", "1000")), - replace = p_lgl(depends = surrogate == "ranger"), - sample.fraction = p_fct(c("0.6", "0.8", "1"), depends = surrogate == "ranger"), - splitrule = p_fct(c("variance", "extratrees"), depends = surrogate == "ranger"), - num.random.splits = p_fct(c("1", "2", "10"), depends = surrogate == "ranger" && splitrule == "extratrees"), - min.node.size = p_fct(c("1", "5"), depends = surrogate == "ranger"), - se.method = p_fct(c("jack", "infjack"), depends = surrogate == "ranger"), - se.simple.spatial = p_lgl(depends = surrogate == "ranger_custom"), - - acqf = p_fct(c("EI", "CB")), - lambda = p_fct(c("1", "2", "3"), depends = acqf == "CB"), + random_interleave_iter = p_fct(c("2", "4", "10"), depends = random_interleave == TRUE), + + rf_type = p_fct(c("standard", "smaclike_boot", "smaclike_no_boot", "smaclike_variance_boot")), + + acqf = p_fct(c("EI", "TTEI", "CB", "PI", "Mean")), + lambda = p_int(lower = 1, upper = 3, depends = acqf == "CB"), acqopt = p_fct(c("RS_1000", "RS", "FS", "LS")) ) -search_space$trafo = function(x, param_set) { - x[["random_interleave_iter"]] = as.integer(x[["random_interleave_iter"]]) - x[["num.trees"]] = as.integer(x[["num.trees"]]) - x[["sample.fraction"]] = as.numeric(x[["sample.fraction"]]) - x[["num.random.splits"]] = as.integer(x[["num.random.splits"]]) - x[["min.node.size"]] = as.integer(x[["min.node.size"]]) - x[["lambda"]] = as.numeric(x[["lambda"]]) - x[map_lgl(x, function(y) length(y) == 0L)] = NA - x -} instances = readRDS("instances.rds") #instances = data.table(scenario = rep(paste0("rbv2_", c("aknn", "glmnet", "ranger", "rpart", "super", "svm", "xgboost")), each = 5L), @@ -103,28 +86,45 @@ evaluate = function(xdt, instance) { optim_instance$eval_batch(init_design) - random_interleave_iter = if(xdt$random_interleave) xdt$random_interleave_iter else 0L + random_interleave_iter = if(xdt$random_interleave) as.numeric(xdt$random_interleave_iter) else 0L - if (xdt$surrogate == "ranger") { - learner = lrn("regr.ranger", keep.inbag = TRUE) - values = as.list(xdt[, c("num.trees", "replace", "sample.fraction", "splitrule", "num.random.splits", "min.node.size", "se.method")]) - values = values[!map_lgl(values, function(x) is.na(x))] - } else if (xdt$surrogate == "ranger_custom") { - learner = lrn("regr.ranger_custom") - values = as.list(xdt[, c("num.trees", "se.simple.spatial")]) - values = values[!map_lgl(values, function(x) is.na(x))] + learner = LearnerRegrRangerCustom$new() + learner$predict_type = "se" + learner$param_set$values$keep.inbag = TRUE + + if (xdt$rf_type == "standard") { + learner$param_set$values$se.method = "jack" + learner$param_set$values$splitrule = "variance" + learner$param_set$values$num.trees = 1000L + } else if (xdt$rf_type == "smaclike_boot") { + learner$param_set$values$se.method = "simple" + learner$param_set$values$splitrule = "extratrees" + learner$param_set$values$num.random.splits = 1L + learner$param_set$values$num.trees = 10 + learner$param_set$values$replace = TRUE + learner$param_set$values$sample.fraction = 1 + learner$param_set$values$min.node.size = 1 + learner$param_set$values$mtry.ratio = 1 + } else if (xdt$rf_type == "smaclike_no_boot") { + learner$param_set$values$se.method = "simple" + learner$param_set$values$splitrule = "extratrees" + learner$param_set$values$num.random.splits = 1L + learner$param_set$values$num.trees = 10 + learner$param_set$values$replace = FALSE + learner$param_set$values$sample.fraction = 1 + learner$param_set$values$min.node.size = 1 + learner$param_set$values$mtry.ratio = 1 + } else if (xdt$rf_type == "smaclike_variance_boot") { + learner$param_set$values$se.method = "simple" + learner$param_set$values$splitrule = "variance" + learner$param_set$values$num.trees = 10 + learner$param_set$values$replace = TRUE + learner$param_set$values$sample.fraction = 1 + learner$param_set$values$min.node.size = 1 + learner$param_set$values$mtry.ratio = 1 } - if (xdt$mtry.ratio == "1") { - values$mtry.ratio = 1 - } - learner$param_set$values = insert_named(learner$param_set$values %??% list(), values) - surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% learner)) - acq_function = if (xdt$acqf == "EI") { - AcqFunctionEI$new() - } else if (xdt$acqf == "CB") { - AcqFunctionCB$new(lambda = xdt$lambda) - } + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor", multiplier = 3) %>>% learner)) acq_optimizer = if (xdt$acqopt == "RS_1000") { AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 1000L)) @@ -139,11 +139,27 @@ evaluate = function(xdt, instance) { optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20020L)) acq_optimizer$param_set$values$warmstart = TRUE - acq_optimizer$param_set$values$warmstart_size = "all" + acq_optimizer$param_set$values$warmstart_size = 10L acq_optimizer } - - bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + + acq_function = if (xdt$acqf == "EI") { + AcqFunctionEI$new() + } else if (xdt$acqf == "TTEI") { + AcqFunctionTTEI$new(toplvl_acq_optimizer = acq_optimizer$clone(deep = TRUE)) + } else if (xdt$acqf == "CB") { + AcqFunctionCB$new(lambda = xdt$lambda) + } else if (xdt$acqf == "PI") { + AcqFunctionPI$new() + } else if (xdt$acqf == "Mean") { + AcqFunctionMean$new() + } + + if (xdt$loop_function == "ego") { + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + } else if (xdt$loop_function == "ego_log") { + bayesopt_ego_log(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + } best = optim_instance$archive$best()[[instance$target]] ecdf_best = instance$ecdf(best) # evaluate the precomputed ecdf for the best value found; our target is effectively P(X <= best) @@ -173,14 +189,16 @@ ac_instance = OptimInstanceSingleCrit$new( terminator = trm("evals", n_evals = 200L) # 100 init design + 100 ) -surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% lrn("regr.ranger", num.trees = 1000L, se.method = "jack", keep.inbag = TRUE))) -acq_function = AcqFunctionCB$new(lambda = 3) +surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% lrn("regr.ranger", keep.inbag = TRUE, se.method = "jack", num.trees = 1000L))) +acq_function = AcqFunctionEI$new() optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20020L)) acq_optimizer$param_set$values$warmstart = TRUE -acq_optimizer$param_set$values$warmstart_size = "all" +acq_optimizer$param_set$values$warmstart_size = 10L design = generate_design_sobol(ac_instance$search_space, n = 100L)$data -ac_instance$eval_batch(design) +for (i in seq_len(nrow(design))) { + ac_instance$eval_batch(design[i, ]) +} saveRDS(ac_instance, paste0("ac_instance_", run_id, ".rds")) bayesopt_ego(ac_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = 5L) saveRDS(ac_instance, paste0("ac_instance_", run_id, ".rds")) diff --git a/attic/so_config_old/OptimizerChain.R b/attic/so_config_old/OptimizerChain.R new file mode 100644 index 00000000..4f04a373 --- /dev/null +++ b/attic/so_config_old/OptimizerChain.R @@ -0,0 +1,83 @@ +OptimizerChain = R6Class("OptimizerChain", inherit = bbotk::Optimizer, + public = list( + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + #' + #' @param optimizers (list of [Optimizer]s). + #' @param terminators (list of [Terminator]s | NULL). + initialize = function(optimizers, terminators = rep(list(NULL), length(optimizers))) { + assert_list(optimizers, types = "Optimizer", any.missing = FALSE) + assert_list(terminators, types = c("Terminator", "NULL"), len = length(optimizers)) + + param_sets = vector(mode = "list", length = length(optimizers)) + ids_taken = character(0L) + # for each optimizer check whether the id of the param_set + # (decuded from the optimizer class) is already taken; + # if necessary postfix the id + for (i_opt in seq_along(optimizers)) { + opt = optimizers[[i_opt]] + ps = opt$param_set$clone(deep = TRUE) + ps$set_id = class(opt)[[1L]] + try_postfix = 0L + while (ps$set_id %in% ids_taken) { + try_postfix = try_postfix + 1L + ps$set_id = paste0(class(opt)[[1L]], "_", try_postfix) + } + ids_taken[[i_opt]] = ps$set_id + param_sets[[i_opt]] = ps + } + private$.ids = map_chr(param_sets, "set_id") + super$initialize( + param_set = ParamSetCollection$new(param_sets), + param_classes = Reduce(intersect, mlr3misc::map(optimizers, "param_classes")), + properties = Reduce(intersect, mlr3misc::map(optimizers, "properties")), + packages = unique(unlist(mlr3misc::map(optimizers, "packages"))) + ) + private$.optimizers = optimizers + private$.terminators = terminators + } + ), + + private = list( + .optimizers = NULL, + .terminators = NULL, + .ids = NULL, + + .optimize = function(inst) { + terminator = inst$terminator + on.exit({inst$terminator = terminator}) + inner_inst = inst$clone(deep = TRUE) + + for (i_opt in seq_along(private$.optimizers)) { + inner_term = private$.terminators[[i_opt]] + if (!is.null(inner_term)) { + inner_inst$terminator = TerminatorCombo$new(list(inner_term, terminator)) + } else { + inner_inst$terminator = terminator + } + opt = private$.optimizers[[i_opt]] + opt$param_set$values = self$param_set$.__enclos_env__$private$.sets[[i_opt]]$values + opt$optimize(inner_inst) + inner_inst$archive$data$batch_nr = max(inst$archive$data$batch_nr, 0L) + + inner_inst$archive$data$batch_nr + inner_inst$archive$data$optimizer = private$.ids[i_opt] + inst$archive$data = rbind(inst$archive$data, inner_inst$archive$data, fill = TRUE) + inner_inst$archive$data = data.table() + if (terminator$is_terminated(inst$archive)) { + break + } + } + }, + + deep_clone = function(name, value) { + switch( + name, + .optimizers = mlr3misc::map(value, .f = function(x) x$clone(deep = TRUE)), + .terminators = mlr3misc::map(value, .f = function(x) if (!is.null(x)) x$clone(deep = TRUE)), + value + ) + } + ) +) + diff --git a/attic/so_config/analyze.R b/attic/so_config_old/analyze.R similarity index 100% rename from attic/so_config/analyze.R rename to attic/so_config_old/analyze.R diff --git a/attic/so_config/analyze_glmnet_458.R b/attic/so_config_old/analyze_glmnet_458.R similarity index 100% rename from attic/so_config/analyze_glmnet_458.R rename to attic/so_config_old/analyze_glmnet_458.R diff --git a/attic/so_config/analyze_yahpo.R b/attic/so_config_old/analyze_yahpo.R similarity index 100% rename from attic/so_config/analyze_yahpo.R rename to attic/so_config_old/analyze_yahpo.R diff --git a/attic/so_config/get_filling_candidate.R b/attic/so_config_old/get_filling_candidate.R similarity index 100% rename from attic/so_config/get_filling_candidate.R rename to attic/so_config_old/get_filling_candidate.R diff --git a/attic/so_config/helpers.R b/attic/so_config_old/helpers.R similarity index 100% rename from attic/so_config/helpers.R rename to attic/so_config_old/helpers.R diff --git a/attic/so_config_old/min_max.R b/attic/so_config_old/min_max.R new file mode 100644 index 00000000..7f396a90 --- /dev/null +++ b/attic/so_config_old/min_max.R @@ -0,0 +1,40 @@ +library(data.table) +library(bbotk) +library(mlr3misc) +library(paradox) +reticulate::use_virtualenv("/home/lps/.local/share/virtualenvs/yahpo_gym-4ygV7ggv/", required = TRUE) +library(reticulate) +library(yahpogym) + +instances = data.table(scenario = rep(paste0("rbv2_", c("aknn", "glmnet", "ranger", "rpart", "super", "svm", "xgboost")), each = 5L), + instance = c("40499", "1476", "6", "12", "41150", + "40979", "1501", "40966", "1478", "40984", + "12", "458", "1510", "1515", "307", + "1478", "40979", "12", "28", "1501", + "41164", "37", "1515", "1510", "42", + "1478", "1501", "40499", "40979", "300", + "40984", "40979", "40966", "28", "22"), + target = "acc", + budget = rep(c(118, 90, 134, 110, 267, 118, 170), each = 5L)) + +make_optim_instance = function(instance) { + benchmark = BenchmarkSet$new(instance$scenario, instance = instance$instance) + benchmark$subset_codomain(instance$target) + objective = benchmark$get_objective(instance$instance, multifidelity = FALSE, check_values = FALSE) + budget = instance$budget + optim_instance = OptimInstanceSingleCrit$new(objective, search_space = benchmark$get_search_space(drop_fidelity_params = TRUE), terminator = trm("evals", n_evals = 1000000L), check_values = FALSE) + optim_instance +} + +evaluate = function(instance) { + optim_instance = make_optim_instance(instance) + opt("random_search", batch_size = 10000L)$optimize(optim_instance) + ys = optim_instance$archive$data[, optim_instance$archive$cols_y, with = FALSE][[1L]] + data.table(min = min(ys), max = max(ys), mean = mean(ys), sd = sd(ys), ecdf = list(ecdf(ys))) +} + +y_stats = map_dtr(seq_len(nrow(instances)), function(i) { + evaluate(instances[i, ]) +}) + +saveRDS(cbind(instances, y_stats), "instances.rds") diff --git a/attic/so_config_old/run.R b/attic/so_config_old/run.R new file mode 100755 index 00000000..28d9b93f --- /dev/null +++ b/attic/so_config_old/run.R @@ -0,0 +1,187 @@ +#!/usr/bin/env Rscript +# chmod ug+x +library(argparse) +library(data.table) +library(mlr3) +library(mlr3misc) +library(mlr3learners) +library(mlr3pipelines) +library(bbotk) +library(paradox) +library(R6) +library(checkmate) +library(mlr3mbo) # @so_config +reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) +library(reticulate) +library(yahpogym) +library("future") +library("future.apply") + +source("OptimizerChain.R") + +parser = ArgumentParser() +parser$add_argument("-r", "--run", type = "integer", default = 1, help = "Id of run, should be within 1-5") +args = parser$parse_args() +run_id = args$run +stopifnot(run_id %in% 1:5) +cat("run_id:", run_id, "\n") + +seeds = c(2409, 2906, 0905, 2412, 3112) + +set.seed(seeds[run_id]) + +search_space = ps( + init = p_fct(c("random", "lhs", "sobol")), + init_size_factor = p_int(lower = 1L, upper = 4L), + random_interleave = p_lgl(), + random_interleave_iter = p_fct(c("2", "3", "5", "10"), depends = random_interleave == TRUE), + + surrogate = p_fct(c("ranger", "ranger_custom")), + mtry.ratio = p_fct(c("default", "1")), + num.trees = p_fct(c("10", "100", "1000")), + replace = p_lgl(depends = surrogate == "ranger"), + sample.fraction = p_fct(c("0.6", "0.8", "1"), depends = surrogate == "ranger"), + splitrule = p_fct(c("variance", "extratrees"), depends = surrogate == "ranger"), + num.random.splits = p_fct(c("1", "2", "10"), depends = surrogate == "ranger" && splitrule == "extratrees"), + min.node.size = p_fct(c("1", "5"), depends = surrogate == "ranger"), + se.method = p_fct(c("jack", "infjack"), depends = surrogate == "ranger"), + se.simple.spatial = p_lgl(depends = surrogate == "ranger_custom"), + + acqf = p_fct(c("EI", "CB")), + lambda = p_fct(c("1", "2", "3"), depends = acqf == "CB"), + acqopt = p_fct(c("RS_1000", "RS", "FS", "LS")) +) +search_space$trafo = function(x, param_set) { + x[["random_interleave_iter"]] = as.integer(x[["random_interleave_iter"]]) + x[["num.trees"]] = as.integer(x[["num.trees"]]) + x[["sample.fraction"]] = as.numeric(x[["sample.fraction"]]) + x[["num.random.splits"]] = as.integer(x[["num.random.splits"]]) + x[["min.node.size"]] = as.integer(x[["min.node.size"]]) + x[["lambda"]] = as.numeric(x[["lambda"]]) + x[map_lgl(x, function(y) length(y) == 0L)] = NA + x +} + +instances = readRDS("instances.rds") +#instances = data.table(scenario = rep(paste0("rbv2_", c("aknn", "glmnet", "ranger", "rpart", "super", "svm", "xgboost")), each = 5L), +# instance = c("40499", "1476", "6", "12", "41150", +# "40979", "1501", "40966", "1478", "40984", +# "12", "458", "1510", "1515", "307", +# "1478", "40979", "12", "28", "1501", +# "41164", "37", "1515", "1510", "42", +# "1478", "1501", "40499", "40979", "300", +# "40984", "40979", "40966", "28", "22"), +# target = "acc", +# budget = rep(c(118, 90, 134, 110, 267, 118, 170), each = 5L)) + +make_optim_instance = function(instance) { + benchmark = BenchmarkSet$new(instance$scenario, instance = instance$instance) + benchmark$subset_codomain(instance$target) + objective = benchmark$get_objective(instance$instance, multifidelity = FALSE, check_values = FALSE) + budget = instance$budget + optim_instance = OptimInstanceSingleCrit$new(objective, search_space = benchmark$get_search_space(drop_fidelity_params = TRUE), terminator = trm("evals", n_evals = budget), check_values = FALSE) + optim_instance +} + +evaluate = function(xdt, instance) { + logger = lgr::get_logger("mlr3") + logger$set_threshold("warn") + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + + optim_instance = make_optim_instance(instance) + + d = optim_instance$search_space$length + init_design_size = d * xdt$init_size_factor + init_design = if (xdt$init == "random") { + generate_design_random(optim_instance$search_space, n = init_design_size)$data + } else if (xdt$init == "lhs") { + generate_design_lhs(optim_instance$search_space, n = init_design_size)$data + } else if (xdt$init == "sobol") { + generate_design_sobol(optim_instance$search_space, n = init_design_size)$data + } + + optim_instance$eval_batch(init_design) + + random_interleave_iter = if(xdt$random_interleave) xdt$random_interleave_iter else 0L + + if (xdt$surrogate == "ranger") { + learner = lrn("regr.ranger", keep.inbag = TRUE) + values = as.list(xdt[, c("num.trees", "replace", "sample.fraction", "splitrule", "num.random.splits", "min.node.size", "se.method")]) + values = values[!map_lgl(values, function(x) is.na(x))] + } else if (xdt$surrogate == "ranger_custom") { + learner = lrn("regr.ranger_custom") + values = as.list(xdt[, c("num.trees", "se.simple.spatial")]) + values = values[!map_lgl(values, function(x) is.na(x))] + } + if (xdt$mtry.ratio == "1") { + values$mtry.ratio = 1 + } + learner$param_set$values = insert_named(learner$param_set$values %??% list(), values) + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% learner)) + + acq_function = if (xdt$acqf == "EI") { + AcqFunctionEI$new() + } else if (xdt$acqf == "CB") { + AcqFunctionCB$new(lambda = xdt$lambda) + } + + acq_optimizer = if (xdt$acqopt == "RS_1000") { + AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 1000L)) + } else if (xdt$acqopt == "RS") { + AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 20000L)) + } else if (xdt$acqopt == "FS") { + n_repeats = 2L + maxit = 9L + batch_size = ceiling((20000L / n_repeats) / (1 + maxit)) # 1000L + AcqOptimizer$new(opt("focus_search", n_points = batch_size, maxit = maxit), terminator = trm("evals", n_evals = 20000L)) + } else if (xdt$acqopt == "LS") { + optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) + acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20020L)) + acq_optimizer$param_set$values$warmstart = TRUE + acq_optimizer$param_set$values$warmstart_size = "all" + acq_optimizer + } + + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + + best = optim_instance$archive$best()[[instance$target]] + ecdf_best = instance$ecdf(best) # evaluate the precomputed ecdf for the best value found; our target is effectively P(X <= best) + cat("scenario:", instance$scenario, "instance:", instance$instance, "ECDF_best:", ecdf_best, "\n") + ecdf_best +} + +objective = ObjectiveRFunDt$new( + fun = function(xdt) { + saveRDS(ac_instance, paste0("ac_instance_", run_id, ".rds")) + map_dtr(seq_len(nrow(xdt)), function(i) { + plan("multicore") + tmp = future_lapply(transpose_list(instances), function(instance) { + res_instance = tryCatch(evaluate(xdt[i, ], instance), error = function(error_condition) 0) + }, future.seed = TRUE) + data.table(mean_perf = mean(unlist(tmp), na.rm = TRUE), raw_perfs = list(tmp), n_na = sum(is.na(unlist(tmp)))) + }) + }, + domain = search_space, + codomain = ps(mean_perf = p_dbl(tags = "maximize")), + check_values = FALSE +) + +ac_instance = OptimInstanceSingleCrit$new( + objective = objective, + search_space = search_space, + terminator = trm("evals", n_evals = 200L) # 100 init design + 100 +) + +surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% lrn("regr.ranger", num.trees = 1000L, se.method = "jack", keep.inbag = TRUE))) +acq_function = AcqFunctionCB$new(lambda = 3) +optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) +acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20020L)) +acq_optimizer$param_set$values$warmstart = TRUE +acq_optimizer$param_set$values$warmstart_size = "all" +design = generate_design_sobol(ac_instance$search_space, n = 100L)$data +ac_instance$eval_batch(design) +saveRDS(ac_instance, paste0("ac_instance_", run_id, ".rds")) +bayesopt_ego(ac_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = 5L) +saveRDS(ac_instance, paste0("ac_instance_", run_id, ".rds")) + diff --git a/attic/so_config_old/run_yahpo.R b/attic/so_config_old/run_yahpo.R new file mode 100644 index 00000000..d40a3217 --- /dev/null +++ b/attic/so_config_old/run_yahpo.R @@ -0,0 +1,465 @@ +library(batchtools) +library(data.table) +library(mlr3) +library(mlr3learners) +library(mlr3pipelines) +library(mlr3misc) +library(mlr3mbo) # @so_config +library(bbotk) +library(paradox) +library(R6) +library(checkmate) + +reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) +library(reticulate) +yahpo_gym = import("yahpo_gym") + +packages = c("data.table", "mlr3", "mlr3learners", "mlr3pipelines", "mlr3misc", "mlr3mbo", "bbotk", "paradox", "mlrintermbo", "R6", "checkmate") + +RhpcBLASctl::blas_set_num_threads(1L) +RhpcBLASctl::omp_set_num_threads(1L) + +root = here::here() +experiments_dir = file.path(root) + +source_files = map_chr(c("helpers.R", "OptimizerChain.R"), function(x) file.path(experiments_dir, x)) +for (sf in source_files) { + source(sf) +} + +reg = makeExperimentRegistry(file.dir = "/gscratch/lschnei8/registry_mlr3mbo_so_config", packages = packages, source = source_files) +#reg = makeExperimentRegistry(file.dir = NA, conf.file = NA, packages = packages, source = source_files) # interactive session +saveRegistry(reg) + +# FIXME: also compare jack vs. infjack? +mlr3mbo_wrapper = function(job, data, instance, ...) { + reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + optim_instance = make_optim_instance(instance) + + xdt = list(init = "lhs", init_size_factor = 4L, random_interleave = FALSE, num.trees = 250L, splitrule = "extratrees", num.random.splits = 8, acqf = "CB", lambda = 2.8, acqopt_iter_factor = 6L, acqopt = "FS", fs_behavior = "global") + + d = optim_instance$search_space$length + init_design_size = d * xdt$init_size_factor + init_design = if (xdt$init == "random") generate_design_random(optim_instance$search_space, n = init_design_size)$data else if (xdt$init == "lhs") generate_design_lhs(optim_instance$search_space, n = init_design_size)$data + optim_instance$eval_batch(init_design) + + random_interleave_iter = if(xdt$random_interleave) xdt$random_interleave_iter else 0L + + learner = if (xdt$splitrule == "extratrees") { + lrn("regr.ranger", num.trees = xdt$num.trees, keep.inbag = TRUE, splitrule = xdt$splitrule, num.random.splits = xdt$num.random.splits) + } else if (xdt$splitrule == "variance") { + lrn("regr.ranger", num.trees = xdt$num.trees, keep.inbag = TRUE, splitrule = xdt$splitrule) + } + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% learner)) + + acq_function = if (xdt$acqf == "EI") { + AcqFunctionEI$new() + } else if (xdt$acqf == "CB") { + AcqFunctionCB$new(lambda = xdt$lambda) + } else if (xdt$acqf == "PI") { + AcqFunctionPI$new() + } + + acq_budget = 1000 * xdt$acqopt_iter_factor + + acq_optimizer = if (xdt$acqopt == "RS") { + AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = acq_budget)) + } else if (xdt$acqopt == "FS") { + if (xdt$fs_behavior == "global") { + n_repeats = 10L + maxit = 2L + batch_size = ceiling((acq_budget / n_repeats) / (1 + maxit)) + } else if (xdt$fs_behavior == "local") { + n_repeats = 2L + maxit = 10L + batch_size = ceiling((acq_budget / n_repeats) / (1 + maxit)) + } + AcqOptimizer$new(opt("focus_search", n_points = batch_size, maxit = maxit), terminator = trm("evals", n_evals = acq_budget)) + } + + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + optim_instance +} + +mlrintermbo_wrapper = function(job, data, instance, ...) { + reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + optim_instance = make_optim_instance(instance) + optimizer = opt("intermbo", on.surrogate.error = "stop") + learner = lrn("regr.ranger", se.method = "jack", keep.inbag = TRUE) + learner$predict_type = "se" + learner = GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% learner) + learner$predict_type = "se" + #learner = mlr::makeLearner("regr.randomForest", se.method = "jackknife", keep.inbag = TRUE) + #learner = mlr::makeImputeWrapper(learner, classes = list(numeric = mlr::imputeMax(2), factor = mlr::imputeConstant("__miss__"), logical = mlr::imputeUniform())) + #learner = mlr::setPredictType(learner, "se") + #optimizer$param_set$values$surrogate.learner = learner + optimizer$optimize(optim_instance) + optim_instance +} + +mlr3mbo_default_wrapper = function(job, data, instance, ...) { + reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + optim_instance = make_optim_instance(instance) + + xdt = list(init = "random", init_size_factor = 4L, random_interleave = FALSE, num.trees = 500L, splitrule = "variance", num.random.splits = NA_integer_, acqf = "EI", lambda = NA_real_, acqopt_iter_factor = 10L, acqopt = "RS", fs_behavior = NA_character_) + + d = optim_instance$search_space$length + init_design_size = d * xdt$init_size_factor + init_design = if (xdt$init == "random") generate_design_random(optim_instance$search_space, n = init_design_size)$data else if (xdt$init == "lhs") generate_design_lhs(optim_instance$search_space, n = init_design_size)$data + optim_instance$eval_batch(init_design) + + random_interleave_iter = if(xdt$random_interleave) xdt$random_interleave_iter else 0L + + learner = if (xdt$splitrule == "extratrees") { + lrn("regr.ranger", num.trees = xdt$num.trees, keep.inbag = TRUE, splitrule = xdt$splitrule, num.random.splits = xdt$num.random.splits) + } else if (xdt$splitrule == "variance") { + lrn("regr.ranger", num.trees = xdt$num.trees, keep.inbag = TRUE, splitrule = xdt$splitrule) + } + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% learner)) + + acq_function = if (xdt$acqf == "EI") { + AcqFunctionEI$new() + } else if (xdt$acqf == "CB") { + AcqFunctionCB$new(lambda = xdt$lambda) + } else if (xdt$acqf == "PI") { + AcqFunctionPI$new() + } + + acq_budget = 1000 * xdt$acqopt_iter_factor + + acq_optimizer = if (xdt$acqopt == "RS") { + AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = acq_budget)) + } else if (xdt$acqopt == "FS") { + if (xdt$fs_behavior == "global") { + n_repeats = 10L + maxit = 2L + batch_size = ceiling((acq_budget / n_repeats) / (1 + maxit)) + } else if (xdt$fs_behavior == "local") { + n_repeats = 2L + maxit = 10L + batch_size = ceiling((acq_budget / n_repeats) / (1 + maxit)) + } + AcqOptimizer$new(opt("focus_search", n_points = batch_size, maxit = maxit), terminator = trm("evals", n_evals = acq_budget)) + } + + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + optim_instance +} + +mlr3mbo_wrapper_custom = function(job, data, instance, ...) { + reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + optim_instance = make_optim_instance(instance) + + xdt = list(init = "lhs", init_size_factor = 1L, random_interleave = FALSE, num.trees = 250L, splitrule = "extratrees", num.random.splits = 10, acqf = "CB", lambda = 3, acqopt_iter_factor = 30L, acqopt = "FS", fs_behavior = "global") + + d = optim_instance$search_space$length + init_design_size = d * xdt$init_size_factor + init_design = if (xdt$init == "random") generate_design_random(optim_instance$search_space, n = init_design_size)$data else if (xdt$init == "lhs") generate_design_lhs(optim_instance$search_space, n = init_design_size)$data + optim_instance$eval_batch(init_design) + + random_interleave_iter = if(xdt$random_interleave) xdt$random_interleave_iter else 0L + + learner = if (xdt$splitrule == "extratrees") { + lrn("regr.ranger", num.trees = xdt$num.trees, keep.inbag = TRUE, splitrule = xdt$splitrule, num.random.splits = xdt$num.random.splits) + } else if (xdt$splitrule == "variance") { + lrn("regr.ranger", num.trees = xdt$num.trees, keep.inbag = TRUE, splitrule = xdt$splitrule) + } + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% learner)) + + acq_function = if (xdt$acqf == "EI") { + AcqFunctionEI$new() + } else if (xdt$acqf == "CB") { + AcqFunctionCB$new(lambda = xdt$lambda) + } else if (xdt$acqf == "PI") { + AcqFunctionPI$new() + } + + acq_budget = 1000 * xdt$acqopt_iter_factor + + acq_optimizer = if (xdt$acqopt == "RS") { + AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = acq_budget)) + } else if (xdt$acqopt == "FS") { + if (xdt$fs_behavior == "global") { + n_repeats = 5L + maxit = 5L + batch_size = ceiling((acq_budget / n_repeats) / (1 + maxit)) + } else if (xdt$fs_behavior == "local") { + n_repeats = 2L + maxit = 10L + batch_size = ceiling((acq_budget / n_repeats) / (1 + maxit)) + } + AcqOptimizer$new(opt("focus_search", n_points = batch_size, maxit = maxit), terminator = trm("evals", n_evals = acq_budget)) + } + + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + optim_instance +} + +mlr3mbo_wrapper_new_rf = function(job, data, instance, ...) { + reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + optim_instance = make_optim_instance(instance) + + d = optim_instance$search_space$length + init_design_size = 4L * d + init_design = generate_design_lhs(optim_instance$search_space, n = init_design_size)$data + optim_instance$eval_batch(init_design) + + random_interleave_iter = 0L + + learner = lrn("regr.ranger_custom") + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% learner)) + + acq_function = AcqFunctionEI$new() + + acq_budget = 20000L + + acq_optimizer = { + n_repeats = 2L + maxit = 9L + batch_size = ceiling((acq_budget / n_repeats) / (1 + maxit)) + AcqOptimizer$new(opt("focus_search", n_points = batch_size, maxit = maxit), terminator = trm("evals", n_evals = acq_budget)) + } + + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + optim_instance +} + +mlr3mbo_wrapper_new_rf_ls = function(job, data, instance, ...) { + reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + optim_instance = make_optim_instance(instance) + + d = optim_instance$search_space$length + init_design_size = 4L * d + init_design = generate_design_lhs(optim_instance$search_space, n = init_design_size)$data + optim_instance$eval_batch(init_design) + + random_interleave_iter = 0L + + learner = lrn("regr.ranger_custom") + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% learner)) + + acq_function = AcqFunctionEI$new() + + optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) + acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20010L)) + + acq_optimizer$param_set$values$warmstart = TRUE + acq_optimizer$param_set$values$warmstart_size = "all" + + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + optim_instance +} + +mlr3mbo_wrapper_xxx_ls = function(job, data, instance, ...) { + reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + optim_instance = make_optim_instance(instance) + + d = optim_instance$search_space$length + init_design_size = 4L * d + init_design = generate_design_sobol(optim_instance$search_space, n = init_design_size)$data + optim_instance$eval_batch(init_design) + + random_interleave_iter = 4L + + learner = lrn("regr.ranger_custom") + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor", multiplier = 2) %>>% learner)) + + acq_function = AcqFunctionEI$new() + + optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) + acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20010L)) + + acq_optimizer$param_set$values$warmstart = TRUE + acq_optimizer$param_set$values$warmstart_size = "all" + + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + optim_instance +} + +mlr3mbo_wrapper_kknn_ls = function(job, data, instance, ...) { + reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + optim_instance = make_optim_instance(instance) + + d = optim_instance$search_space$length + init_design_size = 4L * d + init_design = generate_design_sobol(optim_instance$search_space, n = init_design_size)$data + optim_instance$eval_batch(init_design) + + random_interleave_iter = 4L + + learner = lrn("regr.kknn") + learner = as_learner(po("imputeoor", multiplier = 2) %>>% po("fixfactors") %>>% po("imputesample") %>>% learner) + learner$predict_types = "response" + surrogate = SurrogateLearner$new(learner) + + acq_function = AcqFunctionMean$new() + + optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) + acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20010L)) + + acq_optimizer$param_set$values$warmstart = TRUE + acq_optimizer$param_set$values$warmstart_size = "all" + + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + optim_instance +} + +# add algorithms +addAlgorithm("mlr3mbo", fun = mlr3mbo_wrapper) +addAlgorithm("mlrintermbo", fun = mlrintermbo_wrapper) +addAlgorithm("mlr3mbo_default", fun = mlr3mbo_default_wrapper) +addAlgorithm("mlr3mbo_custom", fun = mlr3mbo_wrapper_custom) +addAlgorithm("mlr3mbo_new_rf", fun = mlr3mbo_wrapper_new_rf) +addAlgorithm("mlr3mbo_new_rf_ls", fun = mlr3mbo_wrapper_new_rf_ls) +addAlgorithm("mlr3mbo_xxx_ls", fun = mlr3mbo_wrapper_xxx_ls) +addAlgorithm("mlr3mbo_kknn_ls", fun = mlr3mbo_wrapper_kknn_ls) + +# setup scenarios and instances +get_nb301_setup = function(budget_factor = 40L) { + scenario = "nb301" + bench = yahpo_gym$benchmark_set$BenchmarkSet(scenario, instance = "CIFAR10") + fidelity_space = bench$get_fidelity_space() + fidelity_param_id = fidelity_space$get_hyperparameter_names()[1] + min_budget = fidelity_space$get_hyperparameter(fidelity_param_id)$lower + max_budget = fidelity_space$get_hyperparameter(fidelity_param_id)$upper + ndim = length(bench$config_space$get_hyperparameter_names()) - 1L # NOTE: instance is not part of + + instances = "CIFAR10" + target = "val_accuracy" + budget = ceiling(20L * max_budget + sqrt(ndim) * max_budget * budget_factor) + on_integer_scale = TRUE + minimize = bench$config$config$y_minimize[match(target, bench$config$config$y_names)] + setup = setDT(expand.grid(scenario = scenario, instance = instances, target = target, ndim = ndim, max_budget = max_budget, budget = budget, on_integer_scale = on_integer_scale, minimize = minimize, stringsAsFactors = FALSE)) + setup +} + +get_lcbench_setup = function(budget_factor = 40L) { + scenario = "lcbench" + bench = yahpo_gym$benchmark_set$BenchmarkSet(scenario, instance = "167168") + fidelity_space = bench$get_fidelity_space() + fidelity_param_id = fidelity_space$get_hyperparameter_names()[1] + min_budget = fidelity_space$get_hyperparameter(fidelity_param_id)$lower + max_budget = fidelity_space$get_hyperparameter(fidelity_param_id)$upper + ndim = length(bench$config_space$get_hyperparameter_names()) - 2L + + instances = c("167168", "189873", "189906") + target = "val_accuracy" + budget = ceiling(20L * max_budget + sqrt(ndim) * max_budget * budget_factor) + on_integer_scale = TRUE + minimize = bench$config$config$y_minimize[match(target, bench$config$config$y_names)] + setup = setDT(expand.grid(scenario = scenario, instance = instances, target = target, ndim = ndim, max_budget = max_budget, budget = budget, on_integer_scale = on_integer_scale, minimize = minimize, stringsAsFactors = FALSE)) + setup +} + +get_rbv2_setup = function(budget_factor = 40L) { + setup = map_dtr(c("rbv2_glmnet", "rbv2_rpart", "rbv2_ranger", "rbv2_xgboost", "rbv2_super"), function(scenario) { + bench = yahpo_gym$benchmark_set$BenchmarkSet(scenario, instance = "1040") + fidelity_space = bench$get_fidelity_space() + fidelity_param_id = "trainsize" + min_budget = fidelity_space$get_hyperparameter(fidelity_param_id)$lower + max_budget = fidelity_space$get_hyperparameter(fidelity_param_id)$upper + ndim = length(bench$config_space$get_hyperparameter_names()) - 3L # repl and trainsize and instance + + instances = switch(scenario, rbv2_glmnet = c("375", "458"), rbv2_rpart = c("14", "40499"), rbv2_ranger = c("16", "42"), rbv2_xgboost = c("12", "1501", "16", "40499"), rbv2_super = c("1053", "1457", "1063", "1479", "15", "1468")) + target = "acc" + budget = ceiling(20L * max_budget + sqrt(ndim) * max_budget * budget_factor) + on_integer_scale = FALSE + minimize = bench$config$config$y_minimize[match(target, bench$config$config$y_names)] + setup = setDT(expand.grid(scenario = scenario, instance = instances, target = target, ndim = ndim, max_budget = max_budget, budget = budget, on_integer_scale = on_integer_scale, minimize = minimize, stringsAsFactors = FALSE)) + }) +} + +setup = rbind(get_nb301_setup(), get_lcbench_setup(), get_rbv2_setup()) + +setup[, id := seq_len(.N)] + +# add problems +prob_designs = map(seq_len(nrow(setup)), function(i) { + prob_id = paste0(setup[i, ]$scenario, "_", setup[i, ]$instance, "_", setup[i, ]$target) + addProblem(prob_id, data = list(scenario = setup[i, ]$scenario, instance = setup[i, ]$instance, target = setup[i, ]$target, ndim = setup[i, ]$ndim, max_budget = setup[i, ]$max_budget, budget = setup[i, ]$budget, on_integer_scale = setup[i, ]$on_integer_scale, minimize = setup[i, ]$minimize)) + setNames(list(setup[i, ]), nm = prob_id) +}) +nn = sapply(prob_designs, names) +prob_designs = unlist(prob_designs, recursive = FALSE, use.names = FALSE) +names(prob_designs) = nn + +# add jobs for optimizers +optimizers = data.table(algorithm = c("mlr3mbo", "mlrintermbo", "mlr3mbo_default", "mlr3mbo_custom", "mlr3mbo_new_rf", "mlr3mbo_new_rf_ls", "mlr3mbo_xxx_ls", "mlr3mbo_kknn_ls")) + +for (i in seq_len(nrow(optimizers))) { + algo_designs = setNames(list(optimizers[i, ]), nm = optimizers[i, ]$algorithm) + + ids = addExperiments( + prob.designs = prob_designs, + algo.designs = algo_designs, + repls = 30L + ) + addJobTags(ids, as.character(optimizers[i, ]$algorithm)) +} + +jobs = findJobs() +resources.default = list(walltime = 3600 * 12L, memory = 2048L, ntasks = 1L, ncpus = 1L, nodes = 1L, clusters = "teton", max.concurrent.jobs = 9999L) +submitJobs(jobs, resources = resources.default) + +done = findDone() +results = reduceResultsList(done, function(x, job) { + x = x$archive$data + budget_var = if (job$instance$scenario %in% c("lcbench", "nb301")) "epoch" else "trainsize" + target_var = job$instance$target + if (!job$instance$minimize) { + x[, (target_var) := - get(target_var)] + } + pars = job$pars + tmp = x[, target_var, with = FALSE] + tmp[, (budget_var) := job$instance$max_budget] + tmp[, method := pars$algo.pars$algorithm] + tmp[, scenario := pars$prob.pars$scenario] + tmp[, instance := pars$prob.pars$instance] + tmp[, repl := job$repl] + tmp[, iter := seq_len(.N)] + colnames(tmp) = c("target", "budget", "method", "scenario", "instance", "repl", "iter") + tmp +}) +results = rbindlist(results, fill = TRUE) +saveRDS(results, "results_yahpo_own.rds") + diff --git a/attic/so_config_old/submit.sh b/attic/so_config_old/submit.sh new file mode 100644 index 00000000..0069211b --- /dev/null +++ b/attic/so_config_old/submit.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +sbatch <}}\preformatted{mlr_acqfunctions$get("ttei") +acqf("ttei") +}\if{html}{\out{}} +} + +\examples{ +if (requireNamespace("mlr3learners") & + requireNamespace("DiceKriging") & + requireNamespace("rgenoud")) { + library(bbotk) + library(paradox) + library(mlr3learners) + library(data.table) + + fun = function(xs) { + list(y = xs$x ^ 2) + } + domain = ps(x = p_dbl(lower = -10, upper = 10)) + codomain = ps(y = p_dbl(tags = "minimize")) + objective = ObjectiveRFun$new(fun = fun, domain = domain, codomain = codomain) + + instance = OptimInstanceSingleCrit$new( + objective = objective, + terminator = trm("evals", n_evals = 5)) + + instance$eval_batch(data.table(x = c(-6, -5, 3, 9))) + + learner = lrn("regr.km", + covtype = "matern3_2", + optim.method = "gen", + nugget.stability = 10^-8, + control = list(trace = FALSE)) + + surrogate = srlrn(learner, archive = instance$archive) + + acq_function = acqf("ttei", + surrogate = surrogate, + toplvl_acq_optimizer = acqo(opt("random_search"), terminator = trm("evals", n_evals = 100))) + + acq_function$surrogate$update() + acq_function$update() + acq_function$eval_dt(data.table(x = c(-1, 0, 1))) +} +} +\references{ +\itemize{ +\item Qin, Chao, Klabjan, Diego, Russo, Daniel (2017). +\dQuote{Improving the Expected Improvement Algorithm.} +\emph{Advances in Neural Information Processing Systems}, \bold{30}. +} +} +\seealso{ +Other Acquisition Function: +\code{\link{AcqFunction}}, +\code{\link{mlr_acqfunctions_cb}}, +\code{\link{mlr_acqfunctions_eips}}, +\code{\link{mlr_acqfunctions_ei}}, +\code{\link{mlr_acqfunctions_mean}}, +\code{\link{mlr_acqfunctions_pi}}, +\code{\link{mlr_acqfunctions_smsego}}, +\code{\link{mlr_acqfunctions}} +} +\concept{Acquisition Function} +\section{Super classes}{ +\code{\link[bbotk:Objective]{bbotk::Objective}} -> \code{\link[mlr3mbo:AcqFunction]{mlr3mbo::AcqFunction}} -> \code{AcqFunctionTTEI} +} +\section{Public fields}{ +\if{html}{\out{
}} +\describe{ +\item{\code{y_best}}{(\code{numeric(1)})\cr +Best objective function value observed so far.} + +\item{\code{ei_argmax_mean_se}}{(\code{\link[data.table:data.table]{data.table::data.table()}})\cr +data.table::data.table() containing one row with the surrogate mean and standard error +prediction for the current argmax of the Expected Improvement.} + +\item{\code{is_ei}}{(\code{logical(1)})\cr +Whether in the next iteration simply the Expected Improvement is to be optimized or the +Expected Improvement over the Expected Improvement.} +} +\if{html}{\out{
}} +} +\section{Methods}{ +\subsection{Public methods}{ +\itemize{ +\item \href{#method-AcqFunctionTTEI-new}{\code{AcqFunctionTTEI$new()}} +\item \href{#method-AcqFunctionTTEI-update}{\code{AcqFunctionTTEI$update()}} +\item \href{#method-AcqFunctionTTEI-clone}{\code{AcqFunctionTTEI$clone()}} +} +} +\if{html}{\out{ +
Inherited methods + +
+}} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqFunctionTTEI-new}{}}} +\subsection{Method \code{new()}}{ +Creates a new instance of this \link[R6:R6Class]{R6} class. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{AcqFunctionTTEI$new(surrogate = NULL, beta = 0.5, toplvl_acq_optimizer = NULL)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{surrogate}}{(\code{NULL} | \link{SurrogateLearner}).} + +\item{\code{beta}}{(\code{numeric(1)})\cr +With probability \code{beta} the toplevel Expected Improvement acquisition function is to be +optimized and with \code{1 - beta} the Expected Improvement over the Expected Improvement. +Default is \code{0.5}.} + +\item{\code{toplvl_acq_optimizer}}{(\code{NULL} | \link{AcqOptimizer})\cr +Acquisition function optimizer for the toplevel Expected Improvement acquisition function +used to find the current argmax. +If \code{NULL} will be initialized to +\code{acqo(opt("random_search", batch_size = 1000), terminator = trm("evals", n_evals = 10000))}.} +} +\if{html}{\out{
}} +} +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqFunctionTTEI-update}{}}} +\subsection{Method \code{update()}}{ +Updates acquisition function and performs the following steps: +\itemize{ +\item sets \code{y_best} +\item updates the internal toplevel Expected Improvement acquisition function +\item samples whether simply the Expected Improvement is to be optimized or the Expected +Improvement over the Expected Improvement (depending on \code{beta}) and sets \code{is_ei} accordingly +\item if \code{is_ei = FALSE}, proceeds to optimize the toplevel Expected Improvement acquisition function to +find the current argmax and sets \code{ei_argmax_mean_se} +} +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{AcqFunctionTTEI$update()}\if{html}{\out{
}} +} + +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqFunctionTTEI-clone}{}}} +\subsection{Method \code{clone()}}{ +The objects of this class are cloneable with this method. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{AcqFunctionTTEI$clone(deep = FALSE)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{deep}}{Whether to make a deep clone.} +} +\if{html}{\out{
}} +} +} +} diff --git a/man/mlr_loop_functions_mpcl.Rd b/man/mlr_loop_functions_mpcl.Rd index ac704406..d3806599 100644 --- a/man/mlr_loop_functions_mpcl.Rd +++ b/man/mlr_loop_functions_mpcl.Rd @@ -116,7 +116,7 @@ if (requireNamespace("mlr3learners") & \references{ \itemize{ \item Ginsbourger, David, Le Riche, Rodolphe, Carraro, Laurent (2008). -\dQuote{A Multi-Points Criterion for Deterministic Parallel Global Optimization Based on Gaussian processes.} +\dQuote{A Multi-Points Criterion for Deterministic Parallel Global Optimization Based on Gaussian Processes.} \item Wang, Jialei, Clark, C. S, Liu, Eric, Frazier, I. P (2020). \dQuote{Parallel Bayesian Global Optimization of Expensive Functions.} \emph{Operations Research}, \bold{68}(6), 1850--1865. From 032a682e05c91bb79898a266ec21e73d94763c8e Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Wed, 23 Nov 2022 21:50:08 +0100 Subject: [PATCH 027/126] .. --- attic/so_config/min_max.R | 18 +++++++++--------- attic/so_config/submit.sh | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/attic/so_config/min_max.R b/attic/so_config/min_max.R index 7f396a90..bbba8ae1 100644 --- a/attic/so_config/min_max.R +++ b/attic/so_config/min_max.R @@ -6,16 +6,16 @@ reticulate::use_virtualenv("/home/lps/.local/share/virtualenvs/yahpo_gym-4ygV7gg library(reticulate) library(yahpogym) -instances = data.table(scenario = rep(paste0("rbv2_", c("aknn", "glmnet", "ranger", "rpart", "super", "svm", "xgboost")), each = 5L), - instance = c("40499", "1476", "6", "12", "41150", - "40979", "1501", "40966", "1478", "40984", - "12", "458", "1510", "1515", "307", - "1478", "40979", "12", "28", "1501", - "41164", "37", "1515", "1510", "42", - "1478", "1501", "40499", "40979", "300", - "40984", "40979", "40966", "28", "22"), +instances = data.table(scenario = rep(paste0("rbv2_", c("aknn", "glmnet", "ranger", "rpart", "super", "svm", "xgboost")), each = 4L), + instance = c("40499", "1476", "6", "12", + "40979", "1501", "40966", "1478", + "12", "458", "1510", "1515", + "1478", "40979", "12", "28", + "41164", "37", "1515", "1510", + "1478", "1501", "40499", "40979", + "40984", "40979", "40966", "28"), target = "acc", - budget = rep(c(118, 90, 134, 110, 267, 118, 170), each = 5L)) + budget = rep(c(118, 90, 134, 110, 267, 118, 170), each = 4L)) make_optim_instance = function(instance) { benchmark = BenchmarkSet$new(instance$scenario, instance = instance$instance) diff --git a/attic/so_config/submit.sh b/attic/so_config/submit.sh index 0069211b..fec39991 100644 --- a/attic/so_config/submit.sh +++ b/attic/so_config/submit.sh @@ -4,7 +4,7 @@ sbatch < Date: Sun, 27 Nov 2022 20:26:57 +0100 Subject: [PATCH 028/126] new cd version --- attic/so_config/OptimizerCoordinateDescent.R | 90 +++++ attic/so_config/cd_example.R | 31 ++ attic/so_config/coordinate_descent.R | 199 ++++++++++ attic/so_config/helpers.R | 9 + attic/so_config/run.R | 18 +- attic/so_config/run_yahpo.R | 362 +++++-------------- attic/so_config_old/coordinate_descent_old.R | 205 +++++++++++ 7 files changed, 626 insertions(+), 288 deletions(-) create mode 100644 attic/so_config/OptimizerCoordinateDescent.R create mode 100644 attic/so_config/cd_example.R create mode 100644 attic/so_config/coordinate_descent.R create mode 100644 attic/so_config/helpers.R create mode 100644 attic/so_config_old/coordinate_descent_old.R diff --git a/attic/so_config/OptimizerCoordinateDescent.R b/attic/so_config/OptimizerCoordinateDescent.R new file mode 100644 index 00000000..d8cb668c --- /dev/null +++ b/attic/so_config/OptimizerCoordinateDescent.R @@ -0,0 +1,90 @@ +OptimizerCoordinateDescent = R6Class("OptimizerCoordinateDescent", inherit = bbotk::Optimizer, + public = list( + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + initialize = function() { + param_set = ps( + n_coordinate_tryouts = p_int(lower = 1L, default = 10L, tags = "required"), + max_gen = p_int(lower = 1L, default = 10L, tags = "required"), + rds_name = p_uty(default = "cd_instance.rds", tags = "required") + ) + param_set$values = list(n_coordinate_tryouts = 10L, max_gen = 10L, rds_name = "cd_instance.rds") + + super$initialize( + id = "coordinate_descent", + param_set = param_set, + param_classes = c("ParamLgl", "ParamInt", "ParamDbl", "ParamFct"), + properties = c("single-crit", "dependencies"), + label = "Coordinate Descent", + man = "" + ) + } + ), + + private = list( + .optimize = function(inst) { + n_coordinate_tryouts = self$param_set$values$n_coordinate_tryouts + if (inst$archive$n_evals == 0L) { + xdt = generate_design_random(inst$search_space, n = 1L)$data + inst$eval_batch(xdt) + } + incumbent = inst$archive$best()[, inst$archive$cols_x, with = FALSE] + gen = 0L + repeat { + gen = gen + 1L + for (param_id in shuffle(inst$search_space$ids())) { + xdt = get_xdt_coordinate(copy(incumbent), param = inst$search_space$params[[param_id]], n_coordinate_tryouts = n_coordinate_tryouts) + # previously inactive parameters can now be active and need a value + if (inst$search_space$has_deps & param_id %in% inst$search_space$deps$on) { + deps = inst$search_space$deps[on == param_id, ] + for (i in seq_len(nrow(deps))) { + to_replace = which(map_lgl(xdt[[param_id]], function(x) deps$cond[[i]]$test(x))) + set(xdt, i = to_replace, j = deps$id[[i]], value = sample_random(inst$search_space$params[[deps$id[[i]]]], n = length(to_replace))) + } + } + xdt = Design$new(inst$search_space, data = xdt, remove_dupl = TRUE)$data # fixes potentially broken dependencies + set(xdt, j = ".gen", value = gen) + set(xdt, j = ".param", value = param_id) + inst$eval_batch(xdt) + #for (i in seq_len(nrow(xdt))) { # could also do this according to a batch_size parameter + # inst$eval_batch(xdt[i, ]) + #} + incumbent = inst$archive$best()[, inst$archive$cols_x, with = FALSE] + saveRDS(inst, file = self$param_set$values$rds_name) + } + if (gen >= self$param_set$values$max_gen) { + break + } + } + } + ) +) + +get_xdt_coordinate = function(incumbent, param, n_coordinate_tryouts) { + if (param$class %in% c("ParamDbl", "ParamInt")) { + x = runif(n = n_coordinate_tryouts, min = param$lower, max = param$upper) + if (param$class == "ParamInt") { + x = as.integer(round(x, 0L)) + } + } else { + n_coordinate_tryouts = min(n_coordinate_tryouts, param$nlevels - 1L) + x = sample(x = setdiff(param$levels, incumbent[[param$id]]), size = n_coordinate_tryouts, replace = FALSE) + } + xdt = incumbent[rep(1, n_coordinate_tryouts), ] + set(xdt, j = param$id, value = x) + xdt +} + +sample_random = function(param, n) { + if (param$class %in% c("ParamDbl", "ParamInt")) { + x = runif(n = n, min = param$lower, max = param$upper) + if (param$class == "ParamInt") { + x = as.integer(round(x, 0L)) + } + } else { + x = sample(x = param$levels, size = n, replace = TRUE) + } + x +} + diff --git a/attic/so_config/cd_example.R b/attic/so_config/cd_example.R new file mode 100644 index 00000000..6614c668 --- /dev/null +++ b/attic/so_config/cd_example.R @@ -0,0 +1,31 @@ +library(bbotk) +library(paradox) +library(R6) +library(checkmate) +library(mlr3misc) + + +source("OptimizerCoordinateDescent.R") + +domain = ps(x1 = p_dbl(lower = -1, upper = 1), + x2 = p_fct(c("a", "b", "c")), + x3 = p_int(lower = 1, upper = 3, depends = x2 == "a"), + x4 = p_lgl(), + x6 = p_dbl(lower = 0, upper = 10, depends = (x2 == "b" && x4 == TRUE)), + x7 = p_fct(c("A", "B"), depends = x2 %in% c("a", "b"))) + +objective = ObjectiveRFunDt$new( + fun = function(xdt) { + x3 = xdt$x3 + x3[is.na(x3)] = 1 + x6 = xdt$x6 + x6[is.na(x6)] = 0 + y = x3 * xdt$x1 - x6 + data.table(y = y) + }, + domain = domain, + codomain = ps(y = p_dbl(tags = "minimize"))) + +instance = OptimInstanceSingleCrit$new(objective = objective, terminator = trm("none")) +optimizer = OptimizerCoordinateDescent$new() +optimizer$optimize(instance) diff --git a/attic/so_config/coordinate_descent.R b/attic/so_config/coordinate_descent.R new file mode 100644 index 00000000..043fbb6c --- /dev/null +++ b/attic/so_config/coordinate_descent.R @@ -0,0 +1,199 @@ +#!/usr/bin/env Rscript +# chmod ug+x +library(data.table) +library(mlr3) +library(mlr3misc) +library(mlr3learners) +library(mlr3pipelines) +library(bbotk) # @localsearch +library(paradox) +library(R6) +library(checkmate) +library(mlr3mbo) # @so_config +reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) +library(reticulate) +library(yahpogym) +library("future") +library("future.batchtools") +library("future.apply") + +source("OptimizerCoordinateDescent.R") +source("LearnerRegrRangerCustom.R") +source("OptimizerChain.R") + +search_space = ps( + loop_function = p_fct(c("ego", "ego_log")), + init = p_fct(c("random", "lhs", "sobol")), + init_size_factor = p_int(lower = 1L, upper = 4L), + random_interleave = p_lgl(), + random_interleave_iter = p_fct(c("2", "4", "10"), depends = random_interleave == TRUE), + + rf_type = p_fct(c("standard", "smaclike_boot", "smaclike_no_boot", "smaclike_variance_boot")), + + acqf = p_fct(c("EI", "TTEI", "CB", "PI", "Mean")), + lambda = p_int(lower = 1, upper = 3, depends = acqf == "CB"), + acqopt = p_fct(c("RS_1000", "RS", "FS", "LS")) +) + +instances = readRDS("instances.rds") + +evaluate = function(xdt, instance) { + id = xdt$id + xdt$id = NULL + + library(data.table) + library(mlr3) + library(mlr3misc) + library(mlr3learners) + library(mlr3pipelines) + library(bbotk) + library(paradox) + library(R6) + library(checkmate) + library(mlr3mbo) # @so_config + reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) + library(reticulate) + library(yahpogym) + + source("LearnerRegrRangerCustom.R") + source("OptimizerChain.R") + + logger = lgr::get_logger("mlr3") + logger$set_threshold("warn") + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + + make_optim_instance = function(instance) { + benchmark = BenchmarkSet$new(instance$scenario, instance = instance$instance) + benchmark$subset_codomain(instance$target) + objective = benchmark$get_objective(instance$instance, multifidelity = FALSE, check_values = FALSE) + budget = instance$budget + optim_instance = OptimInstanceSingleCrit$new(objective, search_space = benchmark$get_search_space(drop_fidelity_params = TRUE), terminator = trm("evals", n_evals = budget), check_values = FALSE) + optim_instance + } + + optim_instance = make_optim_instance(instance) + + d = optim_instance$search_space$length + init_design_size = d * xdt$init_size_factor + init_design = if (xdt$init == "random") { + generate_design_random(optim_instance$search_space, n = init_design_size)$data + } else if (xdt$init == "lhs") { + generate_design_lhs(optim_instance$search_space, n = init_design_size)$data + } else if (xdt$init == "sobol") { + generate_design_sobol(optim_instance$search_space, n = init_design_size)$data + } + + optim_instance$eval_batch(init_design) + + random_interleave_iter = if(xdt$random_interleave) as.numeric(xdt$random_interleave_iter) else 0L + + learner = LearnerRegrRangerCustom$new() + learner$predict_type = "se" + learner$param_set$values$keep.inbag = TRUE + + if (xdt$rf_type == "standard") { + learner$param_set$values$se.method = "jack" + learner$param_set$values$splitrule = "variance" + learner$param_set$values$num.trees = 1000L + } else if (xdt$rf_type == "smaclike_boot") { + learner$param_set$values$se.method = "simple" + learner$param_set$values$splitrule = "extratrees" + learner$param_set$values$num.random.splits = 1L + learner$param_set$values$num.trees = 10L + learner$param_set$values$replace = TRUE + learner$param_set$values$sample.fraction = 1 + learner$param_set$values$min.node.size = 1 + learner$param_set$values$mtry.ratio = 1 + } else if (xdt$rf_type == "smaclike_no_boot") { + learner$param_set$values$se.method = "simple" + learner$param_set$values$splitrule = "extratrees" + learner$param_set$values$num.random.splits = 1L + learner$param_set$values$num.trees = 10L + learner$param_set$values$replace = FALSE + learner$param_set$values$sample.fraction = 1 + learner$param_set$values$min.node.size = 1 + learner$param_set$values$mtry.ratio = 1 + } else if (xdt$rf_type == "smaclike_variance_boot") { + learner$param_set$values$se.method = "simple" + learner$param_set$values$splitrule = "variance" + learner$param_set$values$num.trees = 10L + learner$param_set$values$replace = TRUE + learner$param_set$values$sample.fraction = 1 + learner$param_set$values$min.node.size = 1 + learner$param_set$values$mtry.ratio = 1 + } + + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor", multiplier = 3) %>>% learner)) + + acq_optimizer = if (xdt$acqopt == "RS_1000") { + AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 1000L)) + } else if (xdt$acqopt == "RS") { + AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 20000L)) + } else if (xdt$acqopt == "FS") { + n_repeats = 2L + maxit = 9L + batch_size = ceiling((20000L / n_repeats) / (1 + maxit)) # 1000L + AcqOptimizer$new(opt("focus_search", n_points = batch_size, maxit = maxit), terminator = trm("evals", n_evals = 20000L)) + } else if (xdt$acqopt == "LS") { + optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) + acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20020L)) + acq_optimizer$param_set$values$warmstart = TRUE + acq_optimizer$param_set$values$warmstart_size = 10L + acq_optimizer + } + + acq_function = if (xdt$acqf == "EI") { + AcqFunctionEI$new() + } else if (xdt$acqf == "TTEI") { + AcqFunctionTTEI$new(toplvl_acq_optimizer = acq_optimizer$clone(deep = TRUE)) + } else if (xdt$acqf == "CB") { + AcqFunctionCB$new(lambda = xdt$lambda) + } else if (xdt$acqf == "PI") { + AcqFunctionPI$new() + } else if (xdt$acqf == "Mean") { + AcqFunctionMean$new() + } + + if (xdt$loop_function == "ego") { + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + } else if (xdt$loop_function == "ego_log") { + bayesopt_ego_log(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + } + + best = optim_instance$archive$best()[[instance$target]] + ecdf_best = instance$ecdf(best) # evaluate the precomputed ecdf for the best value found; our target is effectively P(X <= best) + data.table(ecdf_best = ecdf_best, id = id, instance = paste0(instance$scenario, "_", instance$instance)) +} + +objective = ObjectiveRFunDt$new( + fun = function(xdt) { + xdt[, id := seq_len(.N)] + # FIXME: walltime can be set adaptively based on xdt + # FIXME: we could continuously model the walltime with a surrogate and set this for each xs in xdt + plan("batchtools_slurm", resources = list(walltime = 3600L * 7L, ncpus = 1L, memory = 1000L), template = "slurm_wyoming.tmpl") + res = future_mapply(FUN = evaluate, transpose_list(xdt), transpose_list(instances), SIMPLIFY = FALSE, future.seed = TRUE) # FIXME: tryCatch + res = rbindlist(res) + agg = res[, .(mean_perf = mean(ecdf_best), raw_perfs = list(ecdf_best), n_na = sum(is.na(ecdf_best))), by = .(id)] + setorderv(agg, cols = "id") + agg + }, + domain = search_space, + codomain = ps(mean_perf = p_dbl(tags = "maximize")), + check_values = FALSE +) + +cd_instance = OptimInstanceSingleCrit$new( + objective = objective, + search_space = search_space, + terminator = trm("none") # OptimizerChain currently terminates on its own +) + +optimizer = OptimizerCoordinateDescent$new() +optimizer$param_set$values$max_gen = 1L + +init = data.table(loop_function = "ego", init = "random", init_size_factor = 4L, random_interleave = FALSE, random_interleave_iter = NA_character_, rf_type = "standard", acqf = "EI", lambda = NA_integer_, acqopt = "RS") +set.seed(2906) +cd_instance$eval_batch(init) +optimizer$optimize(cd_instance) + diff --git a/attic/so_config/helpers.R b/attic/so_config/helpers.R new file mode 100644 index 00000000..a66f06b4 --- /dev/null +++ b/attic/so_config/helpers.R @@ -0,0 +1,9 @@ +make_optim_instance = function(instance) { + benchmark = BenchmarkSet$new(as.character(instance$scenario), instance = as.character(instance$instance)) + benchmark$subset_codomain(instance$target) + objective = benchmark$get_objective(as.character(instance$instance), multifidelity = FALSE, check_values = FALSE) + n_evals = as.integer(ceiling(instance$budget / instance$max_budget)) # full budget + optim_instance = OptimInstanceSingleCrit$new(objective, search_space = benchmark$get_search_space(drop_fidelity_params = TRUE), terminator = trm("evals", n_evals = n_evals), check_values = FALSE) + optim_instance +} + diff --git a/attic/so_config/run.R b/attic/so_config/run.R index 75667735..e614f2d7 100755 --- a/attic/so_config/run.R +++ b/attic/so_config/run.R @@ -17,8 +17,8 @@ library(yahpogym) library("future") library("future.apply") -source("OptimizerChain.R") source("LearnerRegrRangerCustom.R") +source("OptimizerChain.R") parser = ArgumentParser() parser$add_argument("-r", "--run", type = "integer", default = 1, help = "Id of run, should be within 1-5") @@ -85,9 +85,9 @@ evaluate = function(xdt, instance) { } optim_instance$eval_batch(init_design) - + random_interleave_iter = if(xdt$random_interleave) as.numeric(xdt$random_interleave_iter) else 0L - + learner = LearnerRegrRangerCustom$new() learner$predict_type = "se" learner$param_set$values$keep.inbag = TRUE @@ -100,7 +100,7 @@ evaluate = function(xdt, instance) { learner$param_set$values$se.method = "simple" learner$param_set$values$splitrule = "extratrees" learner$param_set$values$num.random.splits = 1L - learner$param_set$values$num.trees = 10 + learner$param_set$values$num.trees = 10L learner$param_set$values$replace = TRUE learner$param_set$values$sample.fraction = 1 learner$param_set$values$min.node.size = 1 @@ -109,7 +109,7 @@ evaluate = function(xdt, instance) { learner$param_set$values$se.method = "simple" learner$param_set$values$splitrule = "extratrees" learner$param_set$values$num.random.splits = 1L - learner$param_set$values$num.trees = 10 + learner$param_set$values$num.trees = 10L learner$param_set$values$replace = FALSE learner$param_set$values$sample.fraction = 1 learner$param_set$values$min.node.size = 1 @@ -117,7 +117,7 @@ evaluate = function(xdt, instance) { } else if (xdt$rf_type == "smaclike_variance_boot") { learner$param_set$values$se.method = "simple" learner$param_set$values$splitrule = "variance" - learner$param_set$values$num.trees = 10 + learner$param_set$values$num.trees = 10L learner$param_set$values$replace = TRUE learner$param_set$values$sample.fraction = 1 learner$param_set$values$min.node.size = 1 @@ -154,13 +154,13 @@ evaluate = function(xdt, instance) { } else if (xdt$acqf == "Mean") { AcqFunctionMean$new() } - - if (xdt$loop_function == "ego") { + + if (xdt$loop_function == "ego") { bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) } else if (xdt$loop_function == "ego_log") { bayesopt_ego_log(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) } - + best = optim_instance$archive$best()[[instance$target]] ecdf_best = instance$ecdf(best) # evaluate the precomputed ecdf for the best value found; our target is effectively P(X <= best) cat("scenario:", instance$scenario, "instance:", instance$instance, "ECDF_best:", ecdf_best, "\n") diff --git a/attic/so_config/run_yahpo.R b/attic/so_config/run_yahpo.R index d40a3217..fb600d33 100644 --- a/attic/so_config/run_yahpo.R +++ b/attic/so_config/run_yahpo.R @@ -5,7 +5,7 @@ library(mlr3learners) library(mlr3pipelines) library(mlr3misc) library(mlr3mbo) # @so_config -library(bbotk) +library(bbotk) # @localsearch library(paradox) library(R6) library(checkmate) @@ -16,22 +16,21 @@ yahpo_gym = import("yahpo_gym") packages = c("data.table", "mlr3", "mlr3learners", "mlr3pipelines", "mlr3misc", "mlr3mbo", "bbotk", "paradox", "mlrintermbo", "R6", "checkmate") -RhpcBLASctl::blas_set_num_threads(1L) -RhpcBLASctl::omp_set_num_threads(1L) +#RhpcBLASctl::blas_set_num_threads(1L) +#RhpcBLASctl::omp_set_num_threads(1L) root = here::here() experiments_dir = file.path(root) -source_files = map_chr(c("helpers.R", "OptimizerChain.R"), function(x) file.path(experiments_dir, x)) +source_files = map_chr(c("helpers.R", "LearnerRegrRangerCustom.R", "OptimizerChain.R"), function(x) file.path(experiments_dir, x)) for (sf in source_files) { source(sf) } -reg = makeExperimentRegistry(file.dir = "/gscratch/lschnei8/registry_mlr3mbo_so_config", packages = packages, source = source_files) +reg = makeExperimentRegistry(file.dir = "/gscratch/lschnei8/registry_mlr3mbo_so_config_new", packages = packages, source = source_files) #reg = makeExperimentRegistry(file.dir = NA, conf.file = NA, packages = packages, source = source_files) # interactive session saveRegistry(reg) -# FIXME: also compare jack vs. infjack? mlr3mbo_wrapper = function(job, data, instance, ...) { reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) library(yahpogym) @@ -41,48 +40,95 @@ mlr3mbo_wrapper = function(job, data, instance, ...) { optim_instance = make_optim_instance(instance) - xdt = list(init = "lhs", init_size_factor = 4L, random_interleave = FALSE, num.trees = 250L, splitrule = "extratrees", num.random.splits = 8, acqf = "CB", lambda = 2.8, acqopt_iter_factor = 6L, acqopt = "FS", fs_behavior = "global") + xdt = data.table(loop_function = "ego_log", init = "random", init_size_factor = 4L, random_interleave = FALSE, random_interleave_iter = NA_integer_, rf_type = "smaclike_boot", acqf = "EI", lambda = NA_real_, acqopt = "FS") d = optim_instance$search_space$length init_design_size = d * xdt$init_size_factor - init_design = if (xdt$init == "random") generate_design_random(optim_instance$search_space, n = init_design_size)$data else if (xdt$init == "lhs") generate_design_lhs(optim_instance$search_space, n = init_design_size)$data + init_design = if (xdt$init == "random") { + generate_design_random(optim_instance$search_space, n = init_design_size)$data + } else if (xdt$init == "lhs") { + generate_design_lhs(optim_instance$search_space, n = init_design_size)$data + } else if (xdt$init == "sobol") { + generate_design_sobol(optim_instance$search_space, n = init_design_size)$data + } + optim_instance$eval_batch(init_design) - - random_interleave_iter = if(xdt$random_interleave) xdt$random_interleave_iter else 0L - - learner = if (xdt$splitrule == "extratrees") { - lrn("regr.ranger", num.trees = xdt$num.trees, keep.inbag = TRUE, splitrule = xdt$splitrule, num.random.splits = xdt$num.random.splits) - } else if (xdt$splitrule == "variance") { - lrn("regr.ranger", num.trees = xdt$num.trees, keep.inbag = TRUE, splitrule = xdt$splitrule) + + random_interleave_iter = if(xdt$random_interleave) as.numeric(xdt$random_interleave_iter) else 0L + + learner = LearnerRegrRangerCustom$new() + learner$predict_type = "se" + learner$param_set$values$keep.inbag = TRUE + + if (xdt$rf_type == "standard") { + learner$param_set$values$se.method = "jack" + learner$param_set$values$splitrule = "variance" + learner$param_set$values$num.trees = 1000L + } else if (xdt$rf_type == "smaclike_boot") { + learner$param_set$values$se.method = "simple" + learner$param_set$values$splitrule = "extratrees" + learner$param_set$values$num.random.splits = 1L + learner$param_set$values$num.trees = 10L + learner$param_set$values$replace = TRUE + learner$param_set$values$sample.fraction = 1 + learner$param_set$values$min.node.size = 1 + learner$param_set$values$mtry.ratio = 1 + } else if (xdt$rf_type == "smaclike_no_boot") { + learner$param_set$values$se.method = "simple" + learner$param_set$values$splitrule = "extratrees" + learner$param_set$values$num.random.splits = 1L + learner$param_set$values$num.trees = 10L + learner$param_set$values$replace = FALSE + learner$param_set$values$sample.fraction = 1 + learner$param_set$values$min.node.size = 1 + learner$param_set$values$mtry.ratio = 1 + } else if (xdt$rf_type == "smaclike_variance_boot") { + learner$param_set$values$se.method = "simple" + learner$param_set$values$splitrule = "variance" + learner$param_set$values$num.trees = 10L + learner$param_set$values$replace = TRUE + learner$param_set$values$sample.fraction = 1 + learner$param_set$values$min.node.size = 1 + learner$param_set$values$mtry.ratio = 1 + } + + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor", multiplier = 3) %>>% learner)) + + acq_optimizer = if (xdt$acqopt == "RS_1000") { + AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 1000L)) + } else if (xdt$acqopt == "RS") { + AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 20000L)) + } else if (xdt$acqopt == "FS") { + n_repeats = 2L + maxit = 9L + batch_size = ceiling((20000L / n_repeats) / (1 + maxit)) # 1000L + AcqOptimizer$new(opt("focus_search", n_points = batch_size, maxit = maxit), terminator = trm("evals", n_evals = 20000L)) + } else if (xdt$acqopt == "LS") { + optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) + acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20020L)) + acq_optimizer$param_set$values$warmstart = TRUE + acq_optimizer$param_set$values$warmstart_size = 10L + acq_optimizer } - surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% learner)) acq_function = if (xdt$acqf == "EI") { AcqFunctionEI$new() + } else if (xdt$acqf == "TTEI") { + AcqFunctionTTEI$new(toplvl_acq_optimizer = acq_optimizer$clone(deep = TRUE)) } else if (xdt$acqf == "CB") { AcqFunctionCB$new(lambda = xdt$lambda) } else if (xdt$acqf == "PI") { AcqFunctionPI$new() + } else if (xdt$acqf == "Mean") { + AcqFunctionMean$new() } - - acq_budget = 1000 * xdt$acqopt_iter_factor - - acq_optimizer = if (xdt$acqopt == "RS") { - AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = acq_budget)) - } else if (xdt$acqopt == "FS") { - if (xdt$fs_behavior == "global") { - n_repeats = 10L - maxit = 2L - batch_size = ceiling((acq_budget / n_repeats) / (1 + maxit)) - } else if (xdt$fs_behavior == "local") { - n_repeats = 2L - maxit = 10L - batch_size = ceiling((acq_budget / n_repeats) / (1 + maxit)) - } - AcqOptimizer$new(opt("focus_search", n_points = batch_size, maxit = maxit), terminator = trm("evals", n_evals = acq_budget)) + + if (xdt$loop_function == "ego") { + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + } else if (xdt$loop_function == "ego_log") { + bayesopt_ego_log(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) } - - bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + optim_instance } @@ -107,252 +153,10 @@ mlrintermbo_wrapper = function(job, data, instance, ...) { optim_instance } -mlr3mbo_default_wrapper = function(job, data, instance, ...) { - reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) - library(yahpogym) - logger = lgr::get_logger("bbotk") - logger$set_threshold("warn") - future::plan("sequential") - - optim_instance = make_optim_instance(instance) - - xdt = list(init = "random", init_size_factor = 4L, random_interleave = FALSE, num.trees = 500L, splitrule = "variance", num.random.splits = NA_integer_, acqf = "EI", lambda = NA_real_, acqopt_iter_factor = 10L, acqopt = "RS", fs_behavior = NA_character_) - - d = optim_instance$search_space$length - init_design_size = d * xdt$init_size_factor - init_design = if (xdt$init == "random") generate_design_random(optim_instance$search_space, n = init_design_size)$data else if (xdt$init == "lhs") generate_design_lhs(optim_instance$search_space, n = init_design_size)$data - optim_instance$eval_batch(init_design) - - random_interleave_iter = if(xdt$random_interleave) xdt$random_interleave_iter else 0L - - learner = if (xdt$splitrule == "extratrees") { - lrn("regr.ranger", num.trees = xdt$num.trees, keep.inbag = TRUE, splitrule = xdt$splitrule, num.random.splits = xdt$num.random.splits) - } else if (xdt$splitrule == "variance") { - lrn("regr.ranger", num.trees = xdt$num.trees, keep.inbag = TRUE, splitrule = xdt$splitrule) - } - surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% learner)) - - acq_function = if (xdt$acqf == "EI") { - AcqFunctionEI$new() - } else if (xdt$acqf == "CB") { - AcqFunctionCB$new(lambda = xdt$lambda) - } else if (xdt$acqf == "PI") { - AcqFunctionPI$new() - } - - acq_budget = 1000 * xdt$acqopt_iter_factor - - acq_optimizer = if (xdt$acqopt == "RS") { - AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = acq_budget)) - } else if (xdt$acqopt == "FS") { - if (xdt$fs_behavior == "global") { - n_repeats = 10L - maxit = 2L - batch_size = ceiling((acq_budget / n_repeats) / (1 + maxit)) - } else if (xdt$fs_behavior == "local") { - n_repeats = 2L - maxit = 10L - batch_size = ceiling((acq_budget / n_repeats) / (1 + maxit)) - } - AcqOptimizer$new(opt("focus_search", n_points = batch_size, maxit = maxit), terminator = trm("evals", n_evals = acq_budget)) - } - - bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) - optim_instance -} - -mlr3mbo_wrapper_custom = function(job, data, instance, ...) { - reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) - library(yahpogym) - logger = lgr::get_logger("bbotk") - logger$set_threshold("warn") - future::plan("sequential") - - optim_instance = make_optim_instance(instance) - - xdt = list(init = "lhs", init_size_factor = 1L, random_interleave = FALSE, num.trees = 250L, splitrule = "extratrees", num.random.splits = 10, acqf = "CB", lambda = 3, acqopt_iter_factor = 30L, acqopt = "FS", fs_behavior = "global") - - d = optim_instance$search_space$length - init_design_size = d * xdt$init_size_factor - init_design = if (xdt$init == "random") generate_design_random(optim_instance$search_space, n = init_design_size)$data else if (xdt$init == "lhs") generate_design_lhs(optim_instance$search_space, n = init_design_size)$data - optim_instance$eval_batch(init_design) - - random_interleave_iter = if(xdt$random_interleave) xdt$random_interleave_iter else 0L - - learner = if (xdt$splitrule == "extratrees") { - lrn("regr.ranger", num.trees = xdt$num.trees, keep.inbag = TRUE, splitrule = xdt$splitrule, num.random.splits = xdt$num.random.splits) - } else if (xdt$splitrule == "variance") { - lrn("regr.ranger", num.trees = xdt$num.trees, keep.inbag = TRUE, splitrule = xdt$splitrule) - } - surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% learner)) - - acq_function = if (xdt$acqf == "EI") { - AcqFunctionEI$new() - } else if (xdt$acqf == "CB") { - AcqFunctionCB$new(lambda = xdt$lambda) - } else if (xdt$acqf == "PI") { - AcqFunctionPI$new() - } - - acq_budget = 1000 * xdt$acqopt_iter_factor - - acq_optimizer = if (xdt$acqopt == "RS") { - AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = acq_budget)) - } else if (xdt$acqopt == "FS") { - if (xdt$fs_behavior == "global") { - n_repeats = 5L - maxit = 5L - batch_size = ceiling((acq_budget / n_repeats) / (1 + maxit)) - } else if (xdt$fs_behavior == "local") { - n_repeats = 2L - maxit = 10L - batch_size = ceiling((acq_budget / n_repeats) / (1 + maxit)) - } - AcqOptimizer$new(opt("focus_search", n_points = batch_size, maxit = maxit), terminator = trm("evals", n_evals = acq_budget)) - } - - bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) - optim_instance -} - -mlr3mbo_wrapper_new_rf = function(job, data, instance, ...) { - reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) - library(yahpogym) - logger = lgr::get_logger("bbotk") - logger$set_threshold("warn") - future::plan("sequential") - - optim_instance = make_optim_instance(instance) - - d = optim_instance$search_space$length - init_design_size = 4L * d - init_design = generate_design_lhs(optim_instance$search_space, n = init_design_size)$data - optim_instance$eval_batch(init_design) - - random_interleave_iter = 0L - - learner = lrn("regr.ranger_custom") - surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% learner)) - - acq_function = AcqFunctionEI$new() - - acq_budget = 20000L - - acq_optimizer = { - n_repeats = 2L - maxit = 9L - batch_size = ceiling((acq_budget / n_repeats) / (1 + maxit)) - AcqOptimizer$new(opt("focus_search", n_points = batch_size, maxit = maxit), terminator = trm("evals", n_evals = acq_budget)) - } - - bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) - optim_instance -} - -mlr3mbo_wrapper_new_rf_ls = function(job, data, instance, ...) { - reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) - library(yahpogym) - logger = lgr::get_logger("bbotk") - logger$set_threshold("warn") - future::plan("sequential") - - optim_instance = make_optim_instance(instance) - - d = optim_instance$search_space$length - init_design_size = 4L * d - init_design = generate_design_lhs(optim_instance$search_space, n = init_design_size)$data - optim_instance$eval_batch(init_design) - - random_interleave_iter = 0L - - learner = lrn("regr.ranger_custom") - surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% learner)) - - acq_function = AcqFunctionEI$new() - - optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) - acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20010L)) - - acq_optimizer$param_set$values$warmstart = TRUE - acq_optimizer$param_set$values$warmstart_size = "all" - - bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) - optim_instance -} - -mlr3mbo_wrapper_xxx_ls = function(job, data, instance, ...) { - reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) - library(yahpogym) - logger = lgr::get_logger("bbotk") - logger$set_threshold("warn") - future::plan("sequential") - - optim_instance = make_optim_instance(instance) - - d = optim_instance$search_space$length - init_design_size = 4L * d - init_design = generate_design_sobol(optim_instance$search_space, n = init_design_size)$data - optim_instance$eval_batch(init_design) - - random_interleave_iter = 4L - - learner = lrn("regr.ranger_custom") - surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor", multiplier = 2) %>>% learner)) - - acq_function = AcqFunctionEI$new() - - optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) - acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20010L)) - - acq_optimizer$param_set$values$warmstart = TRUE - acq_optimizer$param_set$values$warmstart_size = "all" - - bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) - optim_instance -} - -mlr3mbo_wrapper_kknn_ls = function(job, data, instance, ...) { - reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) - library(yahpogym) - logger = lgr::get_logger("bbotk") - logger$set_threshold("warn") - future::plan("sequential") - - optim_instance = make_optim_instance(instance) - - d = optim_instance$search_space$length - init_design_size = 4L * d - init_design = generate_design_sobol(optim_instance$search_space, n = init_design_size)$data - optim_instance$eval_batch(init_design) - - random_interleave_iter = 4L - - learner = lrn("regr.kknn") - learner = as_learner(po("imputeoor", multiplier = 2) %>>% po("fixfactors") %>>% po("imputesample") %>>% learner) - learner$predict_types = "response" - surrogate = SurrogateLearner$new(learner) - - acq_function = AcqFunctionMean$new() - - optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) - acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20010L)) - - acq_optimizer$param_set$values$warmstart = TRUE - acq_optimizer$param_set$values$warmstart_size = "all" - - bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) - optim_instance -} # add algorithms addAlgorithm("mlr3mbo", fun = mlr3mbo_wrapper) addAlgorithm("mlrintermbo", fun = mlrintermbo_wrapper) -addAlgorithm("mlr3mbo_default", fun = mlr3mbo_default_wrapper) -addAlgorithm("mlr3mbo_custom", fun = mlr3mbo_wrapper_custom) -addAlgorithm("mlr3mbo_new_rf", fun = mlr3mbo_wrapper_new_rf) -addAlgorithm("mlr3mbo_new_rf_ls", fun = mlr3mbo_wrapper_new_rf_ls) -addAlgorithm("mlr3mbo_xxx_ls", fun = mlr3mbo_wrapper_xxx_ls) -addAlgorithm("mlr3mbo_kknn_ls", fun = mlr3mbo_wrapper_kknn_ls) # setup scenarios and instances get_nb301_setup = function(budget_factor = 40L) { @@ -424,7 +228,7 @@ prob_designs = unlist(prob_designs, recursive = FALSE, use.names = FALSE) names(prob_designs) = nn # add jobs for optimizers -optimizers = data.table(algorithm = c("mlr3mbo", "mlrintermbo", "mlr3mbo_default", "mlr3mbo_custom", "mlr3mbo_new_rf", "mlr3mbo_new_rf_ls", "mlr3mbo_xxx_ls", "mlr3mbo_kknn_ls")) +optimizers = data.table(algorithm = c("mlr3mbo", "mlrintermbo")) for (i in seq_len(nrow(optimizers))) { algo_designs = setNames(list(optimizers[i, ]), nm = optimizers[i, ]$algorithm) @@ -437,8 +241,8 @@ for (i in seq_len(nrow(optimizers))) { addJobTags(ids, as.character(optimizers[i, ]$algorithm)) } -jobs = findJobs() -resources.default = list(walltime = 3600 * 12L, memory = 2048L, ntasks = 1L, ncpus = 1L, nodes = 1L, clusters = "teton", max.concurrent.jobs = 9999L) +jobs = getJobTable() +resources.default = list(walltime = 3600 * 5L, memory = 1024L, ntasks = 1L, ncpus = 1L, nodes = 1L, clusters = "teton", max.concurrent.jobs = 9999L) submitJobs(jobs, resources = resources.default) done = findDone() diff --git a/attic/so_config_old/coordinate_descent_old.R b/attic/so_config_old/coordinate_descent_old.R new file mode 100644 index 00000000..a9d212e4 --- /dev/null +++ b/attic/so_config_old/coordinate_descent_old.R @@ -0,0 +1,205 @@ +#!/usr/bin/env Rscript +# chmod ug+x +library(data.table) +library(mlr3) +library(mlr3misc) +library(mlr3learners) +library(mlr3pipelines) +library(bbotk) +library(paradox) +library(R6) +library(checkmate) +library(mlr3mbo) # @so_config +reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) +library(reticulate) +library(yahpogym) +library("future") +library("future.batchtools") +library("future.apply") + +source("OptimizerCoordinateDescent.R") +source("LearnerRegrRangerCustom.R") + +search_space = ps( + loop_function = p_fct(c("ego", "ego_log")), + init = p_fct(c("random", "lhs", "sobol")), + init_size_factor = p_int(lower = 1L, upper = 4L), + random_interleave = p_lgl(), + random_interleave_iter = p_fct(c("2", "4", "10"), depends = random_interleave == TRUE), + + rf_type = p_fct(c("standard", "smaclike_boot", "smaclike_no_boot", "smaclike_variance_boot")), + + acqf = p_fct(c("EI", "TTEI", "CB", "PI", "Mean")), + lambda = p_int(lower = 1, upper = 3, depends = acqf == "CB"), + acqopt = p_fct(c("RS_1000", "RS", "FS", "LS")) +) + +instances = readRDS("instances.rds") + +evaluate = function(xdt, instance) { + library(data.table) + library(mlr3) + library(mlr3misc) + library(mlr3learners) + library(mlr3pipelines) + library(bbotk) + library(paradox) + library(R6) + library(checkmate) + library(mlr3mbo) # @so_config + reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) + library(reticulate) + library(yahpogym) + + source("LearnerRegrRangerCustom.R") + + logger = lgr::get_logger("mlr3") + logger$set_threshold("warn") + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + + make_optim_instance = function(instance) { + benchmark = BenchmarkSet$new(instance$scenario, instance = instance$instance) + benchmark$subset_codomain(instance$target) + objective = benchmark$get_objective(instance$instance, multifidelity = FALSE, check_values = FALSE) + budget = instance$budget + optim_instance = OptimInstanceSingleCrit$new(objective, search_space = benchmark$get_search_space(drop_fidelity_params = TRUE), terminator = trm("evals", n_evals = budget), check_values = FALSE) + optim_instance + } + + optim_instance = make_optim_instance(instance) + + d = optim_instance$search_space$length + init_design_size = d * xdt$init_size_factor + init_design = if (xdt$init == "random") { + generate_design_random(optim_instance$search_space, n = init_design_size)$data + } else if (xdt$init == "lhs") { + generate_design_lhs(optim_instance$search_space, n = init_design_size)$data + } else if (xdt$init == "sobol") { + generate_design_sobol(optim_instance$search_space, n = init_design_size)$data + } + + optim_instance$eval_batch(init_design) + + random_interleave_iter = if(xdt$random_interleave) as.numeric(xdt$random_interleave_iter) else 0L + + learner = LearnerRegrRangerCustom$new() + learner$predict_type = "se" + learner$param_set$values$keep.inbag = TRUE + + if (xdt$rf_type == "standard") { + learner$param_set$values$se.method = "jack" + learner$param_set$values$splitrule = "variance" + learner$param_set$values$num.trees = 1000L + } else if (xdt$rf_type == "smaclike_boot") { + learner$param_set$values$se.method = "simple" + learner$param_set$values$splitrule = "extratrees" + learner$param_set$values$num.random.splits = 1L + learner$param_set$values$num.trees = 10 + learner$param_set$values$replace = TRUE + learner$param_set$values$sample.fraction = 1 + learner$param_set$values$min.node.size = 1 + learner$param_set$values$mtry.ratio = 1 + } else if (xdt$rf_type == "smaclike_no_boot") { + learner$param_set$values$se.method = "simple" + learner$param_set$values$splitrule = "extratrees" + learner$param_set$values$num.random.splits = 1L + learner$param_set$values$num.trees = 10 + learner$param_set$values$replace = FALSE + learner$param_set$values$sample.fraction = 1 + learner$param_set$values$min.node.size = 1 + learner$param_set$values$mtry.ratio = 1 + } else if (xdt$rf_type == "smaclike_variance_boot") { + learner$param_set$values$se.method = "simple" + learner$param_set$values$splitrule = "variance" + learner$param_set$values$num.trees = 10 + learner$param_set$values$replace = TRUE + learner$param_set$values$sample.fraction = 1 + learner$param_set$values$min.node.size = 1 + learner$param_set$values$mtry.ratio = 1 + } + + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor", multiplier = 3) %>>% learner)) + + acq_optimizer = if (xdt$acqopt == "RS_1000") { + AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 1000L)) + } else if (xdt$acqopt == "RS") { + AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 20000L)) + } else if (xdt$acqopt == "FS") { + n_repeats = 2L + maxit = 9L + batch_size = ceiling((20000L / n_repeats) / (1 + maxit)) # 1000L + AcqOptimizer$new(opt("focus_search", n_points = batch_size, maxit = maxit), terminator = trm("evals", n_evals = 20000L)) + } else if (xdt$acqopt == "LS") { + optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) + acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20020L)) + acq_optimizer$param_set$values$warmstart = TRUE + acq_optimizer$param_set$values$warmstart_size = 10L + acq_optimizer + } + + acq_function = if (xdt$acqf == "EI") { + AcqFunctionEI$new() + } else if (xdt$acqf == "TTEI") { + AcqFunctionTTEI$new(toplvl_acq_optimizer = acq_optimizer$clone(deep = TRUE)) + } else if (xdt$acqf == "CB") { + AcqFunctionCB$new(lambda = xdt$lambda) + } else if (xdt$acqf == "PI") { + AcqFunctionPI$new() + } else if (xdt$acqf == "Mean") { + AcqFunctionMean$new() + } + + if (xdt$loop_function == "ego") { + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + } else if (xdt$loop_function == "ego_log") { + bayesopt_ego_log(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + } + + best = optim_instance$archive$best()[[instance$target]] + ecdf_best = instance$ecdf(best) # evaluate the precomputed ecdf for the best value found; our target is effectively P(X <= best) + cat("scenario:", instance$scenario, "instance:", instance$instance, "ECDF_best:", ecdf_best, "\n") + ecdf_best +} + +evaluate_all = function(xs, instances) { + # evaluate_all will be carried out after future.batchtools scheduled a job to evaluate xs on all instances + library(future) + library(future.apply) + library(future.batchtools) + plan("batchtools_slurm", resources = list(walltime = 3600L * 7L, ncpus = 28L, memory = 1000L), template = "slurm_wyoming.tmpl") + tmp = future_lapply(transpose_list(instances), function(instance) { + res_instance = evaluate(xs, instance) + #res_instance = tryCatch(evaluate(xs, instance), error = function(error_condition) 0) + }, future.seed = TRUE) + data.table(mean_perf = mean(unlist(tmp), na.rm = TRUE), raw_perfs = list(tmp), n_na = sum(is.na(unlist(tmp)))) +} + +objective = ObjectiveRFunDt$new( + fun = function(xdt) { + library(future) + library(future.apply) + library(future.batchtools) + # FIXME: walltime can be set adaptively based on xdt + # FIXME: we could continuously model the walltime with a surrogate and set this for each xs in xdt + plan("batchtools_slurm", resources = list(walltime = 3600L * 7L, ncpus = 1L, memory = 1000L), template = "slurm_wyoming.tmpl") + res = future_lapply(transpose_list(xdt), function(xs) { + evaluate_all(xs, instances = instances) + }, future.seed = TRUE) + rbindlist(res) + }, + domain = search_space, + codomain = ps(mean_perf = p_dbl(tags = "maximize")), + check_values = FALSE +) + +cd_instance = OptimInstanceSingleCrit$new( + objective = objective, + search_space = search_space, + terminator = trm("none") # OptimizerChain currently terminates on its own +) + +optimizer = OptimizerCoordinateDescent$new() +optimizer$param_set$values$max_gen = 1L + +optimizer$optimize(cd_instance) From d0df4f5fc2972e77a74045092ba6b345dde0e772 Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Wed, 30 Nov 2022 17:07:04 +0100 Subject: [PATCH 029/126] some so config try-outs --- attic/so_config/DNGO.R | 49 +++++++++++ attic/so_config/LearnerRegrKKNNEnsemble.R | 41 +++++++++ attic/so_config/OptimizerCoordinateDescent.R | 37 ++++---- attic/so_config/analyze_cd.R | 7 ++ attic/so_config/analyze_yahpo.R | 91 ++++++++++++++++++++ attic/so_config/cd_example.R | 2 +- attic/so_config/coordinate_descent.R | 13 ++- attic/so_config/min_max.R | 45 +++++++++- attic/so_config/run.R | 8 +- attic/so_config/run_yahpo.R | 10 +-- 10 files changed, 267 insertions(+), 36 deletions(-) create mode 100644 attic/so_config/DNGO.R create mode 100644 attic/so_config/LearnerRegrKKNNEnsemble.R create mode 100644 attic/so_config/analyze_cd.R create mode 100644 attic/so_config/analyze_yahpo.R diff --git a/attic/so_config/DNGO.R b/attic/so_config/DNGO.R new file mode 100644 index 00000000..47ff4f0b --- /dev/null +++ b/attic/so_config/DNGO.R @@ -0,0 +1,49 @@ +library(torch) + +x = c(2.7, 4, 4.5, 4.6, 7.4) +f = function(x) sin(x) + sin((10 / 3) * x) +y = f(x) + +x = torch_tensor(as.matrix(x), dtype = torch_float()) +y = torch_tensor(as.matrix(y), dtype = torch_float()) + +input_dim = 1 +n_input = 5 + +model = nn_sequential( + nn_linear(input_dim, 50), + nn_tanh(), + + nn_linear(50, 50), + nn_tanh(), + + nn_linear(50, 50), + nn_tanh(), + + nn_linear(50, 1) +) + +criterion = nn_mse_loss() + +optimizer = optim_adam(model$parameters, lr = 0.01) + +epochs = 200 + +for (i in seq_len(epochs)) { + optimizer$zero_grad() + y_pred = model(x) + loss = criterion(y_pred, y) + loss$backward() + optimizer$step() +} + +phi = as.matrix(model[["5"]]$forward(model[["4"]]$forward(model[["3"]]$forward(model[["2"]]$forward(model[["1"]]$forward(model[["0"]]$forward(x))))))) + +alpha = 1 +beta = 1 +y_ = as.matrix(y) # FIXME: - nu(x) (9) + +K = beta * t(phi) %*% phi + diag(alpha, nrow = 50L, ncol = 50L) +m = beta * solve(K) %*% t(phi) %*% y_ + +mu = t(m) %*% diff --git a/attic/so_config/LearnerRegrKKNNEnsemble.R b/attic/so_config/LearnerRegrKKNNEnsemble.R new file mode 100644 index 00000000..83231edf --- /dev/null +++ b/attic/so_config/LearnerRegrKKNNEnsemble.R @@ -0,0 +1,41 @@ +LearnerRegrEnsemble = R6Class("LearnerRegrEnsemble", + inherit = LearnerRegr, + public = list( + learners = NULL, + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + initialize = function(learners) { + self$learners = assert_learners(learners) # i.e., kknn + + ps = ps( + ) + + super$initialize( + id = "regr.ensemble", + param_set = ps, + feature_types = c("logical", "integer", "numeric", "factor", "ordered"), + packages = "mlr3learners", + man = "" + ) + } + ), + + private = list( + .train = function(task) { + for (i in seq_along(self$learners)) { + self$learners[[i]]$predict_type = "response" + self$learners[[i]]$train(task) + } + list() + }, + + .predict = function(task) { + predictions = map_dtr(self$learners, function(learner) { + as.data.table(learner$predict(task)) + }) + predictions = predictions[, .(mean = mean(response), se = sd(response)), by = .(row_ids)] + list(response = predictions$response, se = predictions$se) + } + ) +) diff --git a/attic/so_config/OptimizerCoordinateDescent.R b/attic/so_config/OptimizerCoordinateDescent.R index d8cb668c..2e58dd62 100644 --- a/attic/so_config/OptimizerCoordinateDescent.R +++ b/attic/so_config/OptimizerCoordinateDescent.R @@ -34,24 +34,29 @@ OptimizerCoordinateDescent = R6Class("OptimizerCoordinateDescent", inherit = bbo repeat { gen = gen + 1L for (param_id in shuffle(inst$search_space$ids())) { - xdt = get_xdt_coordinate(copy(incumbent), param = inst$search_space$params[[param_id]], n_coordinate_tryouts = n_coordinate_tryouts) - # previously inactive parameters can now be active and need a value - if (inst$search_space$has_deps & param_id %in% inst$search_space$deps$on) { - deps = inst$search_space$deps[on == param_id, ] - for (i in seq_len(nrow(deps))) { - to_replace = which(map_lgl(xdt[[param_id]], function(x) deps$cond[[i]]$test(x))) - set(xdt, i = to_replace, j = deps$id[[i]], value = sample_random(inst$search_space$params[[deps$id[[i]]]], n = length(to_replace))) + if (!is.na(incumbent[[param_id]])) { + # we first also reevaluate the incumbent to see how noisy the evaluations are + inst$eval_batch(incumbent) + + xdt = get_xdt_coordinate(copy(incumbent), param = inst$search_space$params[[param_id]], n_coordinate_tryouts = n_coordinate_tryouts) + # previously inactive parameters can now be active and need a value + if (inst$search_space$has_deps & param_id %in% inst$search_space$deps$on) { + deps = inst$search_space$deps[on == param_id, ] + for (i in seq_len(nrow(deps))) { + to_replace = which(map_lgl(xdt[[param_id]], function(x) deps$cond[[i]]$test(x))) + set(xdt, i = to_replace, j = deps$id[[i]], value = sample_random(inst$search_space$params[[deps$id[[i]]]], n = length(to_replace))) + } } + xdt = Design$new(inst$search_space, data = xdt, remove_dupl = TRUE)$data # fixes potentially broken dependencies + set(xdt, j = ".gen", value = gen) + set(xdt, j = ".param", value = param_id) + inst$eval_batch(xdt) + #for (i in seq_len(nrow(xdt))) { # could also do this according to a batch_size parameter + # inst$eval_batch(xdt[i, ]) + #} + incumbent = inst$archive$best()[, inst$archive$cols_x, with = FALSE] + saveRDS(inst, file = self$param_set$values$rds_name) } - xdt = Design$new(inst$search_space, data = xdt, remove_dupl = TRUE)$data # fixes potentially broken dependencies - set(xdt, j = ".gen", value = gen) - set(xdt, j = ".param", value = param_id) - inst$eval_batch(xdt) - #for (i in seq_len(nrow(xdt))) { # could also do this according to a batch_size parameter - # inst$eval_batch(xdt[i, ]) - #} - incumbent = inst$archive$best()[, inst$archive$cols_x, with = FALSE] - saveRDS(inst, file = self$param_set$values$rds_name) } if (gen >= self$param_set$values$max_gen) { break diff --git a/attic/so_config/analyze_cd.R b/attic/so_config/analyze_cd.R new file mode 100644 index 00000000..37dd2307 --- /dev/null +++ b/attic/so_config/analyze_cd.R @@ -0,0 +1,7 @@ +library(data.table) +library(ggplot2) +library(pammtools) +library(mlr3misc) + +dat = readRDS("cd_instance.rds") + diff --git a/attic/so_config/analyze_yahpo.R b/attic/so_config/analyze_yahpo.R new file mode 100644 index 00000000..f2f14a6e --- /dev/null +++ b/attic/so_config/analyze_yahpo.R @@ -0,0 +1,91 @@ +library(data.table) +library(ggplot2) +library(pammtools) +library(mlr3misc) + +dat = rbind(readRDS("results_yahpo.rds"), readRDS("results_yahpo_own.rds"))[method != c("mlrintermbo")] +#dat = dat[scenario %nin% c("nb301", "rbv2_super")] +dat[, cumbudget := cumsum(budget), by = .(method, scenario, instance, repl)] +dat[, cumbudget_scaled := cumbudget / max(cumbudget), by = .(method, scenario, instance, repl)] +dat[, normalized_regret := (target - min(target)) / (max(target) - min(target)), by = .(scenario, instance)] +dat[, incumbent := cummin(normalized_regret), by = .(method, scenario, instance, repl)] + +get_incumbent_cumbudget = function(incumbent, cumbudget_scaled) { + budgets = seq(0, 1, length.out = 101) + map_dbl(budgets, function(budget) { + ind = which(cumbudget_scaled <= budget) + if (length(ind) == 0L) { + max(incumbent) + } else { + min(incumbent[ind]) + } + }) +} + +dat_budget = dat[, .(incumbent_budget = get_incumbent_cumbudget(incumbent, cumbudget_scaled), cumbudget_scaled = seq(0, 1, length.out = 101)), by = .(method, scenario, instance, repl)] + +agg_budget = dat_budget[, .(mean = mean(incumbent_budget), se = sd(incumbent_budget) / sqrt(.N)), by = .(cumbudget_scaled, method, scenario, instance)] +#agg_budget[, method := factor(method, levels = c("random", "smac4hpo", "hb", "bohb", "dehb", "smac4mf", "optuna", "mlr3mbo", "mlr3mbo_default"), labels = c("Random", "SMAC", "HB", "BOHB", "DEHB", "SMAC-HB", "optuna", "mlr3mbo", "mlr3mbo_default"))] + +g = ggplot(aes(x = cumbudget_scaled, y = mean, colour = method, fill = method), data = agg_budget[cumbudget_scaled > 0.10]) + + scale_y_log10() + + geom_step() + + geom_stepribbon(aes(min = mean - se, max = mean + se), colour = NA, alpha = 0.3) + + labs(x = "Fraction of Budget Used", y = "Mean Normalized Regret", colour = "Optimizer", fill = "Optimizer") + + facet_wrap(~ scenario + instance, scales = "free", ncol = 5) + + theme_minimal() + + theme(legend.position = "bottom", legend.title = element_text(size = rel(0.75)), legend.text = element_text(size = rel(0.5))) +ggsave("anytime.png", plot = g, device = "png", width = 15, height = 10) + +overall_budget = agg_budget[, .(mean = mean(mean), se = sd(mean) / sqrt(.N)), by = .(method, cumbudget_scaled)] + +g = ggplot(aes(x = cumbudget_scaled, y = mean, colour = method, fill = method), data = overall_budget[cumbudget_scaled > 0.10]) + + scale_y_log10() + + geom_step() + + geom_stepribbon(aes(min = mean - se, max = mean + se), colour = NA, alpha = 0.1) + + labs(x = "Fraction of Budget Used", y = "Mean Normalized Regret", colour = "Optimizer", fill = "Optimizer") + + theme_minimal() + + theme(legend.position = "bottom", legend.title = element_text(size = rel(0.75)), legend.text = element_text(size = rel(0.75))) +ggsave("anytime_average.png", plot = g, device = "png", width = 6, height = 4) + +methods = unique(agg_budget$method) +ranks = map_dtr(unique(agg_budget$scenario), function(scenario_) { + map_dtr(unique(agg_budget$instance), function(instance_) { + map_dtr(unique(agg_budget$cumbudget_scaled), function(cumbudget_scaled_) { + res = agg_budget[scenario == scenario_ & instance == instance_ & cumbudget_scaled == cumbudget_scaled_] + if (nrow(res) == 0L) { + return(data.table()) + } + setorderv(res, "mean") + data.table(rank = match(methods, res$method), method = methods, scenario = scenario_, instance = instance_, cumbudget_scaled = cumbudget_scaled_) + }) + }) +}) + +ranks_overall = ranks[, .(mean = mean(rank), se = sd(rank) / sqrt(.N)), by = .(method, cumbudget_scaled)] + +g = ggplot(aes(x = cumbudget_scaled, y = mean, colour = method, fill = method), data = ranks_overall[cumbudget_scaled > 0.10]) + + geom_line() + + geom_ribbon(aes(min = mean - se, max = mean + se), colour = NA, alpha = 0.3) + + labs(x = "Fraction of Budget Used", y = "Mean Rank", colour = "Optimizer", fill = "Optimizer") + + theme_minimal() + + theme(legend.position = "bottom", legend.title = element_text(size = rel(0.75)), legend.text = element_text(size = rel(0.75))) +ggsave("anytime_average_rank.png", plot = g, device = "png", width = 6, height = 4) + +library(scmamp) +best_agg = agg_budget[cumbudget_scaled == 0.25] +best_agg[, problem := paste0(scenario, "_", instance)] +tmp = - as.matrix(dcast(best_agg, problem ~ method, value.var = "mean")[, -1]) +friedmanTest(tmp) +png("cd_025_mf.png", width = 6, height = 4, pointsize = 10, units = "in", res = 150) +plotCD(tmp, cex = 1) +dev.off() + +best_agg = agg_budget[cumbudget_scaled == 1] +best_agg[, problem := paste0(scenario, "_", instance)] +tmp = - as.matrix(dcast(best_agg, problem ~ method, value.var = "mean")[, -1]) +friedmanTest(tmp) +png("cd_1_mf.png", width = 6, height = 4, pointsize = 10, units = "in", res = 150) +plotCD(tmp, cex = 1) +dev.off() + diff --git a/attic/so_config/cd_example.R b/attic/so_config/cd_example.R index 6614c668..494cc6c8 100644 --- a/attic/so_config/cd_example.R +++ b/attic/so_config/cd_example.R @@ -1,10 +1,10 @@ library(bbotk) library(paradox) +library(data.table) library(R6) library(checkmate) library(mlr3misc) - source("OptimizerCoordinateDescent.R") domain = ps(x1 = p_dbl(lower = -1, upper = 1), diff --git a/attic/so_config/coordinate_descent.R b/attic/so_config/coordinate_descent.R index 043fbb6c..0c985232 100644 --- a/attic/so_config/coordinate_descent.R +++ b/attic/so_config/coordinate_descent.R @@ -24,13 +24,13 @@ source("OptimizerChain.R") search_space = ps( loop_function = p_fct(c("ego", "ego_log")), init = p_fct(c("random", "lhs", "sobol")), - init_size_factor = p_int(lower = 1L, upper = 4L), + init_size_fraction = p_fct(c("0.0625", "0.125", "0.25")), random_interleave = p_lgl(), random_interleave_iter = p_fct(c("2", "4", "10"), depends = random_interleave == TRUE), rf_type = p_fct(c("standard", "smaclike_boot", "smaclike_no_boot", "smaclike_variance_boot")), - acqf = p_fct(c("EI", "TTEI", "CB", "PI", "Mean")), + acqf = p_fct(c("EI", "CB", "PI", "Mean")), lambda = p_int(lower = 1, upper = 3, depends = acqf == "CB"), acqopt = p_fct(c("RS_1000", "RS", "FS", "LS")) ) @@ -75,7 +75,7 @@ evaluate = function(xdt, instance) { optim_instance = make_optim_instance(instance) d = optim_instance$search_space$length - init_design_size = d * xdt$init_size_factor + init_design_size = ceiling(as.numeric(xdt$init_size_fraction) * optim_instance$terminator$param_set$values$n_evals) init_design = if (xdt$init == "random") { generate_design_random(optim_instance$search_space, n = init_design_size)$data } else if (xdt$init == "lhs") { @@ -145,8 +145,6 @@ evaluate = function(xdt, instance) { acq_function = if (xdt$acqf == "EI") { AcqFunctionEI$new() - } else if (xdt$acqf == "TTEI") { - AcqFunctionTTEI$new(toplvl_acq_optimizer = acq_optimizer$clone(deep = TRUE)) } else if (xdt$acqf == "CB") { AcqFunctionCB$new(lambda = xdt$lambda) } else if (xdt$acqf == "PI") { @@ -171,9 +169,10 @@ objective = ObjectiveRFunDt$new( xdt[, id := seq_len(.N)] # FIXME: walltime can be set adaptively based on xdt # FIXME: we could continuously model the walltime with a surrogate and set this for each xs in xdt - plan("batchtools_slurm", resources = list(walltime = 3600L * 7L, ncpus = 1L, memory = 1000L), template = "slurm_wyoming.tmpl") + plan("batchtools_slurm", resources = list(walltime = 3600L * 12L, ncpus = 1L, memory = 1000L), template = "slurm_wyoming.tmpl") res = future_mapply(FUN = evaluate, transpose_list(xdt), transpose_list(instances), SIMPLIFY = FALSE, future.seed = TRUE) # FIXME: tryCatch res = rbindlist(res) + stopifnot(nrow(res) = nrow(xdt) * nrow(instances)) agg = res[, .(mean_perf = mean(ecdf_best), raw_perfs = list(ecdf_best), n_na = sum(is.na(ecdf_best))), by = .(id)] setorderv(agg, cols = "id") agg @@ -192,7 +191,7 @@ cd_instance = OptimInstanceSingleCrit$new( optimizer = OptimizerCoordinateDescent$new() optimizer$param_set$values$max_gen = 1L -init = data.table(loop_function = "ego", init = "random", init_size_factor = 4L, random_interleave = FALSE, random_interleave_iter = NA_character_, rf_type = "standard", acqf = "EI", lambda = NA_integer_, acqopt = "RS") +init = data.table(loop_function = "ego", init = "random", init_size_fraction = "0.25", random_interleave = FALSE, random_interleave_iter = NA_character_, rf_type = "standard", acqf = "EI", lambda = NA_integer_, acqopt = "RS") set.seed(2906) cd_instance$eval_batch(init) optimizer$optimize(cd_instance) diff --git a/attic/so_config/min_max.R b/attic/so_config/min_max.R index bbba8ae1..27d2335b 100644 --- a/attic/so_config/min_max.R +++ b/attic/so_config/min_max.R @@ -6,6 +6,7 @@ reticulate::use_virtualenv("/home/lps/.local/share/virtualenvs/yahpo_gym-4ygV7gg library(reticulate) library(yahpogym) + instances = data.table(scenario = rep(paste0("rbv2_", c("aknn", "glmnet", "ranger", "rpart", "super", "svm", "xgboost")), each = 4L), instance = c("40499", "1476", "6", "12", "40979", "1501", "40966", "1478", @@ -15,7 +16,18 @@ instances = data.table(scenario = rep(paste0("rbv2_", c("aknn", "glmnet", "range "1478", "1501", "40499", "40979", "40984", "40979", "40966", "28"), target = "acc", - budget = rep(c(118, 90, 134, 110, 267, 118, 170), each = 4L)) + budget = rep(c(118L, 90L, 134L, 110L, 267L, 118L, 170L), each = 4L)) + +instances_so = data.table(scenario = rep(c("lcbench", "nb301", "rbv2_glmnet", "rbv2_ranger", "rbv2_rpart", "rbv2_super", "rbv2_xgboost"), c(3L, 1L, 2L, 2L, 2L, 6L, 4L)), + instance = c("167168", "189873", "189906", + "CIFAR10", + "375", "458", + "16", "42", + "14", "40499", + "1053", "1457", "1063", "1479", "15", "1468", + "12", "1501", "16", "40499"), + target = rep(c("val_accuracy", "val_accuracy", "acc", "acc", "acc", "acc", "acc"), c(3L, 1L, 2L, 2L, 2L, 6L, 4L)), + budget = rep(c(126L, 250L, 90L, 134L, 110L, 267L, 170L), c(3L, 1L, 2L, 2L, 2L, 6L, 4L))) make_optim_instance = function(instance) { benchmark = BenchmarkSet$new(instance$scenario, instance = instance$instance) @@ -38,3 +50,34 @@ y_stats = map_dtr(seq_len(nrow(instances)), function(i) { }) saveRDS(cbind(instances, y_stats), "instances.rds") + +y_stats_so = map_dtr(seq_len(nrow(instances_so)), function(i) { + evaluate(instances_so[i, ]) +}) + +saveRDS(cbind(instances_so, y_stats_so), "instances_so.rds") + +instances_so = readRDS("instances_so.rds") +smac = readRDS("results_yahpo.rds")[method == "smac4hpo"] +smac_best = smac[, .(best = min(target)), by = .(scenario, instance, repl)] + +results = map_dtr(unique(instances_so$scenario), function(scenario_) { + map_dtr(unique(instances_so$instance), function(instance_) { + tmp = smac_best[scenario == scenario_ & instance == instance_] + if (nrow(tmp) == 0) return(NULL) + ecdf = instances_so[scenario == scenario_ & instance == instance_]$ecdf[[1L]](-tmp$best) + diff = instances_so[scenario == scenario_ & instance == instance_]$max - (-tmp$best) + if (scenario_ %in% c("lcbench", "nb301")) diff = diff / 100 + data.table(ecdf = ecdf, diff = diff, problem = paste0(scenario_, "_", instance_)) + }) +}) + +library(ggplot2) +g = ggplot(aes(x = problem, y = ecdf), data = results) + + geom_boxplot() + + theme(axis.text.x = element_text(angle = 90, vjust = 0.5, hjust=1)) + +g = ggplot(aes(x = problem, y = diff), data = results) + + geom_boxplot() + + theme(axis.text.x = element_text(angle = 90, vjust = 0.5, hjust=1)) + diff --git a/attic/so_config/run.R b/attic/so_config/run.R index e614f2d7..442632cd 100755 --- a/attic/so_config/run.R +++ b/attic/so_config/run.R @@ -34,13 +34,13 @@ set.seed(seeds[run_id]) search_space = ps( loop_function = p_fct(c("ego", "ego_log")), init = p_fct(c("random", "lhs", "sobol")), - init_size_factor = p_int(lower = 1L, upper = 4L), + init_size_fraction = p_fct(c("0.0625", "0.125", "0.25")), random_interleave = p_lgl(), random_interleave_iter = p_fct(c("2", "4", "10"), depends = random_interleave == TRUE), rf_type = p_fct(c("standard", "smaclike_boot", "smaclike_no_boot", "smaclike_variance_boot")), - acqf = p_fct(c("EI", "TTEI", "CB", "PI", "Mean")), + acqf = p_fct(c("EI", "CB", "PI", "Mean")), lambda = p_int(lower = 1, upper = 3, depends = acqf == "CB"), acqopt = p_fct(c("RS_1000", "RS", "FS", "LS")) ) @@ -75,7 +75,7 @@ evaluate = function(xdt, instance) { optim_instance = make_optim_instance(instance) d = optim_instance$search_space$length - init_design_size = d * xdt$init_size_factor + init_design_size = ceiling(as.numeric(xdt$init_size_fraction) * optim_instance$terminator$param_set$values$n_evals) init_design = if (xdt$init == "random") { generate_design_random(optim_instance$search_space, n = init_design_size)$data } else if (xdt$init == "lhs") { @@ -145,8 +145,6 @@ evaluate = function(xdt, instance) { acq_function = if (xdt$acqf == "EI") { AcqFunctionEI$new() - } else if (xdt$acqf == "TTEI") { - AcqFunctionTTEI$new(toplvl_acq_optimizer = acq_optimizer$clone(deep = TRUE)) } else if (xdt$acqf == "CB") { AcqFunctionCB$new(lambda = xdt$lambda) } else if (xdt$acqf == "PI") { diff --git a/attic/so_config/run_yahpo.R b/attic/so_config/run_yahpo.R index fb600d33..10dc58a0 100644 --- a/attic/so_config/run_yahpo.R +++ b/attic/so_config/run_yahpo.R @@ -40,10 +40,10 @@ mlr3mbo_wrapper = function(job, data, instance, ...) { optim_instance = make_optim_instance(instance) - xdt = data.table(loop_function = "ego_log", init = "random", init_size_factor = 4L, random_interleave = FALSE, random_interleave_iter = NA_integer_, rf_type = "smaclike_boot", acqf = "EI", lambda = NA_real_, acqopt = "FS") + xdt = data.table(loop_function = "ego_log", init = "random", init_size_fraction = "0.25", random_interleave = TRUE, random_interleave_iter = "4", rf_type = "smaclike_boot", acqf = "EI", lambda = NA_real_, acqopt = "LS") d = optim_instance$search_space$length - init_design_size = d * xdt$init_size_factor + init_design_size = ceiling(as.numeric(xdt$init_size_fraction) * optim_instance$terminator$param_set$values$n_evals) init_design = if (xdt$init == "random") { generate_design_random(optim_instance$search_space, n = init_design_size)$data } else if (xdt$init == "lhs") { @@ -113,8 +113,6 @@ mlr3mbo_wrapper = function(job, data, instance, ...) { acq_function = if (xdt$acqf == "EI") { AcqFunctionEI$new() - } else if (xdt$acqf == "TTEI") { - AcqFunctionTTEI$new(toplvl_acq_optimizer = acq_optimizer$clone(deep = TRUE)) } else if (xdt$acqf == "CB") { AcqFunctionCB$new(lambda = xdt$lambda) } else if (xdt$acqf == "PI") { @@ -228,7 +226,7 @@ prob_designs = unlist(prob_designs, recursive = FALSE, use.names = FALSE) names(prob_designs) = nn # add jobs for optimizers -optimizers = data.table(algorithm = c("mlr3mbo", "mlrintermbo")) +optimizers = data.table(algorithm = c("mlr3mbo")) for (i in seq_len(nrow(optimizers))) { algo_designs = setNames(list(optimizers[i, ]), nm = optimizers[i, ]$algorithm) @@ -242,7 +240,7 @@ for (i in seq_len(nrow(optimizers))) { } jobs = getJobTable() -resources.default = list(walltime = 3600 * 5L, memory = 1024L, ntasks = 1L, ncpus = 1L, nodes = 1L, clusters = "teton", max.concurrent.jobs = 9999L) +resources.default = list(walltime = 3600 * 12L, memory = 2048L, ntasks = 1L, ncpus = 1L, nodes = 1L, clusters = "teton", max.concurrent.jobs = 9999L) submitJobs(jobs, resources = resources.default) done = findDone() From 03b71af03852912aa54d1e334c72665bafea5dea Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Thu, 1 Dec 2022 16:37:28 +0100 Subject: [PATCH 030/126] mies baseline --- attic/so_config/DNGO.R | 49 ------ attic/so_config/LearnerRegrKKNNEnsemble.R | 41 ----- attic/so_config/min_max.R | 83 --------- attic/so_config/run.R | 203 ---------------------- attic/so_config/run_yahpo.R | 2 +- attic/so_config/run_yahpo_mies.R | 129 ++++++++++++++ attic/so_config/submit.sh | 20 --- 7 files changed, 130 insertions(+), 397 deletions(-) delete mode 100644 attic/so_config/DNGO.R delete mode 100644 attic/so_config/LearnerRegrKKNNEnsemble.R delete mode 100644 attic/so_config/min_max.R delete mode 100755 attic/so_config/run.R create mode 100644 attic/so_config/run_yahpo_mies.R delete mode 100644 attic/so_config/submit.sh diff --git a/attic/so_config/DNGO.R b/attic/so_config/DNGO.R deleted file mode 100644 index 47ff4f0b..00000000 --- a/attic/so_config/DNGO.R +++ /dev/null @@ -1,49 +0,0 @@ -library(torch) - -x = c(2.7, 4, 4.5, 4.6, 7.4) -f = function(x) sin(x) + sin((10 / 3) * x) -y = f(x) - -x = torch_tensor(as.matrix(x), dtype = torch_float()) -y = torch_tensor(as.matrix(y), dtype = torch_float()) - -input_dim = 1 -n_input = 5 - -model = nn_sequential( - nn_linear(input_dim, 50), - nn_tanh(), - - nn_linear(50, 50), - nn_tanh(), - - nn_linear(50, 50), - nn_tanh(), - - nn_linear(50, 1) -) - -criterion = nn_mse_loss() - -optimizer = optim_adam(model$parameters, lr = 0.01) - -epochs = 200 - -for (i in seq_len(epochs)) { - optimizer$zero_grad() - y_pred = model(x) - loss = criterion(y_pred, y) - loss$backward() - optimizer$step() -} - -phi = as.matrix(model[["5"]]$forward(model[["4"]]$forward(model[["3"]]$forward(model[["2"]]$forward(model[["1"]]$forward(model[["0"]]$forward(x))))))) - -alpha = 1 -beta = 1 -y_ = as.matrix(y) # FIXME: - nu(x) (9) - -K = beta * t(phi) %*% phi + diag(alpha, nrow = 50L, ncol = 50L) -m = beta * solve(K) %*% t(phi) %*% y_ - -mu = t(m) %*% diff --git a/attic/so_config/LearnerRegrKKNNEnsemble.R b/attic/so_config/LearnerRegrKKNNEnsemble.R deleted file mode 100644 index 83231edf..00000000 --- a/attic/so_config/LearnerRegrKKNNEnsemble.R +++ /dev/null @@ -1,41 +0,0 @@ -LearnerRegrEnsemble = R6Class("LearnerRegrEnsemble", - inherit = LearnerRegr, - public = list( - learners = NULL, - - #' @description - #' Creates a new instance of this [R6][R6::R6Class] class. - initialize = function(learners) { - self$learners = assert_learners(learners) # i.e., kknn - - ps = ps( - ) - - super$initialize( - id = "regr.ensemble", - param_set = ps, - feature_types = c("logical", "integer", "numeric", "factor", "ordered"), - packages = "mlr3learners", - man = "" - ) - } - ), - - private = list( - .train = function(task) { - for (i in seq_along(self$learners)) { - self$learners[[i]]$predict_type = "response" - self$learners[[i]]$train(task) - } - list() - }, - - .predict = function(task) { - predictions = map_dtr(self$learners, function(learner) { - as.data.table(learner$predict(task)) - }) - predictions = predictions[, .(mean = mean(response), se = sd(response)), by = .(row_ids)] - list(response = predictions$response, se = predictions$se) - } - ) -) diff --git a/attic/so_config/min_max.R b/attic/so_config/min_max.R deleted file mode 100644 index 27d2335b..00000000 --- a/attic/so_config/min_max.R +++ /dev/null @@ -1,83 +0,0 @@ -library(data.table) -library(bbotk) -library(mlr3misc) -library(paradox) -reticulate::use_virtualenv("/home/lps/.local/share/virtualenvs/yahpo_gym-4ygV7ggv/", required = TRUE) -library(reticulate) -library(yahpogym) - - -instances = data.table(scenario = rep(paste0("rbv2_", c("aknn", "glmnet", "ranger", "rpart", "super", "svm", "xgboost")), each = 4L), - instance = c("40499", "1476", "6", "12", - "40979", "1501", "40966", "1478", - "12", "458", "1510", "1515", - "1478", "40979", "12", "28", - "41164", "37", "1515", "1510", - "1478", "1501", "40499", "40979", - "40984", "40979", "40966", "28"), - target = "acc", - budget = rep(c(118L, 90L, 134L, 110L, 267L, 118L, 170L), each = 4L)) - -instances_so = data.table(scenario = rep(c("lcbench", "nb301", "rbv2_glmnet", "rbv2_ranger", "rbv2_rpart", "rbv2_super", "rbv2_xgboost"), c(3L, 1L, 2L, 2L, 2L, 6L, 4L)), - instance = c("167168", "189873", "189906", - "CIFAR10", - "375", "458", - "16", "42", - "14", "40499", - "1053", "1457", "1063", "1479", "15", "1468", - "12", "1501", "16", "40499"), - target = rep(c("val_accuracy", "val_accuracy", "acc", "acc", "acc", "acc", "acc"), c(3L, 1L, 2L, 2L, 2L, 6L, 4L)), - budget = rep(c(126L, 250L, 90L, 134L, 110L, 267L, 170L), c(3L, 1L, 2L, 2L, 2L, 6L, 4L))) - -make_optim_instance = function(instance) { - benchmark = BenchmarkSet$new(instance$scenario, instance = instance$instance) - benchmark$subset_codomain(instance$target) - objective = benchmark$get_objective(instance$instance, multifidelity = FALSE, check_values = FALSE) - budget = instance$budget - optim_instance = OptimInstanceSingleCrit$new(objective, search_space = benchmark$get_search_space(drop_fidelity_params = TRUE), terminator = trm("evals", n_evals = 1000000L), check_values = FALSE) - optim_instance -} - -evaluate = function(instance) { - optim_instance = make_optim_instance(instance) - opt("random_search", batch_size = 10000L)$optimize(optim_instance) - ys = optim_instance$archive$data[, optim_instance$archive$cols_y, with = FALSE][[1L]] - data.table(min = min(ys), max = max(ys), mean = mean(ys), sd = sd(ys), ecdf = list(ecdf(ys))) -} - -y_stats = map_dtr(seq_len(nrow(instances)), function(i) { - evaluate(instances[i, ]) -}) - -saveRDS(cbind(instances, y_stats), "instances.rds") - -y_stats_so = map_dtr(seq_len(nrow(instances_so)), function(i) { - evaluate(instances_so[i, ]) -}) - -saveRDS(cbind(instances_so, y_stats_so), "instances_so.rds") - -instances_so = readRDS("instances_so.rds") -smac = readRDS("results_yahpo.rds")[method == "smac4hpo"] -smac_best = smac[, .(best = min(target)), by = .(scenario, instance, repl)] - -results = map_dtr(unique(instances_so$scenario), function(scenario_) { - map_dtr(unique(instances_so$instance), function(instance_) { - tmp = smac_best[scenario == scenario_ & instance == instance_] - if (nrow(tmp) == 0) return(NULL) - ecdf = instances_so[scenario == scenario_ & instance == instance_]$ecdf[[1L]](-tmp$best) - diff = instances_so[scenario == scenario_ & instance == instance_]$max - (-tmp$best) - if (scenario_ %in% c("lcbench", "nb301")) diff = diff / 100 - data.table(ecdf = ecdf, diff = diff, problem = paste0(scenario_, "_", instance_)) - }) -}) - -library(ggplot2) -g = ggplot(aes(x = problem, y = ecdf), data = results) + - geom_boxplot() + - theme(axis.text.x = element_text(angle = 90, vjust = 0.5, hjust=1)) - -g = ggplot(aes(x = problem, y = diff), data = results) + - geom_boxplot() + - theme(axis.text.x = element_text(angle = 90, vjust = 0.5, hjust=1)) - diff --git a/attic/so_config/run.R b/attic/so_config/run.R deleted file mode 100755 index 442632cd..00000000 --- a/attic/so_config/run.R +++ /dev/null @@ -1,203 +0,0 @@ -#!/usr/bin/env Rscript -# chmod ug+x -library(argparse) -library(data.table) -library(mlr3) -library(mlr3misc) -library(mlr3learners) -library(mlr3pipelines) -library(bbotk) # @localsearch -library(paradox) -library(R6) -library(checkmate) -library(mlr3mbo) # @so_config -reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) -library(reticulate) -library(yahpogym) -library("future") -library("future.apply") - -source("LearnerRegrRangerCustom.R") -source("OptimizerChain.R") - -parser = ArgumentParser() -parser$add_argument("-r", "--run", type = "integer", default = 1, help = "Id of run, should be within 1-5") -args = parser$parse_args() -run_id = args$run -stopifnot(run_id %in% 1:5) -cat("run_id:", run_id, "\n") - -seeds = c(2409, 2906, 0905, 2412, 3112) - -set.seed(seeds[run_id]) - -search_space = ps( - loop_function = p_fct(c("ego", "ego_log")), - init = p_fct(c("random", "lhs", "sobol")), - init_size_fraction = p_fct(c("0.0625", "0.125", "0.25")), - random_interleave = p_lgl(), - random_interleave_iter = p_fct(c("2", "4", "10"), depends = random_interleave == TRUE), - - rf_type = p_fct(c("standard", "smaclike_boot", "smaclike_no_boot", "smaclike_variance_boot")), - - acqf = p_fct(c("EI", "CB", "PI", "Mean")), - lambda = p_int(lower = 1, upper = 3, depends = acqf == "CB"), - acqopt = p_fct(c("RS_1000", "RS", "FS", "LS")) -) - -instances = readRDS("instances.rds") -#instances = data.table(scenario = rep(paste0("rbv2_", c("aknn", "glmnet", "ranger", "rpart", "super", "svm", "xgboost")), each = 5L), -# instance = c("40499", "1476", "6", "12", "41150", -# "40979", "1501", "40966", "1478", "40984", -# "12", "458", "1510", "1515", "307", -# "1478", "40979", "12", "28", "1501", -# "41164", "37", "1515", "1510", "42", -# "1478", "1501", "40499", "40979", "300", -# "40984", "40979", "40966", "28", "22"), -# target = "acc", -# budget = rep(c(118, 90, 134, 110, 267, 118, 170), each = 5L)) - -make_optim_instance = function(instance) { - benchmark = BenchmarkSet$new(instance$scenario, instance = instance$instance) - benchmark$subset_codomain(instance$target) - objective = benchmark$get_objective(instance$instance, multifidelity = FALSE, check_values = FALSE) - budget = instance$budget - optim_instance = OptimInstanceSingleCrit$new(objective, search_space = benchmark$get_search_space(drop_fidelity_params = TRUE), terminator = trm("evals", n_evals = budget), check_values = FALSE) - optim_instance -} - -evaluate = function(xdt, instance) { - logger = lgr::get_logger("mlr3") - logger$set_threshold("warn") - logger = lgr::get_logger("bbotk") - logger$set_threshold("warn") - - optim_instance = make_optim_instance(instance) - - d = optim_instance$search_space$length - init_design_size = ceiling(as.numeric(xdt$init_size_fraction) * optim_instance$terminator$param_set$values$n_evals) - init_design = if (xdt$init == "random") { - generate_design_random(optim_instance$search_space, n = init_design_size)$data - } else if (xdt$init == "lhs") { - generate_design_lhs(optim_instance$search_space, n = init_design_size)$data - } else if (xdt$init == "sobol") { - generate_design_sobol(optim_instance$search_space, n = init_design_size)$data - } - - optim_instance$eval_batch(init_design) - - random_interleave_iter = if(xdt$random_interleave) as.numeric(xdt$random_interleave_iter) else 0L - - learner = LearnerRegrRangerCustom$new() - learner$predict_type = "se" - learner$param_set$values$keep.inbag = TRUE - - if (xdt$rf_type == "standard") { - learner$param_set$values$se.method = "jack" - learner$param_set$values$splitrule = "variance" - learner$param_set$values$num.trees = 1000L - } else if (xdt$rf_type == "smaclike_boot") { - learner$param_set$values$se.method = "simple" - learner$param_set$values$splitrule = "extratrees" - learner$param_set$values$num.random.splits = 1L - learner$param_set$values$num.trees = 10L - learner$param_set$values$replace = TRUE - learner$param_set$values$sample.fraction = 1 - learner$param_set$values$min.node.size = 1 - learner$param_set$values$mtry.ratio = 1 - } else if (xdt$rf_type == "smaclike_no_boot") { - learner$param_set$values$se.method = "simple" - learner$param_set$values$splitrule = "extratrees" - learner$param_set$values$num.random.splits = 1L - learner$param_set$values$num.trees = 10L - learner$param_set$values$replace = FALSE - learner$param_set$values$sample.fraction = 1 - learner$param_set$values$min.node.size = 1 - learner$param_set$values$mtry.ratio = 1 - } else if (xdt$rf_type == "smaclike_variance_boot") { - learner$param_set$values$se.method = "simple" - learner$param_set$values$splitrule = "variance" - learner$param_set$values$num.trees = 10L - learner$param_set$values$replace = TRUE - learner$param_set$values$sample.fraction = 1 - learner$param_set$values$min.node.size = 1 - learner$param_set$values$mtry.ratio = 1 - } - - surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor", multiplier = 3) %>>% learner)) - - acq_optimizer = if (xdt$acqopt == "RS_1000") { - AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 1000L)) - } else if (xdt$acqopt == "RS") { - AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 20000L)) - } else if (xdt$acqopt == "FS") { - n_repeats = 2L - maxit = 9L - batch_size = ceiling((20000L / n_repeats) / (1 + maxit)) # 1000L - AcqOptimizer$new(opt("focus_search", n_points = batch_size, maxit = maxit), terminator = trm("evals", n_evals = 20000L)) - } else if (xdt$acqopt == "LS") { - optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) - acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20020L)) - acq_optimizer$param_set$values$warmstart = TRUE - acq_optimizer$param_set$values$warmstart_size = 10L - acq_optimizer - } - - acq_function = if (xdt$acqf == "EI") { - AcqFunctionEI$new() - } else if (xdt$acqf == "CB") { - AcqFunctionCB$new(lambda = xdt$lambda) - } else if (xdt$acqf == "PI") { - AcqFunctionPI$new() - } else if (xdt$acqf == "Mean") { - AcqFunctionMean$new() - } - - if (xdt$loop_function == "ego") { - bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) - } else if (xdt$loop_function == "ego_log") { - bayesopt_ego_log(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) - } - - best = optim_instance$archive$best()[[instance$target]] - ecdf_best = instance$ecdf(best) # evaluate the precomputed ecdf for the best value found; our target is effectively P(X <= best) - cat("scenario:", instance$scenario, "instance:", instance$instance, "ECDF_best:", ecdf_best, "\n") - ecdf_best -} - -objective = ObjectiveRFunDt$new( - fun = function(xdt) { - saveRDS(ac_instance, paste0("ac_instance_", run_id, ".rds")) - map_dtr(seq_len(nrow(xdt)), function(i) { - plan("multicore") - tmp = future_lapply(transpose_list(instances), function(instance) { - res_instance = tryCatch(evaluate(xdt[i, ], instance), error = function(error_condition) 0) - }, future.seed = TRUE) - data.table(mean_perf = mean(unlist(tmp), na.rm = TRUE), raw_perfs = list(tmp), n_na = sum(is.na(unlist(tmp)))) - }) - }, - domain = search_space, - codomain = ps(mean_perf = p_dbl(tags = "maximize")), - check_values = FALSE -) - -ac_instance = OptimInstanceSingleCrit$new( - objective = objective, - search_space = search_space, - terminator = trm("evals", n_evals = 200L) # 100 init design + 100 -) - -surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% lrn("regr.ranger", keep.inbag = TRUE, se.method = "jack", num.trees = 1000L))) -acq_function = AcqFunctionEI$new() -optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) -acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20020L)) -acq_optimizer$param_set$values$warmstart = TRUE -acq_optimizer$param_set$values$warmstart_size = 10L -design = generate_design_sobol(ac_instance$search_space, n = 100L)$data -for (i in seq_len(nrow(design))) { - ac_instance$eval_batch(design[i, ]) -} -saveRDS(ac_instance, paste0("ac_instance_", run_id, ".rds")) -bayesopt_ego(ac_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = 5L) -saveRDS(ac_instance, paste0("ac_instance_", run_id, ".rds")) - diff --git a/attic/so_config/run_yahpo.R b/attic/so_config/run_yahpo.R index 10dc58a0..6f8fc7cb 100644 --- a/attic/so_config/run_yahpo.R +++ b/attic/so_config/run_yahpo.R @@ -7,6 +7,7 @@ library(mlr3misc) library(mlr3mbo) # @so_config library(bbotk) # @localsearch library(paradox) +library(miesmuschel) library(R6) library(checkmate) @@ -151,7 +152,6 @@ mlrintermbo_wrapper = function(job, data, instance, ...) { optim_instance } - # add algorithms addAlgorithm("mlr3mbo", fun = mlr3mbo_wrapper) addAlgorithm("mlrintermbo", fun = mlrintermbo_wrapper) diff --git a/attic/so_config/run_yahpo_mies.R b/attic/so_config/run_yahpo_mies.R new file mode 100644 index 00000000..0744a1f7 --- /dev/null +++ b/attic/so_config/run_yahpo_mies.R @@ -0,0 +1,129 @@ +library(batchtools) +library(data.table) +library(mlr3) +library(mlr3misc) +library(bbotk) # @localsearch +library(paradox) +library(miesmuschel) # @mlr3mbo_config +library(R6) +library(checkmate) + +reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) +library(reticulate) +yahpo_gym = import("yahpo_gym") + +packages = c("data.table", "mlr3", "mlr3misc", "bbotk", "paradox", "miesmuschel", "R6", "checkmate") + +#RhpcBLASctl::blas_set_num_threads(1L) +#RhpcBLASctl::omp_set_num_threads(1L) + +reg = makeExperimentRegistry(file.dir = "/gscratch/lschnei8/registry_mies_yahpo", packages = packages) +#reg = makeExperimentRegistry(file.dir = NA, conf.file = NA, packages = packages) # interactive session +saveRegistry(reg) + +mies_wrapper = function(job, data, instance, ...) { + # mies is our baseline with 1000 x more budget + reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + make_optim_instance = function(instance) { + benchmark = BenchmarkSet$new(instance$scenario, instance = instance$instance) + benchmark$subset_codomain(instance$target) + objective = benchmark$get_objective(instance$instance, multifidelity = FALSE, check_values = FALSE) + budget = instance$budget + optim_instance = OptimInstanceSingleCrit$new(objective, search_space = benchmark$get_search_space(drop_fidelity_params = TRUE), terminator = trm("evals", n_evals = budget), check_values = FALSE) + optim_instance + } + + recombine = rec("maybe", recombinator = rec("cmpmaybe", recombinator = rec("swap"), p = 0.5), p = 0.7) + + mutate_gauss = mut("cmpmaybe", mutator = mut("gauss", sdev = 0.1, sdev_is_relative = TRUE), p = 0.2) + mutate_uniform = mut("cmpmaybe", mutator = mut("unif", can_mutate_to_same = FALSE), p = 0.2) + + mutate = mut("maybe", + mutator = mut("combine", operators = list(ParamDbl = mutate_gauss$clone(deep = TRUE), + ParamInt = mutate_gauss$clone(deep = TRUE), + ParamFct = mutate_uniform$clone(deep = TRUE), + ParamLgl = mutate_uniform$clone(deep = TRUE))), + p = 0.3) + + select = sel("random", sample_unique = "no") + + survive = sel("best") + + optimizer = opt("mies", recombinator = recombine, mutator = mutate, parent_selector = select, survival_selector = survive, + mu = 100, lambda = 100, survival_strategy = "plus") + + optim_instance = make_optim_instance(instance) + optimizer$optimize(optim_instance) + optim_instance +} + + +# add algorithms +addAlgorithm("mies", fun = mies_wrapper) + +setup = data.table(scenario = rep(c("lcbench", paste0("rbv2_", c("aknn", "glmnet", "ranger", "rpart", "super", "svm", "xgboost"))), each = 4L), + instance = c("167185", "167152", "168910", "189908", + "40499", "1476", "6", "12", + "40979", "1501", "40966", "1478", + "12", "458", "1510", "1515", + "1478", "40979", "12", "28", + "41164", "37", "1515", "1510", + "1478", "1501", "40499", "40979", + "40984", "40979", "40966", "28"), + target = rep(c("val_accuracy", "acc"), c(4L, 28L)), + budget = rep(c(126L, 118L, 90L, 134L, 110L, 267L, 118L, 170L), each = 4L)) +setup[, budget := budget * 1000L] + +setup[, id := seq_len(.N)] + +# add problems +prob_designs = map(seq_len(nrow(setup)), function(i) { + prob_id = paste0(setup[i, ]$scenario, "_", setup[i, ]$instance, "_", setup[i, ]$target) + addProblem(prob_id, data = list(scenario = setup[i, ]$scenario, instance = setup[i, ]$instance, target = setup[i, ]$target, budget = setup[i, ]$budget, on_integer_scale = setup[i, ]$on_integer_scale)) + setNames(list(setup[i, ]), nm = prob_id) +}) +nn = sapply(prob_designs, names) +prob_designs = unlist(prob_designs, recursive = FALSE, use.names = FALSE) +names(prob_designs) = nn + +# add jobs for optimizers +optimizers = data.table(algorithm = c("mies")) + +for (i in seq_len(nrow(optimizers))) { + algo_designs = setNames(list(optimizers[i, ]), nm = optimizers[i, ]$algorithm) + + ids = addExperiments( + prob.designs = prob_designs, + algo.designs = algo_designs, + repls = 30L + ) + addJobTags(ids, as.character(optimizers[i, ]$algorithm)) +} + +# rbv2_super 267000 budget needs ~ 20 minutes so 30 repls results in roughly 10 hours +jobs = findJobs() +jobs[, chunk := batchtools::chunk(job.id, chunk.size = 30L)] +resources.default = list(walltime = 3600 * 12L, memory = 2048L, ntasks = 1L, ncpus = 1L, nodes = 1L, clusters = "teton", max.concurrent.jobs = 9999L) +submitJobs(jobs, resources = resources.default) + +done = findDone() +results = reduceResultsList(done, function(x, job) { + x = x$archive$data + pars = job$pars + target_var = pars$prob.pars$target + tmp = x[, eval(target_var), with = FALSE] + tmp[, method := pars$algo.pars$algorithm] + tmp[, scenario := pars$prob.pars$scenario] + tmp[, instance := pars$prob.pars$instance] + tmp[, repl := job$repl] + tmp[, iter := seq_len(.N)] + tmp +}) +results = rbindlist(results, fill = TRUE) +saveRDS(results, "results_yahpo_mies.rds") + diff --git a/attic/so_config/submit.sh b/attic/so_config/submit.sh deleted file mode 100644 index fec39991..00000000 --- a/attic/so_config/submit.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -sbatch < Date: Fri, 23 Dec 2022 14:47:29 +0100 Subject: [PATCH 031/126] new eval scheme for so_config --- attic/so_config/coordinate_descent.R | 39 ++++++++++++++++++++-------- attic/so_config/run_yahpo_mies.R | 11 +++++--- 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/attic/so_config/coordinate_descent.R b/attic/so_config/coordinate_descent.R index 0c985232..cc8b50cc 100644 --- a/attic/so_config/coordinate_descent.R +++ b/attic/so_config/coordinate_descent.R @@ -35,7 +35,19 @@ search_space = ps( acqopt = p_fct(c("RS_1000", "RS", "FS", "LS")) ) -instances = readRDS("instances.rds") +instances = setup = data.table(scenario = rep(c("lcbench", paste0("rbv2_", c("aknn", "glmnet", "ranger", "rpart", "super", "svm", "xgboost"))), each = 4L), + instance = c("167185", "167152", "168910", "189908", + "40499", "1476", "6", "12", + "40979", "1501", "40966", "1478", + "12", "458", "1510", "1515", + "1478", "40979", "12", "28", + "41164", "37", "1515", "1510", + "1478", "1501", "40499", "40979", + "40984", "40979", "40966", "28"), + target = rep(c("val_accuracy", "acc"), c(4L, 28L)), + budget = rep(c(126L, 118L, 90L, 134L, 110L, 267L, 118L, 170L), each = 4L)) + +mies_average = readRDS("results_yahpo_mies_average.rds") evaluate = function(xdt, instance) { id = xdt$id @@ -74,7 +86,6 @@ evaluate = function(xdt, instance) { optim_instance = make_optim_instance(instance) - d = optim_instance$search_space$length init_design_size = ceiling(as.numeric(xdt$init_size_fraction) * optim_instance$terminator$param_set$values$n_evals) init_design = if (xdt$init == "random") { generate_design_random(optim_instance$search_space, n = init_design_size)$data @@ -129,14 +140,14 @@ evaluate = function(xdt, instance) { acq_optimizer = if (xdt$acqopt == "RS_1000") { AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 1000L)) } else if (xdt$acqopt == "RS") { - AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 20000L)) + AcqOptimizer$new(opt("random_search", batch_size = 10000L), terminator = trm("evals", n_evals = 20000L)) } else if (xdt$acqopt == "FS") { n_repeats = 2L maxit = 9L batch_size = ceiling((20000L / n_repeats) / (1 + maxit)) # 1000L AcqOptimizer$new(opt("focus_search", n_points = batch_size, maxit = maxit), terminator = trm("evals", n_evals = 20000L)) } else if (xdt$acqopt == "LS") { - optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) + optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 10000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20020L)) acq_optimizer$param_set$values$warmstart = TRUE acq_optimizer$param_set$values$warmstart_size = 10L @@ -160,8 +171,14 @@ evaluate = function(xdt, instance) { } best = optim_instance$archive$best()[[instance$target]] - ecdf_best = instance$ecdf(best) # evaluate the precomputed ecdf for the best value found; our target is effectively P(X <= best) - data.table(ecdf_best = ecdf_best, id = id, instance = paste0(instance$scenario, "_", instance$instance)) + scenario_ = instance$scenario + instance_ = instance$instance + target_ = paste0("mean_", instance$target) + + k = min(mies_average[scenario == scenario_ & instance == instance_ & get(target_) >= best]$iter) # minimum k so that best_mies[k] >= best_mbo[final] + k = k / instance$budget # sample efficiency compared to mies + + data.table(k = k, id = id, instance = paste0(instance$scenario, "_", instance$instance)) } objective = ObjectiveRFunDt$new( @@ -169,16 +186,16 @@ objective = ObjectiveRFunDt$new( xdt[, id := seq_len(.N)] # FIXME: walltime can be set adaptively based on xdt # FIXME: we could continuously model the walltime with a surrogate and set this for each xs in xdt - plan("batchtools_slurm", resources = list(walltime = 3600L * 12L, ncpus = 1L, memory = 1000L), template = "slurm_wyoming.tmpl") - res = future_mapply(FUN = evaluate, transpose_list(xdt), transpose_list(instances), SIMPLIFY = FALSE, future.seed = TRUE) # FIXME: tryCatch + #plan("batchtools_slurm", resources = list(walltime = 3600L * 12L, ncpus = 1L, memory = 2000L), template = "slurm_wyoming.tmpl") + res = future_mapply(FUN = evaluate, transpose_list(xdt), transpose_list(instances), SIMPLIFY = FALSE, future.seed = TRUE) res = rbindlist(res) - stopifnot(nrow(res) = nrow(xdt) * nrow(instances)) - agg = res[, .(mean_perf = mean(ecdf_best), raw_perfs = list(ecdf_best), n_na = sum(is.na(ecdf_best))), by = .(id)] + stopifnot(nrow(res) == nrow(xdt) * nrow(instances)) + agg = res[, .(mean_k = exp(mean(log(k))), raw_k = list(k), n_na = sum(is.na(k))), by = .(id)] setorderv(agg, cols = "id") agg }, domain = search_space, - codomain = ps(mean_perf = p_dbl(tags = "maximize")), + codomain = ps(mean_k = p_dbl(tags = "maximize")), check_values = FALSE ) diff --git a/attic/so_config/run_yahpo_mies.R b/attic/so_config/run_yahpo_mies.R index 0744a1f7..39039832 100644 --- a/attic/so_config/run_yahpo_mies.R +++ b/attic/so_config/run_yahpo_mies.R @@ -17,7 +17,7 @@ packages = c("data.table", "mlr3", "mlr3misc", "bbotk", "paradox", "miesmuschel" #RhpcBLASctl::blas_set_num_threads(1L) #RhpcBLASctl::omp_set_num_threads(1L) -reg = makeExperimentRegistry(file.dir = "/gscratch/lschnei8/registry_mies_yahpo", packages = packages) +reg = makeExperimentRegistry(file.dir = "/gscratch/lschnei8/registry_yahpo_mies", packages = packages) #reg = makeExperimentRegistry(file.dir = NA, conf.file = NA, packages = packages) # interactive session saveRegistry(reg) @@ -107,8 +107,8 @@ for (i in seq_len(nrow(optimizers))) { # rbv2_super 267000 budget needs ~ 20 minutes so 30 repls results in roughly 10 hours jobs = findJobs() -jobs[, chunk := batchtools::chunk(job.id, chunk.size = 30L)] -resources.default = list(walltime = 3600 * 12L, memory = 2048L, ntasks = 1L, ncpus = 1L, nodes = 1L, clusters = "teton", max.concurrent.jobs = 9999L) +jobs[, chunk := batchtools::chunk(job.id, chunk.size = 10L)] +resources.default = list(walltime = 3600 * 12L, memory = 8192, ntasks = 1L, ncpus = 2L, nodes = 1L, clusters = "teton", max.concurrent.jobs = 9999L) submitJobs(jobs, resources = resources.default) done = findDone() @@ -127,3 +127,8 @@ results = reduceResultsList(done, function(x, job) { results = rbindlist(results, fill = TRUE) saveRDS(results, "results_yahpo_mies.rds") +mean_results_lcbench = results[grepl("lcbench", x = scenario), .(mean_val_accuracy = mean(val_accuracy), se_val_accuracy = sd(val_accuracy) / sqrt(.N)), by = .(scenario, instance, iter)] +mean_results_rbv2 = results[grepl("rbv2", x = scenario), .(mean_acc = mean(acc), se_acc = sd(acc) / sqrt(.N)), by = .(scenario, instance, iter)] +mean_results = rbind(mean_results_lcbench, mean_results_rbv2, fill = TRUE) +saveRDS(mean_results, "results_yahpo_mies_average.rds") + From 3842d29a6dd25ef5b4f362e1a3753b01fc5581a8 Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Fri, 23 Dec 2022 18:37:49 +0100 Subject: [PATCH 032/126] .. --- attic/so_config/coordinate_descent.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/attic/so_config/coordinate_descent.R b/attic/so_config/coordinate_descent.R index cc8b50cc..4e6b6ca8 100644 --- a/attic/so_config/coordinate_descent.R +++ b/attic/so_config/coordinate_descent.R @@ -186,7 +186,7 @@ objective = ObjectiveRFunDt$new( xdt[, id := seq_len(.N)] # FIXME: walltime can be set adaptively based on xdt # FIXME: we could continuously model the walltime with a surrogate and set this for each xs in xdt - #plan("batchtools_slurm", resources = list(walltime = 3600L * 12L, ncpus = 1L, memory = 2000L), template = "slurm_wyoming.tmpl") + plan("batchtools_slurm", resources = list(walltime = 3600L * 12L, ncpus = 1L, memory = 2000L), template = "slurm_wyoming.tmpl") res = future_mapply(FUN = evaluate, transpose_list(xdt), transpose_list(instances), SIMPLIFY = FALSE, future.seed = TRUE) res = rbindlist(res) stopifnot(nrow(res) == nrow(xdt) * nrow(instances)) From f141a0907dcedee898fc148de27f1e196788b2d2 Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Mon, 2 Jan 2023 12:07:05 +0100 Subject: [PATCH 033/126] .. --- attic/so_config/LearnerRegrTraforest.R | 163 +++++++++++++++++++++++++ attic/so_config/analyze_yahpo.R | 6 +- attic/so_config/coordinate_descent.R | 37 ++++-- attic/so_config/run_yahpo.R | 69 ++++++++--- attic/so_config/trafbo.R | 128 +++++++++++++++++++ attic/so_config/trafbo_old.R | 123 +++++++++++++++++++ 6 files changed, 495 insertions(+), 31 deletions(-) create mode 100644 attic/so_config/LearnerRegrTraforest.R create mode 100644 attic/so_config/trafbo.R create mode 100644 attic/so_config/trafbo_old.R diff --git a/attic/so_config/LearnerRegrTraforest.R b/attic/so_config/LearnerRegrTraforest.R new file mode 100644 index 00000000..a402352f --- /dev/null +++ b/attic/so_config/LearnerRegrTraforest.R @@ -0,0 +1,163 @@ +#' @title Ranger Regression Learner Custom +#' +#' @name mlr_learners_regr.ranger_custom +#' +#' @description +#' Random regression forest. +#' Calls [ranger::ranger()] from package \CRANpkg{ranger}. +#' +#' @inheritSection mlr_learners_classif.ranger Custom mlr3 parameters +#' @inheritSection mlr_learners_classif.ranger Initial parameter values +#' +#' @templateVar id regr.ranger_custom +#' @template learner +#' +#' @references +#' `r format_bib("wright_2017", "breiman_2001")` +#' +#' @export +#' @template seealso_learner +#' @template example +LearnerRegrRangerCustom = R6Class("LearnerRegrRangerCustom", + inherit = LearnerRegr, + + public = list( + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + initialize = function() { + ps = ps( + alpha = p_dbl(default = 0.5, tags = "train"), + always.split.variables = p_uty(tags = "train"), + holdout = p_lgl(default = FALSE, tags = "train"), + importance = p_fct(c("none", "impurity", "impurity_corrected", "permutation"), tags = "train"), + keep.inbag = p_lgl(default = FALSE, tags = "train"), + max.depth = p_int(default = NULL, lower = 0L, special_vals = list(NULL), tags = "train"), + min.node.size = p_int(1L, default = 5L, special_vals = list(NULL), tags = "train"), + min.prop = p_dbl(default = 0.1, tags = "train"), + minprop = p_dbl(default = 0.1, tags = "train"), + mtry = p_int(lower = 1L, special_vals = list(NULL), tags = "train"), + mtry.ratio = p_dbl(lower = 0, upper = 1, tags = "train"), + num.random.splits = p_int(1L, default = 1L, tags = "train"), + num.threads = p_int(1L, default = 1L, tags = c("train", "predict", "threads")), + num.trees = p_int(1L, default = 500L, tags = c("train", "predict", "hotstart")), + oob.error = p_lgl(default = TRUE, tags = "train"), + quantreg = p_lgl(default = FALSE, tags = "train"), + regularization.factor = p_uty(default = 1, tags = "train"), + regularization.usedepth = p_lgl(default = FALSE, tags = "train"), + replace = p_lgl(default = TRUE, tags = "train"), + respect.unordered.factors = p_fct(c("ignore", "order", "partition"), default = "ignore", tags = "train"), + sample.fraction = p_dbl(0L, 1L, tags = "train"), + save.memory = p_lgl(default = FALSE, tags = "train"), + scale.permutation.importance = p_lgl(default = FALSE, tags = "train"), + se.method = p_fct(c("jack", "infjack", "simple"), default = "infjack", tags = "predict"), # FIXME: only works if predict_type == "se". How to set dependency? + seed = p_int(default = NULL, special_vals = list(NULL), tags = c("train", "predict")), + split.select.weights = p_uty(default = NULL, tags = "train"), + splitrule = p_fct(c("variance", "extratrees", "maxstat"), default = "variance", tags = "train"), + verbose = p_lgl(default = TRUE, tags = c("train", "predict")), + write.forest = p_lgl(default = TRUE, tags = "train") + ) + + ps$values = list(num.threads = 1L) + + # deps + ps$add_dep("num.random.splits", "splitrule", CondEqual$new("extratrees")) + ps$add_dep("alpha", "splitrule", CondEqual$new("maxstat")) + ps$add_dep("minprop", "splitrule", CondEqual$new("maxstat")) + ps$add_dep("scale.permutation.importance", "importance", CondEqual$new("permutation")) + + + super$initialize( + id = "regr.ranger_custom", + param_set = ps, + predict_types = c("response", "se"), + feature_types = c("logical", "integer", "numeric", "character", "factor", "ordered"), + properties = c("weights", "importance", "oob_error", "hotstart_backward"), + packages = c("mlr3learners", "ranger"), + man = "mlr3learners::mlr_learners_regr.ranger_custom" + ) + }, + + #' @description + #' The importance scores are extracted from the model slot `variable.importance`. + #' Parameter `importance.mode` must be set to `"impurity"`, `"impurity_corrected"`, or + #' `"permutation"` + #' + #' @return Named `numeric()`. + importance = function() { + if (is.null(self$model)) { + stopf("No model stored") + } + if (self$model$importance.mode == "none") { + stopf("No importance stored") + } + + sort(self$model$variable.importance, decreasing = TRUE) + }, + + #' @description + #' The out-of-bag error, extracted from model slot `prediction.error`. + #' + #' @return `numeric(1)`. + oob_error = function() { + if (is.null(self$model)) { + stopf("No model stored") + } + self$model$prediction.error + } + ), + + private = list( + .train = function(task) { + pv = self$param_set$get_values(tags = "train") + pv = mlr3learners:::convert_ratio(pv, "mtry", "mtry.ratio", length(task$feature_names)) + + if (self$predict_type == "se") { + pv$keep.inbag = TRUE # nolint + } + + mlr3misc::invoke(ranger::ranger, + dependent.variable.name = task$target_names, + data = task$data(), + case.weights = task$weights$weight, + .args = pv + ) + }, + + .predict = function(task) { + pv = self$param_set$get_values(tags = "predict") + newdata = mlr3learners:::ordered_features(task, self) + + if (isTRUE(pv$se.method == "simple")) { + # the variance is only correct if min.node.size = 1 and therefore sigma^2_b is always 0 or rather undefined (as + # there is always only a single observation in each leaf) + sigma_min = 0.01 + prediction = mlr3misc::invoke(predict, self$model, data = newdata, type = "response", .args = pv[setdiff(names(pv), "se.method")], predict.all = TRUE) + response = rowMeans(prediction$predictions) + tmp = sigma_min ^ 2 + prediction$predictions ^ 2 + variance = rowMeans(tmp) - response ^ 2 + list(response = response, se = sqrt(variance)) + } else { + prediction = mlr3misc::invoke(predict, self$model, data = newdata, type = self$predict_type, .args = pv) + list(response = prediction$predictions, se = prediction$se) + } + }, + + .hotstart = function(task) { + model = self$model + model$num.trees = self$param_set$values$num.trees + model + } + ) +) + +#' @export +default_values.LearnerRegrRangerCustom = function(x, search_space, task, ...) { # nolint + special_defaults = list( + mtry = floor(sqrt(length(task$feature_names))), + mtry.ratio = floor(sqrt(length(task$feature_names))) / length(task$feature_names), + sample.fraction = 1 + ) + defaults = insert_named(default_values(x$param_set), special_defaults) + defaults[search_space$ids()] +} diff --git a/attic/so_config/analyze_yahpo.R b/attic/so_config/analyze_yahpo.R index f2f14a6e..3f55faee 100644 --- a/attic/so_config/analyze_yahpo.R +++ b/attic/so_config/analyze_yahpo.R @@ -3,8 +3,10 @@ library(ggplot2) library(pammtools) library(mlr3misc) -dat = rbind(readRDS("results_yahpo.rds"), readRDS("results_yahpo_own.rds"))[method != c("mlrintermbo")] -#dat = dat[scenario %nin% c("nb301", "rbv2_super")] +dat = rbind(readRDS("results_yahpo.rds"), readRDS("results_yahpo_trafbo.rds")) +dat[method == "trafbo", method := "trafbo_new"] +dat = dat[scenario %in% "lcbench"] +dat = dat[method %in% c("random", "smac4hpo", "mlr3mbo", "mlr3mbo_gp", "trafbo", "trafbo_new")] dat[, cumbudget := cumsum(budget), by = .(method, scenario, instance, repl)] dat[, cumbudget_scaled := cumbudget / max(cumbudget), by = .(method, scenario, instance, repl)] dat[, normalized_regret := (target - min(target)) / (max(target) - min(target)), by = .(scenario, instance)] diff --git a/attic/so_config/coordinate_descent.R b/attic/so_config/coordinate_descent.R index 4e6b6ca8..e18a8f7c 100644 --- a/attic/so_config/coordinate_descent.R +++ b/attic/so_config/coordinate_descent.R @@ -35,17 +35,29 @@ search_space = ps( acqopt = p_fct(c("RS_1000", "RS", "FS", "LS")) ) -instances = setup = data.table(scenario = rep(c("lcbench", paste0("rbv2_", c("aknn", "glmnet", "ranger", "rpart", "super", "svm", "xgboost"))), each = 4L), - instance = c("167185", "167152", "168910", "189908", - "40499", "1476", "6", "12", - "40979", "1501", "40966", "1478", - "12", "458", "1510", "1515", - "1478", "40979", "12", "28", - "41164", "37", "1515", "1510", - "1478", "1501", "40499", "40979", - "40984", "40979", "40966", "28"), - target = rep(c("val_accuracy", "acc"), c(4L, 28L)), - budget = rep(c(126L, 118L, 90L, 134L, 110L, 267L, 118L, 170L), each = 4L)) +#instances = setup = data.table(scenario = rep(c("lcbench", paste0("rbv2_", c("aknn", "glmnet", "ranger", "rpart", "super", "svm", "xgboost"))), each = 4L), +# instance = c("167185", "167152", "168910", "189908", +# "40499", "1476", "6", "12", +# "40979", "1501", "40966", "1478", +# "12", "458", "1510", "1515", +# "1478", "40979", "12", "28", +# "41164", "37", "1515", "1510", +# "1478", "1501", "40499", "40979", +# "40984", "40979", "40966", "28"), +# target = rep(c("val_accuracy", "acc"), c(4L, 28L)), +# budget = rep(c(126L, 118L, 90L, 134L, 110L, 267L, 118L, 170L), each = 4L)) + +instances = setup = data.table(scenario = rep(c("lcbench", paste0("rbv2_", c("aknn", "glmnet", "ranger", "rpart", "super", "svm", "xgboost"))), each = 3L), + instance = c("167185", "167152", "168910", #"189908", + "40499", "1476", "6", #"12", + "40979", "1501", "40966", #"1478", + "12", "458", "1510", #"1515", + "1478", "40979", "12", #"28", + "41164", "37", "1515", #"1510", + "1478", "1501", "40499", #"40979", + "40984", "40979", "40966"), #"28"), + target = rep(c("val_accuracy", "acc"), c(3L, 21L)), + budget = rep(c(126L, 118L, 90L, 134L, 110L, 267L, 118L, 170L), each = 3L)) mies_average = readRDS("results_yahpo_mies_average.rds") @@ -186,7 +198,6 @@ objective = ObjectiveRFunDt$new( xdt[, id := seq_len(.N)] # FIXME: walltime can be set adaptively based on xdt # FIXME: we could continuously model the walltime with a surrogate and set this for each xs in xdt - plan("batchtools_slurm", resources = list(walltime = 3600L * 12L, ncpus = 1L, memory = 2000L), template = "slurm_wyoming.tmpl") res = future_mapply(FUN = evaluate, transpose_list(xdt), transpose_list(instances), SIMPLIFY = FALSE, future.seed = TRUE) res = rbindlist(res) stopifnot(nrow(res) == nrow(xdt) * nrow(instances)) @@ -210,6 +221,8 @@ optimizer$param_set$values$max_gen = 1L init = data.table(loop_function = "ego", init = "random", init_size_fraction = "0.25", random_interleave = FALSE, random_interleave_iter = NA_character_, rf_type = "standard", acqf = "EI", lambda = NA_integer_, acqopt = "RS") set.seed(2906) +plan("batchtools_slurm", resources = list(walltime = 3600L * 12L, ncpus = 1L, memory = 4000L), template = "slurm_wyoming.tmpl") cd_instance$eval_batch(init) optimizer$optimize(cd_instance) + diff --git a/attic/so_config/run_yahpo.R b/attic/so_config/run_yahpo.R index 6f8fc7cb..04f0e3f9 100644 --- a/attic/so_config/run_yahpo.R +++ b/attic/so_config/run_yahpo.R @@ -10,12 +10,13 @@ library(paradox) library(miesmuschel) library(R6) library(checkmate) +library(trtf) reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) library(reticulate) yahpo_gym = import("yahpo_gym") -packages = c("data.table", "mlr3", "mlr3learners", "mlr3pipelines", "mlr3misc", "mlr3mbo", "bbotk", "paradox", "mlrintermbo", "R6", "checkmate") +packages = c("data.table", "mlr3", "mlr3learners", "mlr3pipelines", "mlr3misc", "mlr3mbo", "bbotk", "paradox", "mlrintermbo", "R6", "checkmate", "trtf") #RhpcBLASctl::blas_set_num_threads(1L) #RhpcBLASctl::omp_set_num_threads(1L) @@ -23,12 +24,12 @@ packages = c("data.table", "mlr3", "mlr3learners", "mlr3pipelines", "mlr3misc", root = here::here() experiments_dir = file.path(root) -source_files = map_chr(c("helpers.R", "LearnerRegrRangerCustom.R", "OptimizerChain.R"), function(x) file.path(experiments_dir, x)) +source_files = map_chr(c("helpers.R", "LearnerRegrRangerCustom.R", "OptimizerChain.R", "trafbo.R"), function(x) file.path(experiments_dir, x)) for (sf in source_files) { source(sf) } -reg = makeExperimentRegistry(file.dir = "/gscratch/lschnei8/registry_mlr3mbo_so_config_new", packages = packages, source = source_files) +reg = makeExperimentRegistry(file.dir = "/gscratch/lschnei8/registry_mlr3mbo_so_config_trafbo", packages = packages, source = source_files) #reg = makeExperimentRegistry(file.dir = NA, conf.file = NA, packages = packages, source = source_files) # interactive session saveRegistry(reg) @@ -131,7 +132,7 @@ mlr3mbo_wrapper = function(job, data, instance, ...) { optim_instance } -mlrintermbo_wrapper = function(job, data, instance, ...) { +trafbo_wrapper = function(job, data, instance, ...) { reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) library(yahpogym) logger = lgr::get_logger("bbotk") @@ -139,22 +140,55 @@ mlrintermbo_wrapper = function(job, data, instance, ...) { future::plan("sequential") optim_instance = make_optim_instance(instance) - optimizer = opt("intermbo", on.surrogate.error = "stop") - learner = lrn("regr.ranger", se.method = "jack", keep.inbag = TRUE) - learner$predict_type = "se" - learner = GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor") %>>% learner) + + bayesopt_trafbo(optim_instance) + + optim_instance +} + +mlr3mbo_gp_wrapper = function(job, data, instance, ...) { + reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + optim_instance = make_optim_instance(instance) + + d = optim_instance$search_space$length + init_design_size = ceiling(0.25 * optim_instance$terminator$param_set$values$n_evals) + init_design = generate_design_sobol(optim_instance$search_space, n = init_design_size)$data + + optim_instance$eval_batch(init_design) + + random_interleave_iter = 0L + + learner = LearnerRegrKM$new() learner$predict_type = "se" - #learner = mlr::makeLearner("regr.randomForest", se.method = "jackknife", keep.inbag = TRUE) - #learner = mlr::makeImputeWrapper(learner, classes = list(numeric = mlr::imputeMax(2), factor = mlr::imputeConstant("__miss__"), logical = mlr::imputeUniform())) - #learner = mlr::setPredictType(learner, "se") - #optimizer$param_set$values$surrogate.learner = learner - optimizer$optimize(optim_instance) + learner$param_set$values$covtype = "matern3_2" + learner$param_set$values$optim.method = "gen" + learner$param_set$values$control = list(trace = FALSE) + learner$param_set$values$nugget.stability = 10^-8 + + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor", multiplier = 3) %>>% learner)) + + optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) + acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20020L)) + acq_optimizer$param_set$values$warmstart = TRUE + acq_optimizer$param_set$values$warmstart_size = 10L + acq_optimizer + + acq_function = AcqFunctionEI$new() + + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + optim_instance } # add algorithms addAlgorithm("mlr3mbo", fun = mlr3mbo_wrapper) -addAlgorithm("mlrintermbo", fun = mlrintermbo_wrapper) +addAlgorithm("trafbo", fun = trafbo_wrapper) +addAlgorithm("mlr3mbo_gp", fun = mlr3mbo_gp_wrapper) # setup scenarios and instances get_nb301_setup = function(budget_factor = 40L) { @@ -226,7 +260,7 @@ prob_designs = unlist(prob_designs, recursive = FALSE, use.names = FALSE) names(prob_designs) = nn # add jobs for optimizers -optimizers = data.table(algorithm = c("mlr3mbo")) +optimizers = data.table(algorithm = c("mlr3mbo", "trafbo", "mlr3mbo_gp")) for (i in seq_len(nrow(optimizers))) { algo_designs = setNames(list(optimizers[i, ]), nm = optimizers[i, ]$algorithm) @@ -240,7 +274,8 @@ for (i in seq_len(nrow(optimizers))) { } jobs = getJobTable() -resources.default = list(walltime = 3600 * 12L, memory = 2048L, ntasks = 1L, ncpus = 1L, nodes = 1L, clusters = "teton", max.concurrent.jobs = 9999L) +jobs = jobs[grepl("lcbench", x = problem) & tags == "trafbo", ] +resources.default = list(walltime = 3600 * 3L, memory = 2048L, ntasks = 1L, ncpus = 1L, nodes = 1L, clusters = "teton", max.concurrent.jobs = 9999L) submitJobs(jobs, resources = resources.default) done = findDone() @@ -263,5 +298,5 @@ results = reduceResultsList(done, function(x, job) { tmp }) results = rbindlist(results, fill = TRUE) -saveRDS(results, "results_yahpo_own.rds") +saveRDS(results, "results_yahpo_trafbo.rds") diff --git a/attic/so_config/trafbo.R b/attic/so_config/trafbo.R new file mode 100644 index 00000000..5aef6249 --- /dev/null +++ b/attic/so_config/trafbo.R @@ -0,0 +1,128 @@ +bayesopt_trafbo = function(instance, init_design_size = NULL) { + if (is.null(init_design_size)) { + init_design_size = ceiling(instance$terminator$param_set$values$n_evals / 4L) + } + design = generate_design_random(instance$search_space, init_design_size)$data + instance$eval_batch(design) + repeat { + if (runif(1) <= 0.2) { + candidate = generate_design_random(instance$search_space, 1L)$data + instance$eval_batch(candidate) + } else { + # FIXME: adapt for arbitrary target, optim direction and search space + dat = copy(instance$archive$data[, c(instance$archive$cols_x, instance$archive$cols_y), with = FALSE]) + + y = - dat$val_accuracy + min_y = min(y) - 0.05 * diff(range(y)) + max_y = max(y) + y_log = log((y - min_y) / (max_y - min_y)) + + lower = min(y_log) + upper = max(y_log) + support = c(lower - 0.05 * (upper - lower), upper + 0.05 * (upper - lower)) + + dat$val_accuracy = y_log + + y = numeric_var("val_accuracy", support = support, bounds = support) + yy = Bernstein_basis(y, order = 4, ui = "increasing") + + ctmm = ctm(response = yy, todistr = "Normal") + ctrl = ctree_control(minbucket = 5L, alpha = 0.05, maxdepth = 10L) + tree = trafotree(ctmm, formula = val_accuracy ~ ., data = dat, control = ctrl, min_update = 10L) + plot(tree, tp_args = list(type = "density", id = FALSE, ylines = 0, K = 200)) + + q = seq(support[1L], support[2L], length.out = 10001L) + imp = pmax(lower - q, 0) + + nodes = nodeids(tree, terminal = TRUE) + + if (length(nodes) > 1L) { + eis = map_dbl(nodes, function(node) { + xdt = generate_design_random(create_tmp_ss(instance$search_space, tree = tree, node = node), 100L)$data + xdt[predict(tree, xdt) == node, ][1L, ] + d = predict(tree, xdt, type = "density", q = q)[, 1L] + ei = sum(d / sum(d, na.rm = TRUE) * imp, na.rm = TRUE) + if (ei < sqrt(.Machine$double.eps)) ei = 0 + ei + }) + + best_node = nodes[which.max(eis)] + tmp = dat[predict(tree, dat) == best_node, ] + tmp_ss = create_tmp_ss(instance$search_space, tree = tree, node = best_node) + } else { + tmp = dat + tmp_ss = instance$search_space$clone(deep = TRUE) + } + + task = TaskRegr$new("inner", target = "val_accuracy", backend = tmp) + learner = as_learner(po("imputeoor", multiplier = 10L) %>>% lrn("regr.ranger", splitrule = "extratrees", num.trees = 1000L, min.node.size = 1L)) + learner$train(task) + setorderv(tmp, col = "val_accuracy") + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + if (runif(1) <= 0.5) { + f_instance = make_greedy_f_instance(tmp_ss, learner = learner, n_evals = 10010L) + f_instance$eval_batch(tmp[1:10, ]) + opt("local_search", n_points = 100L)$optimize(f_instance) + f_instance$archive$data = f_instance$archive$data[11:10010, ] + f_instance$terminator$param_set$values$n_evals = 20000L + opt("random_search", batch_size = 1000L)$optimize(f_instance) + } else { + f_instance = make_greedy_f_instance(tmp_ss, learner = learner, n_evals = 20000L) + opt("random_search", batch_size = 10000L)$optimize(f_instance) + } + f_instance$archive$data[, x_domain := list()] # FIXME: needed due to f_instance ObjectiveRFunDt + + if (length(nodes) > 1L) { + f_instance$archive$data = f_instance$archive$data[predict(tree, f_instance$archive$data[, f_instance$archive$cols_x, with = FALSE]) == best_node, ] + } + + candidate = mlr3mbo:::get_best_not_evaluated(f_instance, evaluated = instance$archive$data) + logger$set_threshold("info") + instance$eval_batch(candidate[, instance$archive$cols_x, with = FALSE]) + } + if (instance$is_terminated) break + } +} + +# FIXME: fails with everything not numeric/integer +create_tmp_ss = function(ss, tree, node) { + tmp_ss = ss$clone(deep = TRUE) + splits = partykit:::.list.rules.party(tree)[[as.character(node)]] + split_info = strsplit(splits, " & ")[[1L]] + + larger = which(sapply(split_info, grepl, pattern = ">")) + smaller = which(sapply(split_info, grepl, pattern = "<=")) + + split_info = strsplit(split_info, " > | <= ") + + for (i in larger) { + if (tmp_ss$params[[split_info[[i]][[1]]]]$class == "ParamDbl") { + tmp_ss$params[[split_info[[i]][1]]]$lower = as.numeric(split_info[[i]][2]) + } else if (tmp_ss$params[[split_info[[i]][[1]]]]$class == "ParamInt") { + tmp_ss$params[[split_info[[i]][1]]]$lower = min(as.integer(split_info[[i]][2]) + 1L, tmp_ss$params[[split_info[[i]][1]]]$upper) + } + } + + for (i in smaller) { + tmp_ss$params[[split_info[[i]][1]]]$upper = as.numeric(split_info[[i]][2]) + } + + tmp_ss +} + +make_greedy_f_instance = function(ss, learner, n_evals) { + ss = ss$clone(deep = TRUE) + ss$trafo = NULL + f = ObjectiveRFunDt$new( + fun = function(xdt) data.table(mean = learner$predict_newdata(xdt)$response), + domain = ss, + codomain = ps(mean = p_dbl(tags = "minimize")) + ) + f_instance = OptimInstanceSingleCrit$new( + objective = f, + search_space = ss, + terminator = trm("evals", n_evals = n_evals) + ) + f_instance +} diff --git a/attic/so_config/trafbo_old.R b/attic/so_config/trafbo_old.R new file mode 100644 index 00000000..23f6e9f6 --- /dev/null +++ b/attic/so_config/trafbo_old.R @@ -0,0 +1,123 @@ +bayesopt_trafbo = function(instance, init_design_size = NULL) { + if (is.null(init_design_size)) { + init_design_size = ceiling(instance$terminator$param_set$values$n_evals / 4L) + } + design = generate_design_random(instance$search_space, init_design_size)$data + instance$eval_batch(design) + repeat { + if (runif(1) <= 0.25) { + candidate = generate_design_random(instance$search_space, 1L)$data + instance$eval_batch(candidate) + } else { + # FIXME: adapt for arbitrary target, optim direction and search space + dat = copy(instance$archive$data[, c(instance$archive$cols_x, instance$archive$cols_y), with = FALSE]) + + y = - dat$val_accuracy + min_y = min(y) - 0.05 * diff(range(y)) + max_y = max(y) + y_log = log((y - min_y) / (max_y - min_y)) + + lower = min(y_log) + upper = max(y_log) + support = c(lower - 0.05 * (upper - lower), upper + 0.05 * (upper - lower)) + + dat$val_accuracy = y_log + + y = numeric_var("val_accuracy", support = support, bounds = support) + yy = Bernstein_basis(y, order = 4, ui = "increasing") + + ctmm = ctm(response = yy, todistr = "Normal") + ctrl = ctree_control(minbucket = 10L, alpha = 0.05, maxdepth = 10L) + tree = trafotree(ctmm, formula = val_accuracy ~ ., data = dat, control = ctrl, min_update = 10L) + plot(tree, tp_args = list(type = "density", id = FALSE, ylines = 0, K = 200)) + + q = seq(support[1L], support[2L], length.out = 10001L) + imp = pmax(lower - q, 0) + + nodes = nodeids(tree, terminal = TRUE) + + if (length(nodes) > 1L) { + eis = map_dbl(nodes, function(node) { + xdt = generate_design_random(create_tmp_ss(instance$search_space, tree = tree, node = node), 100L)$data + xdt[predict(tree, xdt) == node, ][1L, ] + d = predict(tree, xdt, type = "density", q = q)[, 1L] + ei = sum(d / sum(d, na.rm = TRUE) * imp, na.rm = TRUE) + if (ei < sqrt(.Machine$double.eps)) ei = 0 + ei + }) + + best_node = nodes[which.max(eis)] + tmp = dat[predict(tree, dat) == best_node, ] + tmp_ss = create_tmp_ss(instance$search_space, tree = tree, node = best_node) + } else { + tmp = dat + tmp_ss = instance$search_space$clone(deep = TRUE) + } + + task = TaskRegr$new("inner", target = "val_accuracy", backend = tmp) + learner = as_learner(po("imputeoor", multiplier = 10L) %>>% lrn("regr.ranger", splitrule = "extratrees", num.trees = 1000L)) + learner$train(task) + f_instance = make_greedy_f_instance(tmp_ss, learner = learner, n_evals = 10010L) + setorderv(tmp, col = "val_accuracy") + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + f_instance$eval_batch(tmp[1:10, ]) + opt("local_search")$optimize(f_instance) + f_instance$archive$data = f_instance$archive$data[11:10010, ] + f_instance$terminator$param_set$values$n_evals = 20000L + opt("random_search", batch_size = 10000L)$optimize(f_instance) + f_instance$archive$data[, x_domain := list()] # FIXME: needed due to f_instance ObjectiveRFunDt + + if (length(nodes) > 1L) { + f_instance$archive$data = f_instance$archive$data[predict(tree, f_instance$archive$data[, f_instance$archive$cols_x, with = FALSE]) == best_node, ] + } + + candidate = mlr3mbo:::get_best_not_evaluated(f_instance, evaluated = instance$archive$data) + logger$set_threshold("info") + instance$eval_batch(candidate[, instance$archive$cols_x, with = FALSE]) + } + if (instance$is_terminated) break + } +} + +# FIXME: fails with everything not numeric/integer +create_tmp_ss = function(ss, tree, node) { + tmp_ss = ss$clone(deep = TRUE) + splits = partykit:::.list.rules.party(tree)[[as.character(node)]] + split_info = strsplit(splits, " & ")[[1L]] + + larger = which(sapply(split_info, grepl, pattern = ">")) + smaller = which(sapply(split_info, grepl, pattern = "<=")) + + split_info = strsplit(split_info, " > | <= ") + + for (i in larger) { + if (tmp_ss$params[[split_info[[i]][[1]]]]$class == "ParamDbl") { + tmp_ss$params[[split_info[[i]][1]]]$lower = as.numeric(split_info[[i]][2]) + } else if (tmp_ss$params[[split_info[[i]][[1]]]]$class == "ParamInt") { + tmp_ss$params[[split_info[[i]][1]]]$lower = min(as.integer(split_info[[i]][2]) + 1L, tmp_ss$params[[split_info[[i]][1]]]$upper) + } + } + + for (i in smaller) { + tmp_ss$params[[split_info[[i]][1]]]$upper = as.numeric(split_info[[i]][2]) + } + + tmp_ss +} + +make_greedy_f_instance = function(ss, learner, n_evals) { + ss = ss$clone(deep = TRUE) + ss$trafo = NULL + f = ObjectiveRFunDt$new( + fun = function(xdt) data.table(mean = learner$predict_newdata(xdt)$response), + domain = ss, + codomain = ps(mean = p_dbl(tags = "minimize")) + ) + f_instance = OptimInstanceSingleCrit$new( + objective = f, + search_space = ss, + terminator = trm("evals", n_evals = n_evals) + ) + f_instance +} From 9c983e888def5d4e146e0e516f6fba6f21b02c62 Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Sun, 8 Jan 2023 20:52:52 +0100 Subject: [PATCH 034/126] .. --- attic/so_config/AcqFunctionLogEI.R | 64 ++++++++++ attic/so_config/trafbo.R | 196 +++++++++++++++++++++++------ 2 files changed, 222 insertions(+), 38 deletions(-) create mode 100644 attic/so_config/AcqFunctionLogEI.R diff --git a/attic/so_config/AcqFunctionLogEI.R b/attic/so_config/AcqFunctionLogEI.R new file mode 100644 index 00000000..3f2d2035 --- /dev/null +++ b/attic/so_config/AcqFunctionLogEI.R @@ -0,0 +1,64 @@ +#' @title Acquisition Function Log Expected Improvement +#' +#' @include AcqFunction.R +#' @name mlr_acqfunctions_log_ei +#' +#' @templateVar id log_ei +#' @template section_dictionary_acqfunctions +#' +#' @description +#' Expected Improvement assuming that the target variable is to be minimized and has been modeled on log scale. +#' +#' @family Acquisition Function +#' @export +#' @examples +AcqFunctionLogEI = R6Class("AcqFunctionLogEI", + inherit = AcqFunction, + + public = list( + + #' @field y_best (`numeric(1)`)\cr + #' Best objective function value observed so far. + #' On log scale. + y_best = NULL, + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + #' + #' @param surrogate (`NULL` | [SurrogateLearner]). + initialize = function(surrogate = NULL) { + assert_r6(surrogate, "SurrogateLearner", null.ok = TRUE) + super$initialize("acq_log_ei", surrogate = surrogate, direction = "maximize", label = "Log Expected Improvement", man = "mlr3mbo::mlr_acqfunctions_log_ei") + }, + + #' @description + #' Updates acquisition function and sets `y_best`. + update = function() { + if (self$surrogate_max_to_min != 1L) { + stop("Log EI assumes minimization of the log transformed target value.") + } + self$y_best = min(self$surrogate_max_to_min * self$archive$data[[self$surrogate$y_cols]]) + } + ), + + private = list( + .fun = function(xdt) { + if (is.null(self$y_best)) { + stop("$y_best is not set. Missed to call $update()?") + } + if (self$surrogate_max_to_min != 1L) { + stop("Log EI assumes minimization of the log transformed target value.") + } + p = self$surrogate$predict(xdt) + mu = p$mean + se = p$se + d_norm = (self$y_best - mu) / se + log_ei = (exp(self$y_best) * pnorm(d_norm)) - (exp((0.5 * se ^ 2) + mu) * pnorm(d_norm - se)) + log_ei = ifelse(se < 1e-20, 0, log_ei) + data.table(acq_log_ei = log_ei) + } + ) +) + +mlr_acqfunctions$add("log_ei", AcqFunctionLogEI) + diff --git a/attic/so_config/trafbo.R b/attic/so_config/trafbo.R index 5aef6249..71212b33 100644 --- a/attic/so_config/trafbo.R +++ b/attic/so_config/trafbo.R @@ -1,82 +1,128 @@ +# FIXME: check log EI again + +source("LearnerRegrRangerCustom.R") + bayesopt_trafbo = function(instance, init_design_size = NULL) { if (is.null(init_design_size)) { init_design_size = ceiling(instance$terminator$param_set$values$n_evals / 4L) } - design = generate_design_random(instance$search_space, init_design_size)$data + design = generate_design_sobol(instance$search_space, init_design_size)$data instance$eval_batch(design) repeat { - if (runif(1) <= 0.2) { + if (runif(1) <= 0.1) { candidate = generate_design_random(instance$search_space, 1L)$data instance$eval_batch(candidate) } else { # FIXME: adapt for arbitrary target, optim direction and search space dat = copy(instance$archive$data[, c(instance$archive$cols_x, instance$archive$cols_y), with = FALSE]) + fix_NA_and_chars(dat) + max_to_min = instance$archive$codomain$tags[[instance$archive$cols_y]] == "maximize" - y = - dat$val_accuracy + y = dat[[instance$archive$cols_y]] + if (max_to_min) y = - y min_y = min(y) - 0.05 * diff(range(y)) max_y = max(y) - y_log = log((y - min_y) / (max_y - min_y)) + y_log = log((y - min_y) / (max_y - min_y), base = 10L) lower = min(y_log) upper = max(y_log) - support = c(lower - 0.05 * (upper - lower), upper + 0.05 * (upper - lower)) + support = c(lower - 0.2 * (upper - lower), upper + 0.2 * (upper - lower)) + #support = c(pmin(support[1L], exp(support[1L])), pmax(support[2L], exp(support[2L]))) # density transformation FIXME: exp() - dat$val_accuracy = y_log + dat$y_log = y_log - y = numeric_var("val_accuracy", support = support, bounds = support) - yy = Bernstein_basis(y, order = 4, ui = "increasing") + y_var = numeric_var("y_log", support = support, bounds = support) + y_bern = Bernstein_basis(y_var, order = 8, ui = "increasing") - ctmm = ctm(response = yy, todistr = "Normal") - ctrl = ctree_control(minbucket = 5L, alpha = 0.05, maxdepth = 10L) - tree = trafotree(ctmm, formula = val_accuracy ~ ., data = dat, control = ctrl, min_update = 10L) + ctmm = ctm(response = y_bern, todistr = "Normal") + ctrl = ctree_control(minbucket = 10L, alpha = 0.05, maxdepth = Inf) + tree = trafotree(ctmm, formula = as.formula(paste0("y_log ~ ", paste0(instance$archive$cols_x, collapse = " + "))), data = dat, control = ctrl, min_update = 1L) plot(tree, tp_args = list(type = "density", id = FALSE, ylines = 0, K = 200)) - q = seq(support[1L], support[2L], length.out = 10001L) - imp = pmax(lower - q, 0) + q = seq(support[1L], lower, length.out = 1001L) + #imp = pmax(lower - q, 0) + imp = pmax(exp(lower) - exp(q), 0) # density transformation nodes = nodeids(tree, terminal = TRUE) + xdt = copy(dat) + xdt[, .node := predict(tree, dat[, instance$archive$cols_x, with = FALSE])] + if (length(nodes) > 1L) { + # actually log EI eis = map_dbl(nodes, function(node) { - xdt = generate_design_random(create_tmp_ss(instance$search_space, tree = tree, node = node), 100L)$data - xdt[predict(tree, xdt) == node, ][1L, ] - d = predict(tree, xdt, type = "density", q = q)[, 1L] - ei = sum(d / sum(d, na.rm = TRUE) * imp, na.rm = TRUE) + xdt_node = xdt[.node == node, ][, instance$archive$cols_x, with = FALSE][1L, ] + d = predict(tree, xdt_node, type = "density", q = q)[, 1L] + d = d * abs((1 / exp(q))) # density transformation FIXME: exp + + ei = sum(d * rev(diff(rev(c(imp, 0)))), na.rm = TRUE) if (ei < sqrt(.Machine$double.eps)) ei = 0 ei }) + names(eis) = nodes + + best_node = as.numeric(names(shuffle(which(eis == max(eis)))[1L])) # random tie breaks which can occur due to min_update > minbucket - best_node = nodes[which.max(eis)] - tmp = dat[predict(tree, dat) == best_node, ] + # FIXME: we could also proceed to evaluate every other second best node + + tmp = dat[predict(tree, dat[, instance$archive$cols_x, with = FALSE]) == best_node, ][, c("y_log", instance$archive$cols_x), with = FALSE] tmp_ss = create_tmp_ss(instance$search_space, tree = tree, node = best_node) } else { - tmp = dat + tmp = dat[, c("y_log", instance$archive$cols_x), with = FALSE] tmp_ss = instance$search_space$clone(deep = TRUE) } - task = TaskRegr$new("inner", target = "val_accuracy", backend = tmp) - learner = as_learner(po("imputeoor", multiplier = 10L) %>>% lrn("regr.ranger", splitrule = "extratrees", num.trees = 1000L, min.node.size = 1L)) + task = TaskRegr$new("inner", target = "y_log", backend = tmp) + + ranger = LearnerRegrRangerCustom$new() + ranger$param_set$values$se.method = "simple" + ranger$param_set$values$splitrule = "extratrees" + ranger$param_set$values$num.random.splits = 1L + ranger$param_set$values$num.trees = 10L + ranger$param_set$values$replace = TRUE + ranger$param_set$values$sample.fraction = 1 + ranger$param_set$values$min.node.size = 1 + ranger$param_set$values$mtry.ratio = 1 + learner = GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor", multiplier = 3) %>>% ranger) + learner$predict_type = "se" learner$train(task) - setorderv(tmp, col = "val_accuracy") logger = lgr::get_logger("bbotk") logger$set_threshold("warn") if (runif(1) <= 0.5) { - f_instance = make_greedy_f_instance(tmp_ss, learner = learner, n_evals = 10010L) - f_instance$eval_batch(tmp[1:10, ]) + f_instance = make_greedy_f_instance(tmp_ss, learner = learner, n_evals = 10010L, lower = lower) + best = copy(instance$archive$data) + best_tmp = best[, instance$archive$cols_x, with = FALSE] + fix_NA_and_chars(best_tmp) + if (length(nodes) > 1L) { + best[, .node := predict(tree, best_tmp)] + best = best[.node == best_node, ] + } + if (max_to_min) { + setorderv(best, col = instance$archive$cols_y, order = -1L) + } else { + setorderv(best, col = instance$archive$cols_y, order = 1L) + } + # FIXME: actually, we should evaluate all in the archive and then choose the 10 best within Local Search + best = best[1:min(10, nrow(best)), instance$archive$cols_x, with = FALSE] + f_instance$eval_batch(best) opt("local_search", n_points = 100L)$optimize(f_instance) - f_instance$archive$data = f_instance$archive$data[11:10010, ] - f_instance$terminator$param_set$values$n_evals = 20000L - opt("random_search", batch_size = 1000L)$optimize(f_instance) + f_instance$terminator$param_set$values$n_evals = 20010L + opt("random_search", batch_size = 10000L)$optimize(f_instance) } else { - f_instance = make_greedy_f_instance(tmp_ss, learner = learner, n_evals = 20000L) + f_instance = make_greedy_f_instance(tmp_ss, learner = learner, n_evals = 20000L, lower = lower) opt("random_search", batch_size = 10000L)$optimize(f_instance) } - f_instance$archive$data[, x_domain := list()] # FIXME: needed due to f_instance ObjectiveRFunDt - - if (length(nodes) > 1L) { - f_instance$archive$data = f_instance$archive$data[predict(tree, f_instance$archive$data[, f_instance$archive$cols_x, with = FALSE]) == best_node, ] + if ("x_domain" %nin% colnames(f_instance$archive$data)) { + f_instance$archive$data[, x_domain := list()] # FIXME: why is this necessary } + # FIXME: we want to make sure we evaluate only within the best node + #if (length(nodes) > 1L) { + # tmp = f_instance$archive$data[, f_instance$archive$cols_x, with = FALSE] + # fix_NA_and_chars(tmp) + # f_instance$archive$data = f_instance$archive$data[predict(tree, tmp) == best_node, ] + #} + candidate = mlr3mbo:::get_best_not_evaluated(f_instance, evaluated = instance$archive$data) logger$set_threshold("info") instance$eval_batch(candidate[, instance$archive$cols_x, with = FALSE]) @@ -85,7 +131,6 @@ bayesopt_trafbo = function(instance, init_design_size = NULL) { } } -# FIXME: fails with everything not numeric/integer create_tmp_ss = function(ss, tree, node) { tmp_ss = ss$clone(deep = TRUE) splits = partykit:::.list.rules.party(tree)[[as.character(node)]] @@ -93,8 +138,9 @@ create_tmp_ss = function(ss, tree, node) { larger = which(sapply(split_info, grepl, pattern = ">")) smaller = which(sapply(split_info, grepl, pattern = "<=")) + subset = which(sapply(split_info, grepl, pattern = "%in%")) - split_info = strsplit(split_info, " > | <= ") + split_info = strsplit(split_info, " > | <= | %in%") for (i in larger) { if (tmp_ss$params[[split_info[[i]][[1]]]]$class == "ParamDbl") { @@ -108,16 +154,77 @@ create_tmp_ss = function(ss, tree, node) { tmp_ss$params[[split_info[[i]][1]]]$upper = as.numeric(split_info[[i]][2]) } + for (i in subset) { + new_levels = unlist(map(tmp_ss$params[[split_info[[i]][1]]]$levels, function(pattern) if (grepl(pattern, x = split_info[[i]][2])) pattern else NULL)) + new_levels = setdiff(new_levels, c(".MISSING")) # FIXME: depends on imputeoor + if (length(new_levels) > 0L) { + tmp_ss$params[[split_info[[i]][1]]]$levels = new_levels + } + } + tmp_ss } -make_greedy_f_instance = function(ss, learner, n_evals) { +## Mean +#make_greedy_f_instance = function(ss, learner, n_evals, lower) { +# ss = ss$clone(deep = TRUE) +# ss$trafo = NULL +# f = ObjectiveRFunDt$new( +# fun = function(xdt) data.table(mean = learner$predict_newdata(fix_NA_and_chars(xdt))$response), +# domain = ss, +# codomain = ps(mean = p_dbl(tags = "minimize")) +# ) +# f_instance = OptimInstanceSingleCrit$new( +# objective = f, +# search_space = ss, +# terminator = trm("evals", n_evals = n_evals) +# ) +# f_instance +#} + +## EI +#make_greedy_f_instance = function(ss, learner, n_evals, lower) { +# ss = ss$clone(deep = TRUE) +# ss$trafo = NULL +# f = ObjectiveRFunDt$new( +# fun = function(xdt) { +# p = learner$predict_newdata(fix_NA_and_chars(xdt)) +# mu = p$response +# se = p$se +# d = lower - mu +# d_norm = d / se +# ei = d * pnorm(d_norm) + se * dnorm(d_norm) +# ei = ifelse(se < 1e-20, 0, ei) +# data.table(acq_ei = ei) +# }, +# domain = ss, +# codomain = ps(acq_ei = p_dbl(tags = "maximize")) +# ) +# f_instance = OptimInstanceSingleCrit$new( +# objective = f, +# search_space = ss, +# terminator = trm("evals", n_evals = n_evals) +# ) +# f_instance +#} + +# Log EI +make_greedy_f_instance = function(ss, learner, n_evals, lower) { ss = ss$clone(deep = TRUE) ss$trafo = NULL f = ObjectiveRFunDt$new( - fun = function(xdt) data.table(mean = learner$predict_newdata(xdt)$response), + fun = function(xdt) { + p = learner$predict_newdata(fix_NA_and_chars(xdt)) + mu = p$response + se = p$se + d = lower - mu + d_norm = d / se + ei = (exp(lower) * pnorm(d_norm)) - (exp((0.5 * se ^ 2) + mu) * pnorm(d_norm - se)) + ei = ifelse(se < 1e-20, 0, ei) + data.table(acq_log_ei = ei) + }, domain = ss, - codomain = ps(mean = p_dbl(tags = "minimize")) + codomain = ps(acq_log_ei = p_dbl(tags = "maximize")) ) f_instance = OptimInstanceSingleCrit$new( objective = f, @@ -126,3 +233,16 @@ make_greedy_f_instance = function(ss, learner, n_evals) { ) f_instance } + +fix_NA_and_chars = function(xydt) { + chr_cols = names(xydt)[map_chr(xydt, class) == "character"] + if (length(chr_cols)) { + xydt[, `:=`((chr_cols), map(.SD, as_factor_NA_fix)), .SDcols = chr_cols] + } + xydt +} + +as_factor_NA_fix = function(x) { + x[is.na(x)] = ".MISSING" + as.factor(x) +} From 5590bb5ff0fe3e7e7792fafe4bb92161b795269d Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Tue, 17 Jan 2023 00:07:49 +0100 Subject: [PATCH 035/126] .. --- attic/so_config/LearnerRegrTraforest.R | 163 ------------- attic/so_config/run_yahpo.R | 72 +----- attic/so_config/run_yahpo_trafbo.R | 304 +++++++++++++++++++++++++ 3 files changed, 313 insertions(+), 226 deletions(-) delete mode 100644 attic/so_config/LearnerRegrTraforest.R create mode 100644 attic/so_config/run_yahpo_trafbo.R diff --git a/attic/so_config/LearnerRegrTraforest.R b/attic/so_config/LearnerRegrTraforest.R deleted file mode 100644 index a402352f..00000000 --- a/attic/so_config/LearnerRegrTraforest.R +++ /dev/null @@ -1,163 +0,0 @@ -#' @title Ranger Regression Learner Custom -#' -#' @name mlr_learners_regr.ranger_custom -#' -#' @description -#' Random regression forest. -#' Calls [ranger::ranger()] from package \CRANpkg{ranger}. -#' -#' @inheritSection mlr_learners_classif.ranger Custom mlr3 parameters -#' @inheritSection mlr_learners_classif.ranger Initial parameter values -#' -#' @templateVar id regr.ranger_custom -#' @template learner -#' -#' @references -#' `r format_bib("wright_2017", "breiman_2001")` -#' -#' @export -#' @template seealso_learner -#' @template example -LearnerRegrRangerCustom = R6Class("LearnerRegrRangerCustom", - inherit = LearnerRegr, - - public = list( - - #' @description - #' Creates a new instance of this [R6][R6::R6Class] class. - initialize = function() { - ps = ps( - alpha = p_dbl(default = 0.5, tags = "train"), - always.split.variables = p_uty(tags = "train"), - holdout = p_lgl(default = FALSE, tags = "train"), - importance = p_fct(c("none", "impurity", "impurity_corrected", "permutation"), tags = "train"), - keep.inbag = p_lgl(default = FALSE, tags = "train"), - max.depth = p_int(default = NULL, lower = 0L, special_vals = list(NULL), tags = "train"), - min.node.size = p_int(1L, default = 5L, special_vals = list(NULL), tags = "train"), - min.prop = p_dbl(default = 0.1, tags = "train"), - minprop = p_dbl(default = 0.1, tags = "train"), - mtry = p_int(lower = 1L, special_vals = list(NULL), tags = "train"), - mtry.ratio = p_dbl(lower = 0, upper = 1, tags = "train"), - num.random.splits = p_int(1L, default = 1L, tags = "train"), - num.threads = p_int(1L, default = 1L, tags = c("train", "predict", "threads")), - num.trees = p_int(1L, default = 500L, tags = c("train", "predict", "hotstart")), - oob.error = p_lgl(default = TRUE, tags = "train"), - quantreg = p_lgl(default = FALSE, tags = "train"), - regularization.factor = p_uty(default = 1, tags = "train"), - regularization.usedepth = p_lgl(default = FALSE, tags = "train"), - replace = p_lgl(default = TRUE, tags = "train"), - respect.unordered.factors = p_fct(c("ignore", "order", "partition"), default = "ignore", tags = "train"), - sample.fraction = p_dbl(0L, 1L, tags = "train"), - save.memory = p_lgl(default = FALSE, tags = "train"), - scale.permutation.importance = p_lgl(default = FALSE, tags = "train"), - se.method = p_fct(c("jack", "infjack", "simple"), default = "infjack", tags = "predict"), # FIXME: only works if predict_type == "se". How to set dependency? - seed = p_int(default = NULL, special_vals = list(NULL), tags = c("train", "predict")), - split.select.weights = p_uty(default = NULL, tags = "train"), - splitrule = p_fct(c("variance", "extratrees", "maxstat"), default = "variance", tags = "train"), - verbose = p_lgl(default = TRUE, tags = c("train", "predict")), - write.forest = p_lgl(default = TRUE, tags = "train") - ) - - ps$values = list(num.threads = 1L) - - # deps - ps$add_dep("num.random.splits", "splitrule", CondEqual$new("extratrees")) - ps$add_dep("alpha", "splitrule", CondEqual$new("maxstat")) - ps$add_dep("minprop", "splitrule", CondEqual$new("maxstat")) - ps$add_dep("scale.permutation.importance", "importance", CondEqual$new("permutation")) - - - super$initialize( - id = "regr.ranger_custom", - param_set = ps, - predict_types = c("response", "se"), - feature_types = c("logical", "integer", "numeric", "character", "factor", "ordered"), - properties = c("weights", "importance", "oob_error", "hotstart_backward"), - packages = c("mlr3learners", "ranger"), - man = "mlr3learners::mlr_learners_regr.ranger_custom" - ) - }, - - #' @description - #' The importance scores are extracted from the model slot `variable.importance`. - #' Parameter `importance.mode` must be set to `"impurity"`, `"impurity_corrected"`, or - #' `"permutation"` - #' - #' @return Named `numeric()`. - importance = function() { - if (is.null(self$model)) { - stopf("No model stored") - } - if (self$model$importance.mode == "none") { - stopf("No importance stored") - } - - sort(self$model$variable.importance, decreasing = TRUE) - }, - - #' @description - #' The out-of-bag error, extracted from model slot `prediction.error`. - #' - #' @return `numeric(1)`. - oob_error = function() { - if (is.null(self$model)) { - stopf("No model stored") - } - self$model$prediction.error - } - ), - - private = list( - .train = function(task) { - pv = self$param_set$get_values(tags = "train") - pv = mlr3learners:::convert_ratio(pv, "mtry", "mtry.ratio", length(task$feature_names)) - - if (self$predict_type == "se") { - pv$keep.inbag = TRUE # nolint - } - - mlr3misc::invoke(ranger::ranger, - dependent.variable.name = task$target_names, - data = task$data(), - case.weights = task$weights$weight, - .args = pv - ) - }, - - .predict = function(task) { - pv = self$param_set$get_values(tags = "predict") - newdata = mlr3learners:::ordered_features(task, self) - - if (isTRUE(pv$se.method == "simple")) { - # the variance is only correct if min.node.size = 1 and therefore sigma^2_b is always 0 or rather undefined (as - # there is always only a single observation in each leaf) - sigma_min = 0.01 - prediction = mlr3misc::invoke(predict, self$model, data = newdata, type = "response", .args = pv[setdiff(names(pv), "se.method")], predict.all = TRUE) - response = rowMeans(prediction$predictions) - tmp = sigma_min ^ 2 + prediction$predictions ^ 2 - variance = rowMeans(tmp) - response ^ 2 - list(response = response, se = sqrt(variance)) - } else { - prediction = mlr3misc::invoke(predict, self$model, data = newdata, type = self$predict_type, .args = pv) - list(response = prediction$predictions, se = prediction$se) - } - }, - - .hotstart = function(task) { - model = self$model - model$num.trees = self$param_set$values$num.trees - model - } - ) -) - -#' @export -default_values.LearnerRegrRangerCustom = function(x, search_space, task, ...) { # nolint - special_defaults = list( - mtry = floor(sqrt(length(task$feature_names))), - mtry.ratio = floor(sqrt(length(task$feature_names))) / length(task$feature_names), - sample.fraction = 1 - ) - defaults = insert_named(default_values(x$param_set), special_defaults) - defaults[search_space$ids()] -} diff --git a/attic/so_config/run_yahpo.R b/attic/so_config/run_yahpo.R index 04f0e3f9..64d19594 100644 --- a/attic/so_config/run_yahpo.R +++ b/attic/so_config/run_yahpo.R @@ -7,16 +7,15 @@ library(mlr3misc) library(mlr3mbo) # @so_config library(bbotk) # @localsearch library(paradox) -library(miesmuschel) +library(miesmuschel) # @mlr3mbo_config library(R6) library(checkmate) -library(trtf) -reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) +reticulate::use_condaenv("/home/lschnei8/.conda/envs/env", required = TRUE) library(reticulate) yahpo_gym = import("yahpo_gym") -packages = c("data.table", "mlr3", "mlr3learners", "mlr3pipelines", "mlr3misc", "mlr3mbo", "bbotk", "paradox", "mlrintermbo", "R6", "checkmate", "trtf") +packages = c("data.table", "mlr3", "mlr3learners", "mlr3pipelines", "mlr3misc", "mlr3mbo", "bbotk", "paradox", "miesmuschel", "R6", "checkmate") #RhpcBLASctl::blas_set_num_threads(1L) #RhpcBLASctl::omp_set_num_threads(1L) @@ -24,17 +23,17 @@ packages = c("data.table", "mlr3", "mlr3learners", "mlr3pipelines", "mlr3misc", root = here::here() experiments_dir = file.path(root) -source_files = map_chr(c("helpers.R", "LearnerRegrRangerCustom.R", "OptimizerChain.R", "trafbo.R"), function(x) file.path(experiments_dir, x)) +source_files = map_chr(c("helpers.R", "AcqFunctionLogEI", "LearnerRegrRangerCustom.R", "OptimizerChain.R", "trafbo.R"), function(x) file.path(experiments_dir, x)) for (sf in source_files) { source(sf) } -reg = makeExperimentRegistry(file.dir = "/gscratch/lschnei8/registry_mlr3mbo_so_config_trafbo", packages = packages, source = source_files) +reg = makeExperimentRegistry(file.dir = "/gscratch/lschnei8/registry_mlr3mbo_so_config", packages = packages, source = source_files) #reg = makeExperimentRegistry(file.dir = NA, conf.file = NA, packages = packages, source = source_files) # interactive session saveRegistry(reg) mlr3mbo_wrapper = function(job, data, instance, ...) { - reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) + reticulate::use_condaenv("/home/lschnei8/.conda/envs/env", required = TRUE) library(yahpogym) logger = lgr::get_logger("bbotk") logger$set_threshold("warn") @@ -42,7 +41,7 @@ mlr3mbo_wrapper = function(job, data, instance, ...) { optim_instance = make_optim_instance(instance) - xdt = data.table(loop_function = "ego_log", init = "random", init_size_fraction = "0.25", random_interleave = TRUE, random_interleave_iter = "4", rf_type = "smaclike_boot", acqf = "EI", lambda = NA_real_, acqopt = "LS") + xdt = data.table(loop_function = "ego_log", init = "random", init_size_fraction = "0.25", random_interleave = TRUE, random_interleave_iter = "4", rf_type = "smaclike_boot", acqf = "LogEI", lambda = NA_real_, acqopt = "LS") d = optim_instance$search_space$length init_design_size = ceiling(as.numeric(xdt$init_size_fraction) * optim_instance$terminator$param_set$values$n_evals) @@ -115,6 +114,8 @@ mlr3mbo_wrapper = function(job, data, instance, ...) { acq_function = if (xdt$acqf == "EI") { AcqFunctionEI$new() + } else if (xdt$acqf == "LogEI") { + AcqFunctionLogEI$new() } else if (xdt$acqf == "CB") { AcqFunctionCB$new(lambda = xdt$lambda) } else if (xdt$acqf == "PI") { @@ -132,63 +133,8 @@ mlr3mbo_wrapper = function(job, data, instance, ...) { optim_instance } -trafbo_wrapper = function(job, data, instance, ...) { - reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) - library(yahpogym) - logger = lgr::get_logger("bbotk") - logger$set_threshold("warn") - future::plan("sequential") - - optim_instance = make_optim_instance(instance) - - bayesopt_trafbo(optim_instance) - - optim_instance -} - -mlr3mbo_gp_wrapper = function(job, data, instance, ...) { - reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) - library(yahpogym) - logger = lgr::get_logger("bbotk") - logger$set_threshold("warn") - future::plan("sequential") - - optim_instance = make_optim_instance(instance) - - d = optim_instance$search_space$length - init_design_size = ceiling(0.25 * optim_instance$terminator$param_set$values$n_evals) - init_design = generate_design_sobol(optim_instance$search_space, n = init_design_size)$data - - optim_instance$eval_batch(init_design) - - random_interleave_iter = 0L - - learner = LearnerRegrKM$new() - learner$predict_type = "se" - learner$param_set$values$covtype = "matern3_2" - learner$param_set$values$optim.method = "gen" - learner$param_set$values$control = list(trace = FALSE) - learner$param_set$values$nugget.stability = 10^-8 - - surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor", multiplier = 3) %>>% learner)) - - optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) - acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20020L)) - acq_optimizer$param_set$values$warmstart = TRUE - acq_optimizer$param_set$values$warmstart_size = 10L - acq_optimizer - - acq_function = AcqFunctionEI$new() - - bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) - - optim_instance -} - # add algorithms addAlgorithm("mlr3mbo", fun = mlr3mbo_wrapper) -addAlgorithm("trafbo", fun = trafbo_wrapper) -addAlgorithm("mlr3mbo_gp", fun = mlr3mbo_gp_wrapper) # setup scenarios and instances get_nb301_setup = function(budget_factor = 40L) { diff --git a/attic/so_config/run_yahpo_trafbo.R b/attic/so_config/run_yahpo_trafbo.R new file mode 100644 index 00000000..5752220f --- /dev/null +++ b/attic/so_config/run_yahpo_trafbo.R @@ -0,0 +1,304 @@ +library(batchtools) +library(data.table) +library(mlr3) +library(mlr3learners) +library(mlr3pipelines) +library(mlr3misc) +library(mlr3mbo) # @so_config +library(bbotk) # @localsearch +library(paradox) +library(miesmuschel) # @mlr3mbo_config +library(R6) +library(checkmate) +library(trtf) + +reticulate::use_condaenv("/home/lschnei8/.conda/envs/env", required = TRUE) +library(reticulate) +yahpo_gym = import("yahpo_gym") + +packages = c("data.table", "mlr3", "mlr3learners", "mlr3pipelines", "mlr3misc", "mlr3mbo", "bbotk", "paradox", "miesmuschel", "R6", "checkmate", "trtf") + +#RhpcBLASctl::blas_set_num_threads(1L) +#RhpcBLASctl::omp_set_num_threads(1L) + +root = here::here() +experiments_dir = file.path(root) + +source_files = map_chr(c("helpers.R", "AcqFunctionLogEI", "LearnerRegrRangerCustom.R", "OptimizerChain.R", "trafbo.R"), function(x) file.path(experiments_dir, x)) +for (sf in source_files) { + source(sf) +} + +reg = makeExperimentRegistry(file.dir = "/gscratch/lschnei8/registry_mlr3mbo_so_config", packages = packages, source = source_files) +#reg = makeExperimentRegistry(file.dir = NA, conf.file = NA, packages = packages, source = source_files) # interactive session +saveRegistry(reg) + +mlr3mbo_wrapper = function(job, data, instance, ...) { + reticulate::use_condaenv("/home/lschnei8/.conda/envs/env", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + optim_instance = make_optim_instance(instance) + + xdt = data.table(loop_function = "ego_log", init = "random", init_size_fraction = "0.25", random_interleave = TRUE, random_interleave_iter = "4", rf_type = "smaclike_boot", acqf = "LogEI", lambda = NA_real_, acqopt = "LS") + + d = optim_instance$search_space$length + init_design_size = ceiling(as.numeric(xdt$init_size_fraction) * optim_instance$terminator$param_set$values$n_evals) + init_design = if (xdt$init == "random") { + generate_design_random(optim_instance$search_space, n = init_design_size)$data + } else if (xdt$init == "lhs") { + generate_design_lhs(optim_instance$search_space, n = init_design_size)$data + } else if (xdt$init == "sobol") { + generate_design_sobol(optim_instance$search_space, n = init_design_size)$data + } + + optim_instance$eval_batch(init_design) + + random_interleave_iter = if(xdt$random_interleave) as.numeric(xdt$random_interleave_iter) else 0L + + learner = LearnerRegrRangerCustom$new() + learner$predict_type = "se" + learner$param_set$values$keep.inbag = TRUE + + if (xdt$rf_type == "standard") { + learner$param_set$values$se.method = "jack" + learner$param_set$values$splitrule = "variance" + learner$param_set$values$num.trees = 1000L + } else if (xdt$rf_type == "smaclike_boot") { + learner$param_set$values$se.method = "simple" + learner$param_set$values$splitrule = "extratrees" + learner$param_set$values$num.random.splits = 1L + learner$param_set$values$num.trees = 10L + learner$param_set$values$replace = TRUE + learner$param_set$values$sample.fraction = 1 + learner$param_set$values$min.node.size = 1 + learner$param_set$values$mtry.ratio = 1 + } else if (xdt$rf_type == "smaclike_no_boot") { + learner$param_set$values$se.method = "simple" + learner$param_set$values$splitrule = "extratrees" + learner$param_set$values$num.random.splits = 1L + learner$param_set$values$num.trees = 10L + learner$param_set$values$replace = FALSE + learner$param_set$values$sample.fraction = 1 + learner$param_set$values$min.node.size = 1 + learner$param_set$values$mtry.ratio = 1 + } else if (xdt$rf_type == "smaclike_variance_boot") { + learner$param_set$values$se.method = "simple" + learner$param_set$values$splitrule = "variance" + learner$param_set$values$num.trees = 10L + learner$param_set$values$replace = TRUE + learner$param_set$values$sample.fraction = 1 + learner$param_set$values$min.node.size = 1 + learner$param_set$values$mtry.ratio = 1 + } + + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor", multiplier = 3) %>>% learner)) + + acq_optimizer = if (xdt$acqopt == "RS_1000") { + AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 1000L)) + } else if (xdt$acqopt == "RS") { + AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 20000L)) + } else if (xdt$acqopt == "FS") { + n_repeats = 2L + maxit = 9L + batch_size = ceiling((20000L / n_repeats) / (1 + maxit)) # 1000L + AcqOptimizer$new(opt("focus_search", n_points = batch_size, maxit = maxit), terminator = trm("evals", n_evals = 20000L)) + } else if (xdt$acqopt == "LS") { + optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) + acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20020L)) + acq_optimizer$param_set$values$warmstart = TRUE + acq_optimizer$param_set$values$warmstart_size = 10L + acq_optimizer + } + + acq_function = if (xdt$acqf == "EI") { + AcqFunctionEI$new() + } else if (xdt$acqf == "LogEI") { + AcqFunctionLogEI$new() + } else if (xdt$acqf == "CB") { + AcqFunctionCB$new(lambda = xdt$lambda) + } else if (xdt$acqf == "PI") { + AcqFunctionPI$new() + } else if (xdt$acqf == "Mean") { + AcqFunctionMean$new() + } + + if (xdt$loop_function == "ego") { + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + } else if (xdt$loop_function == "ego_log") { + bayesopt_ego_log(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + } + + optim_instance +} + +trafbo_wrapper = function(job, data, instance, ...) { + reticulate::use_condaenv("/home/lschnei8/.conda/envs/env", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + optim_instance = make_optim_instance(instance) + + bayesopt_trafbo(optim_instance) + + optim_instance +} + +mlr3mbo_gp_wrapper = function(job, data, instance, ...) { + reticulate::use_condaenv("/home/lschnei8/.conda/envs/env", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + optim_instance = make_optim_instance(instance) + + d = optim_instance$search_space$length + init_design_size = ceiling(0.25 * optim_instance$terminator$param_set$values$n_evals) + init_design = generate_design_sobol(optim_instance$search_space, n = init_design_size)$data + + optim_instance$eval_batch(init_design) + + random_interleave_iter = 0L + + learner = LearnerRegrKM$new() + learner$predict_type = "se" + learner$param_set$values$covtype = "matern3_2" + learner$param_set$values$optim.method = "gen" + learner$param_set$values$control = list(trace = FALSE) + learner$param_set$values$nugget.stability = 10^-8 + + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor", multiplier = 3) %>>% learner)) + + optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) + acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20020L)) + acq_optimizer$param_set$values$warmstart = TRUE + acq_optimizer$param_set$values$warmstart_size = 10L + acq_optimizer + + acq_function = AcqFunctionEI$new() + + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + + optim_instance +} + +# add algorithms +addAlgorithm("mlr3mbo", fun = mlr3mbo_wrapper) +#addAlgorithm("trafbo", fun = trafbo_wrapper) +#addAlgorithm("mlr3mbo_gp", fun = mlr3mbo_gp_wrapper) + +# setup scenarios and instances +get_nb301_setup = function(budget_factor = 40L) { + scenario = "nb301" + bench = yahpo_gym$benchmark_set$BenchmarkSet(scenario, instance = "CIFAR10") + fidelity_space = bench$get_fidelity_space() + fidelity_param_id = fidelity_space$get_hyperparameter_names()[1] + min_budget = fidelity_space$get_hyperparameter(fidelity_param_id)$lower + max_budget = fidelity_space$get_hyperparameter(fidelity_param_id)$upper + ndim = length(bench$config_space$get_hyperparameter_names()) - 1L # NOTE: instance is not part of + + instances = "CIFAR10" + target = "val_accuracy" + budget = ceiling(20L * max_budget + sqrt(ndim) * max_budget * budget_factor) + on_integer_scale = TRUE + minimize = bench$config$config$y_minimize[match(target, bench$config$config$y_names)] + setup = setDT(expand.grid(scenario = scenario, instance = instances, target = target, ndim = ndim, max_budget = max_budget, budget = budget, on_integer_scale = on_integer_scale, minimize = minimize, stringsAsFactors = FALSE)) + setup +} + +get_lcbench_setup = function(budget_factor = 40L) { + scenario = "lcbench" + bench = yahpo_gym$benchmark_set$BenchmarkSet(scenario, instance = "167168") + fidelity_space = bench$get_fidelity_space() + fidelity_param_id = fidelity_space$get_hyperparameter_names()[1] + min_budget = fidelity_space$get_hyperparameter(fidelity_param_id)$lower + max_budget = fidelity_space$get_hyperparameter(fidelity_param_id)$upper + ndim = length(bench$config_space$get_hyperparameter_names()) - 2L + + instances = c("167168", "189873", "189906") + target = "val_accuracy" + budget = ceiling(20L * max_budget + sqrt(ndim) * max_budget * budget_factor) + on_integer_scale = TRUE + minimize = bench$config$config$y_minimize[match(target, bench$config$config$y_names)] + setup = setDT(expand.grid(scenario = scenario, instance = instances, target = target, ndim = ndim, max_budget = max_budget, budget = budget, on_integer_scale = on_integer_scale, minimize = minimize, stringsAsFactors = FALSE)) + setup +} + +get_rbv2_setup = function(budget_factor = 40L) { + setup = map_dtr(c("rbv2_glmnet", "rbv2_rpart", "rbv2_ranger", "rbv2_xgboost", "rbv2_super"), function(scenario) { + bench = yahpo_gym$benchmark_set$BenchmarkSet(scenario, instance = "1040") + fidelity_space = bench$get_fidelity_space() + fidelity_param_id = "trainsize" + min_budget = fidelity_space$get_hyperparameter(fidelity_param_id)$lower + max_budget = fidelity_space$get_hyperparameter(fidelity_param_id)$upper + ndim = length(bench$config_space$get_hyperparameter_names()) - 3L # repl and trainsize and instance + + instances = switch(scenario, rbv2_glmnet = c("375", "458"), rbv2_rpart = c("14", "40499"), rbv2_ranger = c("16", "42"), rbv2_xgboost = c("12", "1501", "16", "40499"), rbv2_super = c("1053", "1457", "1063", "1479", "15", "1468")) + target = "acc" + budget = ceiling(20L * max_budget + sqrt(ndim) * max_budget * budget_factor) + on_integer_scale = FALSE + minimize = bench$config$config$y_minimize[match(target, bench$config$config$y_names)] + setup = setDT(expand.grid(scenario = scenario, instance = instances, target = target, ndim = ndim, max_budget = max_budget, budget = budget, on_integer_scale = on_integer_scale, minimize = minimize, stringsAsFactors = FALSE)) + }) +} + +setup = rbind(get_nb301_setup(), get_lcbench_setup(), get_rbv2_setup()) + +setup[, id := seq_len(.N)] + +# add problems +prob_designs = map(seq_len(nrow(setup)), function(i) { + prob_id = paste0(setup[i, ]$scenario, "_", setup[i, ]$instance, "_", setup[i, ]$target) + addProblem(prob_id, data = list(scenario = setup[i, ]$scenario, instance = setup[i, ]$instance, target = setup[i, ]$target, ndim = setup[i, ]$ndim, max_budget = setup[i, ]$max_budget, budget = setup[i, ]$budget, on_integer_scale = setup[i, ]$on_integer_scale, minimize = setup[i, ]$minimize)) + setNames(list(setup[i, ]), nm = prob_id) +}) +nn = sapply(prob_designs, names) +prob_designs = unlist(prob_designs, recursive = FALSE, use.names = FALSE) +names(prob_designs) = nn + +# add jobs for optimizers +optimizers = data.table(algorithm = c("mlr3mbo", "trafbo", "mlr3mbo_gp")) + +for (i in seq_len(nrow(optimizers))) { + algo_designs = setNames(list(optimizers[i, ]), nm = optimizers[i, ]$algorithm) + + ids = addExperiments( + prob.designs = prob_designs, + algo.designs = algo_designs, + repls = 30L + ) + addJobTags(ids, as.character(optimizers[i, ]$algorithm)) +} + +jobs = getJobTable() +jobs = jobs[grepl("lcbench", x = problem) & tags == "trafbo", ] +resources.default = list(walltime = 3600 * 3L, memory = 2048L, ntasks = 1L, ncpus = 1L, nodes = 1L, clusters = "teton", max.concurrent.jobs = 9999L) +submitJobs(jobs, resources = resources.default) + +done = findDone() +results = reduceResultsList(done, function(x, job) { + x = x$archive$data + budget_var = if (job$instance$scenario %in% c("lcbench", "nb301")) "epoch" else "trainsize" + target_var = job$instance$target + if (!job$instance$minimize) { + x[, (target_var) := - get(target_var)] + } + pars = job$pars + tmp = x[, target_var, with = FALSE] + tmp[, (budget_var) := job$instance$max_budget] + tmp[, method := pars$algo.pars$algorithm] + tmp[, scenario := pars$prob.pars$scenario] + tmp[, instance := pars$prob.pars$instance] + tmp[, repl := job$repl] + tmp[, iter := seq_len(.N)] + colnames(tmp) = c("target", "budget", "method", "scenario", "instance", "repl", "iter") + tmp +}) +results = rbindlist(results, fill = TRUE) +saveRDS(results, "results_yahpo_trafbo.rds") + From 61287cc30a4c48708cddbd8aafc7e81c5ce2f74b Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Tue, 17 Jan 2023 11:07:32 +0100 Subject: [PATCH 036/126] .. --- attic/so_config/LearnerRegrRangerCustom.R | 2 +- attic/so_config/analyze_yahpo.R | 6 ++---- attic/so_config/run_yahpo.R | 17 ++++++++--------- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/attic/so_config/LearnerRegrRangerCustom.R b/attic/so_config/LearnerRegrRangerCustom.R index 78633ec5..1a394684 100644 --- a/attic/so_config/LearnerRegrRangerCustom.R +++ b/attic/so_config/LearnerRegrRangerCustom.R @@ -131,7 +131,7 @@ LearnerRegrRangerCustom = R6Class("LearnerRegrRangerCustom", if (isTRUE(pv$se.method == "simple")) { prediction = mlr3misc::invoke(predict, self$model, data = newdata, type = "response", .args = pv[setdiff(names(pv), "se.method")], predict.all = TRUE) response = rowMeans(prediction$predictions) - variance = apply(prediction$predictions, MARGIN = 1L, var) + variance = rowMeans(0.01 + prediction$predictions ^ 2) - (response ^ 2) # law of total variance assuming sigma_b(x) is 0 due to min.node.size = 1 and always splitting; then set 0.01 as lower bound for sigma_b(x) list(response = response, se = sqrt(variance)) } else { prediction = mlr3misc::invoke(predict, self$model, data = newdata, type = self$predict_type, .args = pv) diff --git a/attic/so_config/analyze_yahpo.R b/attic/so_config/analyze_yahpo.R index 3f55faee..60e80a8b 100644 --- a/attic/so_config/analyze_yahpo.R +++ b/attic/so_config/analyze_yahpo.R @@ -3,10 +3,8 @@ library(ggplot2) library(pammtools) library(mlr3misc) -dat = rbind(readRDS("results_yahpo.rds"), readRDS("results_yahpo_trafbo.rds")) -dat[method == "trafbo", method := "trafbo_new"] -dat = dat[scenario %in% "lcbench"] -dat = dat[method %in% c("random", "smac4hpo", "mlr3mbo", "mlr3mbo_gp", "trafbo", "trafbo_new")] +dat = rbind(readRDS("results_yahpo.rds"), readRDS("results_yahpo_mlr3mbo.rds")) +dat = dat[method %in% c("random", "smac4hpo", "mlr3mbo")] dat[, cumbudget := cumsum(budget), by = .(method, scenario, instance, repl)] dat[, cumbudget_scaled := cumbudget / max(cumbudget), by = .(method, scenario, instance, repl)] dat[, normalized_regret := (target - min(target)) / (max(target) - min(target)), by = .(scenario, instance)] diff --git a/attic/so_config/run_yahpo.R b/attic/so_config/run_yahpo.R index 64d19594..3392b282 100644 --- a/attic/so_config/run_yahpo.R +++ b/attic/so_config/run_yahpo.R @@ -23,7 +23,7 @@ packages = c("data.table", "mlr3", "mlr3learners", "mlr3pipelines", "mlr3misc", root = here::here() experiments_dir = file.path(root) -source_files = map_chr(c("helpers.R", "AcqFunctionLogEI", "LearnerRegrRangerCustom.R", "OptimizerChain.R", "trafbo.R"), function(x) file.path(experiments_dir, x)) +source_files = map_chr(c("helpers.R", "AcqFunctionLogEI.R", "LearnerRegrRangerCustom.R", "OptimizerChain.R"), function(x) file.path(experiments_dir, x)) for (sf in source_files) { source(sf) } @@ -41,7 +41,7 @@ mlr3mbo_wrapper = function(job, data, instance, ...) { optim_instance = make_optim_instance(instance) - xdt = data.table(loop_function = "ego_log", init = "random", init_size_fraction = "0.25", random_interleave = TRUE, random_interleave_iter = "4", rf_type = "smaclike_boot", acqf = "LogEI", lambda = NA_real_, acqopt = "LS") + xdt = data.table(loop_function = "ego_log", init = "sobol", init_size_fraction = "0.25", random_interleave = TRUE, random_interleave_iter = "5", rf_type = "smaclike_boot", acqf = "LogEI", lambda = NA_real_, acqopt = "LS") d = optim_instance$search_space$length init_design_size = ceiling(as.numeric(xdt$init_size_fraction) * optim_instance$terminator$param_set$values$n_evals) @@ -105,10 +105,10 @@ mlr3mbo_wrapper = function(job, data, instance, ...) { batch_size = ceiling((20000L / n_repeats) / (1 + maxit)) # 1000L AcqOptimizer$new(opt("focus_search", n_points = batch_size, maxit = maxit), terminator = trm("evals", n_evals = 20000L)) } else if (xdt$acqopt == "LS") { - optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) - acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20020L)) + optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10000L), trm("evals", n_evals = 10000L))) + acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20000L)) acq_optimizer$param_set$values$warmstart = TRUE - acq_optimizer$param_set$values$warmstart_size = 10L + acq_optimizer$param_set$values$warmstart_size = "all" acq_optimizer } @@ -206,7 +206,7 @@ prob_designs = unlist(prob_designs, recursive = FALSE, use.names = FALSE) names(prob_designs) = nn # add jobs for optimizers -optimizers = data.table(algorithm = c("mlr3mbo", "trafbo", "mlr3mbo_gp")) +optimizers = data.table(algorithm = "mlr3mbo") for (i in seq_len(nrow(optimizers))) { algo_designs = setNames(list(optimizers[i, ]), nm = optimizers[i, ]$algorithm) @@ -220,8 +220,7 @@ for (i in seq_len(nrow(optimizers))) { } jobs = getJobTable() -jobs = jobs[grepl("lcbench", x = problem) & tags == "trafbo", ] -resources.default = list(walltime = 3600 * 3L, memory = 2048L, ntasks = 1L, ncpus = 1L, nodes = 1L, clusters = "teton", max.concurrent.jobs = 9999L) +resources.default = list(walltime = 3600 * 12L, memory = 2048L, ntasks = 1L, ncpus = 1L, nodes = 1L, clusters = "beartooth", max.concurrent.jobs = 9999L) submitJobs(jobs, resources = resources.default) done = findDone() @@ -244,5 +243,5 @@ results = reduceResultsList(done, function(x, job) { tmp }) results = rbindlist(results, fill = TRUE) -saveRDS(results, "results_yahpo_trafbo.rds") +saveRDS(results, "/gscratch/lschnei8/results_yahpo_mlr3mbo.rds") From 7c2a3b3bcc59273634cf8cd3c22b5df3bf198bfd Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Tue, 17 Jan 2023 15:30:45 +0100 Subject: [PATCH 037/126] .. --- attic/so_config/OptimizerCoordinateDescent.R | 33 +++--- attic/so_config/cd_example.R | 3 +- attic/so_config/coordinate_descent.R | 108 +++++++++---------- attic/so_config/run_yahpo.R | 25 +++-- 4 files changed, 81 insertions(+), 88 deletions(-) diff --git a/attic/so_config/OptimizerCoordinateDescent.R b/attic/so_config/OptimizerCoordinateDescent.R index 2e58dd62..dfb677e0 100644 --- a/attic/so_config/OptimizerCoordinateDescent.R +++ b/attic/so_config/OptimizerCoordinateDescent.R @@ -35,9 +35,6 @@ OptimizerCoordinateDescent = R6Class("OptimizerCoordinateDescent", inherit = bbo gen = gen + 1L for (param_id in shuffle(inst$search_space$ids())) { if (!is.na(incumbent[[param_id]])) { - # we first also reevaluate the incumbent to see how noisy the evaluations are - inst$eval_batch(incumbent) - xdt = get_xdt_coordinate(copy(incumbent), param = inst$search_space$params[[param_id]], n_coordinate_tryouts = n_coordinate_tryouts) # previously inactive parameters can now be active and need a value if (inst$search_space$has_deps & param_id %in% inst$search_space$deps$on) { @@ -48,13 +45,14 @@ OptimizerCoordinateDescent = R6Class("OptimizerCoordinateDescent", inherit = bbo } } xdt = Design$new(inst$search_space, data = xdt, remove_dupl = TRUE)$data # fixes potentially broken dependencies - set(xdt, j = ".gen", value = gen) set(xdt, j = ".param", value = param_id) + xdt = rbind(xdt, incumbent, fill = TRUE) # also reevaluate the incumbent to see how noisy evaluations are + set(xdt, j = ".gen", value = gen) + set(xdt, i = nrow(xdt), j = ".param", value = "incumbent") + set(xdt, j = "incumbent", value = list(inst$archive$best()[, c(inst$archive$cols_x, inst$archive$cols_y), with = FALSE])) inst$eval_batch(xdt) - #for (i in seq_len(nrow(xdt))) { # could also do this according to a batch_size parameter - # inst$eval_batch(xdt[i, ]) - #} incumbent = inst$archive$best()[, inst$archive$cols_x, with = FALSE] + saveRDS(inst, file = self$param_set$values$rds_name) } } @@ -67,28 +65,29 @@ OptimizerCoordinateDescent = R6Class("OptimizerCoordinateDescent", inherit = bbo ) get_xdt_coordinate = function(incumbent, param, n_coordinate_tryouts) { - if (param$class %in% c("ParamDbl", "ParamInt")) { + if (param$class == "ParamDbl") { x = runif(n = n_coordinate_tryouts, min = param$lower, max = param$upper) - if (param$class == "ParamInt") { - x = as.integer(round(x, 0L)) - } + } else if (param$class == "ParamInt") { + levels = setdiff(seq(param$lower, param$upper, by = 1L), incumbent[[param$id]]) + n_coordinate_tryouts = min(n_coordinate_tryouts, length(levels)) + x = sample(levels, size = n_coordinate_tryouts, replace = FALSE) } else { n_coordinate_tryouts = min(n_coordinate_tryouts, param$nlevels - 1L) x = sample(x = setdiff(param$levels, incumbent[[param$id]]), size = n_coordinate_tryouts, replace = FALSE) } - xdt = incumbent[rep(1, n_coordinate_tryouts), ] + xdt = incumbent[rep(1L, n_coordinate_tryouts), ] set(xdt, j = param$id, value = x) xdt } sample_random = function(param, n) { - if (param$class %in% c("ParamDbl", "ParamInt")) { + if (param$class == "ParamDbl") { x = runif(n = n, min = param$lower, max = param$upper) - if (param$class == "ParamInt") { - x = as.integer(round(x, 0L)) - } + } else if (param$class == "ParamInt") { + levels = seq(param$lower, param$upper, by = 1L) + x = sample(levels, size = n, replace = TRUE) } else { - x = sample(x = param$levels, size = n, replace = TRUE) + x = sample(param$levels, size = n, replace = TRUE) } x } diff --git a/attic/so_config/cd_example.R b/attic/so_config/cd_example.R index 494cc6c8..51f29a65 100644 --- a/attic/so_config/cd_example.R +++ b/attic/so_config/cd_example.R @@ -24,7 +24,8 @@ objective = ObjectiveRFunDt$new( data.table(y = y) }, domain = domain, - codomain = ps(y = p_dbl(tags = "minimize"))) + codomain = ps(y = p_dbl(tags = "minimize")) +) instance = OptimInstanceSingleCrit$new(objective = objective, terminator = trm("none")) optimizer = OptimizerCoordinateDescent$new() diff --git a/attic/so_config/coordinate_descent.R b/attic/so_config/coordinate_descent.R index e18a8f7c..3ea77809 100644 --- a/attic/so_config/coordinate_descent.R +++ b/attic/so_config/coordinate_descent.R @@ -2,15 +2,17 @@ # chmod ug+x library(data.table) library(mlr3) -library(mlr3misc) library(mlr3learners) library(mlr3pipelines) +library(mlr3misc) +library(mlr3mbo) # @so_config library(bbotk) # @localsearch library(paradox) +library(miesmuschel) # @mlr3mbo_config library(R6) library(checkmate) -library(mlr3mbo) # @so_config -reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) + +reticulate::use_condaenv("/home/lschnei8/.conda/envs/env", required = TRUE) library(reticulate) library(yahpogym) library("future") @@ -18,46 +20,34 @@ library("future.batchtools") library("future.apply") source("OptimizerCoordinateDescent.R") +source("AcqFunctionLogEI.R") source("LearnerRegrRangerCustom.R") source("OptimizerChain.R") search_space = ps( - loop_function = p_fct(c("ego", "ego_log")), - init = p_fct(c("random", "lhs", "sobol")), - init_size_fraction = p_fct(c("0.0625", "0.125", "0.25")), - random_interleave = p_lgl(), - random_interleave_iter = p_fct(c("2", "4", "10"), depends = random_interleave == TRUE), - - rf_type = p_fct(c("standard", "smaclike_boot", "smaclike_no_boot", "smaclike_variance_boot")), - - acqf = p_fct(c("EI", "CB", "PI", "Mean")), - lambda = p_int(lower = 1, upper = 3, depends = acqf == "CB"), - acqopt = p_fct(c("RS_1000", "RS", "FS", "LS")) + loop_function = p_fct(c("ego", "ego_log"), default = "ego"), + init = p_fct(c("random", "lhs", "sobol"), default = "random"), + init_size_fraction = p_fct(c("0.0625", "0.125", "0.25"), default = "0.25"), + random_interleave = p_lgl(default = FALSE), + random_interleave_iter = p_fct(c("2", "5", "10"), default = "10", depends = random_interleave == TRUE), + rf_type = p_fct(c("standard", "extratrees", "smaclike_boot", "smaclike_no_boot"), default = "standard"), + acqf = p_fct(c("EI", "CB", "PI", "Mean"), default = "EI"), + acqf_ei_log = p_lgl(depends = loop_function == "ego_log" && acqf == "EI", default = FALSE), + lambda = p_fct(c("1", "3", "5"), depends = acqf == "CB", default = "1"), + acqopt = p_fct(c("RS_1000", "RS", "FS", "LS"), default = "RS") ) -#instances = setup = data.table(scenario = rep(c("lcbench", paste0("rbv2_", c("aknn", "glmnet", "ranger", "rpart", "super", "svm", "xgboost"))), each = 4L), -# instance = c("167185", "167152", "168910", "189908", -# "40499", "1476", "6", "12", -# "40979", "1501", "40966", "1478", -# "12", "458", "1510", "1515", -# "1478", "40979", "12", "28", -# "41164", "37", "1515", "1510", -# "1478", "1501", "40499", "40979", -# "40984", "40979", "40966", "28"), -# target = rep(c("val_accuracy", "acc"), c(4L, 28L)), -# budget = rep(c(126L, 118L, 90L, 134L, 110L, 267L, 118L, 170L), each = 4L)) - -instances = setup = data.table(scenario = rep(c("lcbench", paste0("rbv2_", c("aknn", "glmnet", "ranger", "rpart", "super", "svm", "xgboost"))), each = 3L), - instance = c("167185", "167152", "168910", #"189908", - "40499", "1476", "6", #"12", - "40979", "1501", "40966", #"1478", - "12", "458", "1510", #"1515", - "1478", "40979", "12", #"28", - "41164", "37", "1515", #"1510", - "1478", "1501", "40499", #"40979", - "40984", "40979", "40966"), #"28"), - target = rep(c("val_accuracy", "acc"), c(3L, 21L)), - budget = rep(c(126L, 118L, 90L, 134L, 110L, 267L, 118L, 170L), each = 3L)) +instances = setup = data.table(scenario = rep(c("lcbench", paste0("rbv2_", c("aknn", "glmnet", "ranger", "rpart", "super", "svm", "xgboost"))), each = 4L), + instance = c("167185", "167152", "168910", "189908", + "40499", "1476", "6", "12", + "40979", "1501", "40966", "1478", + "12", "458", "1510", "1515", + "1478", "40979", "12", "28", + "41164", "37", "1515", "1510", + "1478", "1501", "40499", "40979", + "40984", "40979", "40966", "28"), + target = rep(c("val_accuracy", "acc"), c(4L, 28L)), + budget = rep(c(126L, 118L, 90L, 134L, 110L, 267L, 118L, 170L), each = 4L)) mies_average = readRDS("results_yahpo_mies_average.rds") @@ -67,18 +57,20 @@ evaluate = function(xdt, instance) { library(data.table) library(mlr3) - library(mlr3misc) library(mlr3learners) library(mlr3pipelines) - library(bbotk) + library(mlr3misc) + library(mlr3mbo) # @so_config + library(bbotk) # @localsearch library(paradox) + library(miesmuschel) # @mlr3mbo_config library(R6) library(checkmate) - library(mlr3mbo) # @so_config - reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) + reticulate::use_condaenv("/home/lschnei8/.conda/envs/env", required = TRUE) library(reticulate) library(yahpogym) + source("AcqFunctionLogEI.R") source("LearnerRegrRangerCustom.R") source("OptimizerChain.R") @@ -119,6 +111,11 @@ evaluate = function(xdt, instance) { learner$param_set$values$se.method = "jack" learner$param_set$values$splitrule = "variance" learner$param_set$values$num.trees = 1000L + } else if (xdt$rf_type == "extratrees") { + learner$param_set$values$se.method = "jack" + learner$param_set$values$splitrule = "extratrees" + learner$param_set$values$num.random.splits = 1L + learner$param_set$values$num.trees = 1000L } else if (xdt$rf_type == "smaclike_boot") { learner$param_set$values$se.method = "simple" learner$param_set$values$splitrule = "extratrees" @@ -137,14 +134,6 @@ evaluate = function(xdt, instance) { learner$param_set$values$sample.fraction = 1 learner$param_set$values$min.node.size = 1 learner$param_set$values$mtry.ratio = 1 - } else if (xdt$rf_type == "smaclike_variance_boot") { - learner$param_set$values$se.method = "simple" - learner$param_set$values$splitrule = "variance" - learner$param_set$values$num.trees = 10L - learner$param_set$values$replace = TRUE - learner$param_set$values$sample.fraction = 1 - learner$param_set$values$min.node.size = 1 - learner$param_set$values$mtry.ratio = 1 } surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor", multiplier = 3) %>>% learner)) @@ -152,24 +141,28 @@ evaluate = function(xdt, instance) { acq_optimizer = if (xdt$acqopt == "RS_1000") { AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 1000L)) } else if (xdt$acqopt == "RS") { - AcqOptimizer$new(opt("random_search", batch_size = 10000L), terminator = trm("evals", n_evals = 20000L)) + AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 20000L)) } else if (xdt$acqopt == "FS") { n_repeats = 2L maxit = 9L batch_size = ceiling((20000L / n_repeats) / (1 + maxit)) # 1000L AcqOptimizer$new(opt("focus_search", n_points = batch_size, maxit = maxit), terminator = trm("evals", n_evals = 20000L)) } else if (xdt$acqopt == "LS") { - optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 10000L)), terminators = list(trm("evals", n_evals = 10010L), trm("evals", n_evals = 10000L))) - acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20020L)) + optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10000L), trm("evals", n_evals = 10000L))) + acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20000L)) acq_optimizer$param_set$values$warmstart = TRUE - acq_optimizer$param_set$values$warmstart_size = 10L + acq_optimizer$param_set$values$warmstart_size = "all" acq_optimizer } acq_function = if (xdt$acqf == "EI") { - AcqFunctionEI$new() + if (isTRUE(xdt$acqf_ei_log)) { + AcqFunctionLogEI$new() + } else { + AcqFunctionEI$new() + } } else if (xdt$acqf == "CB") { - AcqFunctionCB$new(lambda = xdt$lambda) + AcqFunctionCB$new(lambda = as.numeric(xdt$lambda)) } else if (xdt$acqf == "PI") { AcqFunctionPI$new() } else if (xdt$acqf == "Mean") { @@ -187,6 +180,7 @@ evaluate = function(xdt, instance) { instance_ = instance$instance target_ = paste0("mean_", instance$target) + # assumes maximization k = min(mies_average[scenario == scenario_ & instance == instance_ & get(target_) >= best]$iter) # minimum k so that best_mies[k] >= best_mbo[final] k = k / instance$budget # sample efficiency compared to mies @@ -198,6 +192,7 @@ objective = ObjectiveRFunDt$new( xdt[, id := seq_len(.N)] # FIXME: walltime can be set adaptively based on xdt # FIXME: we could continuously model the walltime with a surrogate and set this for each xs in xdt + # FIXME: tryCatch needed? res = future_mapply(FUN = evaluate, transpose_list(xdt), transpose_list(instances), SIMPLIFY = FALSE, future.seed = TRUE) res = rbindlist(res) stopifnot(nrow(res) == nrow(xdt) * nrow(instances)) @@ -219,10 +214,9 @@ cd_instance = OptimInstanceSingleCrit$new( optimizer = OptimizerCoordinateDescent$new() optimizer$param_set$values$max_gen = 1L -init = data.table(loop_function = "ego", init = "random", init_size_fraction = "0.25", random_interleave = FALSE, random_interleave_iter = NA_character_, rf_type = "standard", acqf = "EI", lambda = NA_integer_, acqopt = "RS") +init = data.table(loop_function = "ego", init = "random", init_size_fraction = "0.25", random_interleave = FALSE, random_interleave_iter = NA_character_, rf_type = "standard", acqf = "EI", acqf_ei_log = NA, lambda = NA_character_, acqopt = "RS") set.seed(2906) plan("batchtools_slurm", resources = list(walltime = 3600L * 12L, ncpus = 1L, memory = 4000L), template = "slurm_wyoming.tmpl") cd_instance$eval_batch(init) optimizer$optimize(cd_instance) - diff --git a/attic/so_config/run_yahpo.R b/attic/so_config/run_yahpo.R index 3392b282..8fd58530 100644 --- a/attic/so_config/run_yahpo.R +++ b/attic/so_config/run_yahpo.R @@ -41,7 +41,7 @@ mlr3mbo_wrapper = function(job, data, instance, ...) { optim_instance = make_optim_instance(instance) - xdt = data.table(loop_function = "ego_log", init = "sobol", init_size_fraction = "0.25", random_interleave = TRUE, random_interleave_iter = "5", rf_type = "smaclike_boot", acqf = "LogEI", lambda = NA_real_, acqopt = "LS") + xdt = data.table(loop_function = "ego_log", init = "sobol", init_size_fraction = "0.25", random_interleave = TRUE, random_interleave_iter = "5", rf_type = "smaclike_boot", acqf = "EI", acqf_ei_log = TRUE, lambda = NA_character_, acqopt = "LS") d = optim_instance$search_space$length init_design_size = ceiling(as.numeric(xdt$init_size_fraction) * optim_instance$terminator$param_set$values$n_evals) @@ -65,6 +65,11 @@ mlr3mbo_wrapper = function(job, data, instance, ...) { learner$param_set$values$se.method = "jack" learner$param_set$values$splitrule = "variance" learner$param_set$values$num.trees = 1000L + } else if (xdt$rf_type == "extratrees") { + learner$param_set$values$se.method = "jack" + learner$param_set$values$splitrule = "extratrees" + learner$param_set$values$num.random.splits = 1L + learner$param_set$values$num.trees = 1000L } else if (xdt$rf_type == "smaclike_boot") { learner$param_set$values$se.method = "simple" learner$param_set$values$splitrule = "extratrees" @@ -83,14 +88,6 @@ mlr3mbo_wrapper = function(job, data, instance, ...) { learner$param_set$values$sample.fraction = 1 learner$param_set$values$min.node.size = 1 learner$param_set$values$mtry.ratio = 1 - } else if (xdt$rf_type == "smaclike_variance_boot") { - learner$param_set$values$se.method = "simple" - learner$param_set$values$splitrule = "variance" - learner$param_set$values$num.trees = 10L - learner$param_set$values$replace = TRUE - learner$param_set$values$sample.fraction = 1 - learner$param_set$values$min.node.size = 1 - learner$param_set$values$mtry.ratio = 1 } surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor", multiplier = 3) %>>% learner)) @@ -113,11 +110,13 @@ mlr3mbo_wrapper = function(job, data, instance, ...) { } acq_function = if (xdt$acqf == "EI") { - AcqFunctionEI$new() - } else if (xdt$acqf == "LogEI") { - AcqFunctionLogEI$new() + if (isTRUE(xdt$acqf_ei_log)) { + AcqFunctionLogEI$new() + } else { + AcqFunctionEI$new() + } } else if (xdt$acqf == "CB") { - AcqFunctionCB$new(lambda = xdt$lambda) + AcqFunctionCB$new(lambda = as.numeric(xdt$lambda)) } else if (xdt$acqf == "PI") { AcqFunctionPI$new() } else if (xdt$acqf == "Mean") { From 748ec169e747191b3788013fc55c90a6da9dcdad Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Wed, 18 Jan 2023 12:28:59 +0100 Subject: [PATCH 038/126] .. --- attic/so_config/OptimizerCoordinateDescent.R | 7 +++++- attic/so_config/analyze_yahpo.R | 1 - attic/so_config/coordinate_descent.R | 24 +++++++++++--------- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/attic/so_config/OptimizerCoordinateDescent.R b/attic/so_config/OptimizerCoordinateDescent.R index dfb677e0..67e50408 100644 --- a/attic/so_config/OptimizerCoordinateDescent.R +++ b/attic/so_config/OptimizerCoordinateDescent.R @@ -30,7 +30,12 @@ OptimizerCoordinateDescent = R6Class("OptimizerCoordinateDescent", inherit = bbo inst$eval_batch(xdt) } incumbent = inst$archive$best()[, inst$archive$cols_x, with = FALSE] - gen = 0L + # check if .gen is already present, if yes continue from there + gen = if (inst$archive$n_evals > 0L & ".gen" %in% colnames(inst$archive$data)) { + max(inst$archive$data[[".gen"]], na.rm = TRUE) + } else { + 0L + } repeat { gen = gen + 1L for (param_id in shuffle(inst$search_space$ids())) { diff --git a/attic/so_config/analyze_yahpo.R b/attic/so_config/analyze_yahpo.R index 60e80a8b..235f8a25 100644 --- a/attic/so_config/analyze_yahpo.R +++ b/attic/so_config/analyze_yahpo.R @@ -4,7 +4,6 @@ library(pammtools) library(mlr3misc) dat = rbind(readRDS("results_yahpo.rds"), readRDS("results_yahpo_mlr3mbo.rds")) -dat = dat[method %in% c("random", "smac4hpo", "mlr3mbo")] dat[, cumbudget := cumsum(budget), by = .(method, scenario, instance, repl)] dat[, cumbudget_scaled := cumbudget / max(cumbudget), by = .(method, scenario, instance, repl)] dat[, normalized_regret := (target - min(target)) / (max(target) - min(target)), by = .(scenario, instance)] diff --git a/attic/so_config/coordinate_descent.R b/attic/so_config/coordinate_descent.R index 3ea77809..6e5533f5 100644 --- a/attic/so_config/coordinate_descent.R +++ b/attic/so_config/coordinate_descent.R @@ -33,8 +33,8 @@ search_space = ps( rf_type = p_fct(c("standard", "extratrees", "smaclike_boot", "smaclike_no_boot"), default = "standard"), acqf = p_fct(c("EI", "CB", "PI", "Mean"), default = "EI"), acqf_ei_log = p_lgl(depends = loop_function == "ego_log" && acqf == "EI", default = FALSE), - lambda = p_fct(c("1", "3", "5"), depends = acqf == "CB", default = "1"), - acqopt = p_fct(c("RS_1000", "RS", "FS", "LS"), default = "RS") + lambda = p_fct(c("1", "3", "10"), depends = acqf == "CB", default = "1"), + acqopt = p_fct(c("RS_1000", "RS", "FS", "LS"), default = "RS_1000") ) instances = setup = data.table(scenario = rep(c("lcbench", paste0("rbv2_", c("aknn", "glmnet", "ranger", "rpart", "super", "svm", "xgboost"))), each = 4L), @@ -49,7 +49,7 @@ instances = setup = data.table(scenario = rep(c("lcbench", paste0("rbv2_", c("ak target = rep(c("val_accuracy", "acc"), c(4L, 28L)), budget = rep(c(126L, 118L, 90L, 134L, 110L, 267L, 118L, 170L), each = 4L)) -mies_average = readRDS("results_yahpo_mies_average.rds") +mies_average = readRDS("/gscratch/lschnei8/results_yahpo_mies_average.rds") evaluate = function(xdt, instance) { id = xdt$id @@ -190,14 +190,16 @@ evaluate = function(xdt, instance) { objective = ObjectiveRFunDt$new( fun = function(xdt) { xdt[, id := seq_len(.N)] - # FIXME: walltime can be set adaptively based on xdt - # FIXME: we could continuously model the walltime with a surrogate and set this for each xs in xdt - # FIXME: tryCatch needed? - res = future_mapply(FUN = evaluate, transpose_list(xdt), transpose_list(instances), SIMPLIFY = FALSE, future.seed = TRUE) + xdt_tmp = do.call("rbind", replicate(nrow(instances), xdt, simplify = FALSE)) + setorderv(xdt_tmp, col = "id") + instances_tmp = do.call("rbind", replicate(nrow(xdt), instances, simplify = FALSE)) + # FIXME: shuffle instances_tmp or maybe make sure that expensive are sheduled first but be careful that xdt matches + res = future_mapply(FUN = evaluate, transpose_list(xdt_tmp), transpose_list(instances_tmp), SIMPLIFY = FALSE, future.seed = TRUE) res = rbindlist(res) - stopifnot(nrow(res) == nrow(xdt) * nrow(instances)) - agg = res[, .(mean_k = exp(mean(log(k))), raw_k = list(k), n_na = sum(is.na(k))), by = .(id)] - setorderv(agg, cols = "id") + setorderv(res, col = "instance") + setorderv(res, col = "id") + agg = res[, .(mean_k = exp(mean(log(k))), raw_k = list(k), n_na = sum(is.na(k)), n = .N), by = .(id)] + agg[n < nrow(instances), mean_k := 0] agg }, domain = search_space, @@ -214,7 +216,7 @@ cd_instance = OptimInstanceSingleCrit$new( optimizer = OptimizerCoordinateDescent$new() optimizer$param_set$values$max_gen = 1L -init = data.table(loop_function = "ego", init = "random", init_size_fraction = "0.25", random_interleave = FALSE, random_interleave_iter = NA_character_, rf_type = "standard", acqf = "EI", acqf_ei_log = NA, lambda = NA_character_, acqopt = "RS") +init = data.table(loop_function = "ego", init = "random", init_size_fraction = "0.25", random_interleave = FALSE, random_interleave_iter = NA_character_, rf_type = "standard", acqf = "EI", acqf_ei_log = NA, lambda = NA_character_, acqopt = "RS_1000") set.seed(2906) plan("batchtools_slurm", resources = list(walltime = 3600L * 12L, ncpus = 1L, memory = 4000L), template = "slurm_wyoming.tmpl") cd_instance$eval_batch(init) From e9242afcef1a6d3135f238da1f813153cbd74c01 Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Thu, 19 Jan 2023 14:12:53 +0100 Subject: [PATCH 039/126] .. --- .gitignore | 1 + attic/so_config/coordinate_descent.R | 13 +++-- attic/so_config/run_yahpo_mies.R | 77 +++++++++++++++++++++++++--- 3 files changed, 81 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 2d7b002a..a2e314d5 100644 --- a/.gitignore +++ b/.gitignore @@ -108,6 +108,7 @@ Meta attic/so_config/fanova.py attic/so_config/Pipfile attic/so_config/*.rds +attic/so_config/*.png attic/so_config_old /doc/ /Meta/ diff --git a/attic/so_config/coordinate_descent.R b/attic/so_config/coordinate_descent.R index 6e5533f5..ea086379 100644 --- a/attic/so_config/coordinate_descent.R +++ b/attic/so_config/coordinate_descent.R @@ -50,6 +50,7 @@ instances = setup = data.table(scenario = rep(c("lcbench", paste0("rbv2_", c("ak budget = rep(c(126L, 118L, 90L, 134L, 110L, 267L, 118L, 170L), each = 4L)) mies_average = readRDS("/gscratch/lschnei8/results_yahpo_mies_average.rds") +mies_extrapolation readRDS("/gscratch/lschnei8/mies_extrapolation.rds") evaluate = function(xdt, instance) { id = xdt$id @@ -181,10 +182,16 @@ evaluate = function(xdt, instance) { target_ = paste0("mean_", instance$target) # assumes maximization - k = min(mies_average[scenario == scenario_ & instance == instance_ & get(target_) >= best]$iter) # minimum k so that best_mies[k] >= best_mbo[final] + if (best > max(mies_average[scenario == scenario_ & instance == instance_][["mean_best"]])) { + extrapolate = TRUE + k = mies_extrapolation[scenario == scenario_ & instance == instance_][["model"]][[1L]](best) + } else { + extrapolate = FALSE + k = min(mies_average[scenario == scenario_ & instance == instance_ & mean_best >= best]$iter) # minimum k so that mean_best_mies[k] >= best_mbo[final] + } k = k / instance$budget # sample efficiency compared to mies - data.table(k = k, id = id, instance = paste0(instance$scenario, "_", instance$instance)) + data.table(k = k, extrapolate = extrapolate, id = id, instance = paste0(instance$scenario, "_", instance$instance)) } objective = ObjectiveRFunDt$new( @@ -198,7 +205,7 @@ objective = ObjectiveRFunDt$new( res = rbindlist(res) setorderv(res, col = "instance") setorderv(res, col = "id") - agg = res[, .(mean_k = exp(mean(log(k))), raw_k = list(k), n_na = sum(is.na(k)), n = .N), by = .(id)] + agg = res[, .(mean_k = exp(mean(log(k))), raw_k = list(k), n_na = sum(is.na(k)), raw_extrapolate = list(extrapolate) n = .N), by = .(id)] agg[n < nrow(instances), mean_k := 0] agg }, diff --git a/attic/so_config/run_yahpo_mies.R b/attic/so_config/run_yahpo_mies.R index 39039832..fe2d7475 100644 --- a/attic/so_config/run_yahpo_mies.R +++ b/attic/so_config/run_yahpo_mies.R @@ -8,7 +8,7 @@ library(miesmuschel) # @mlr3mbo_config library(R6) library(checkmate) -reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) +reticulate::use_condaenv("/home/lschnei8/.conda/envs/env", required = TRUE) library(reticulate) yahpo_gym = import("yahpo_gym") @@ -23,7 +23,7 @@ saveRegistry(reg) mies_wrapper = function(job, data, instance, ...) { # mies is our baseline with 1000 x more budget - reticulate::use_virtualenv("/home/lschnei8/yahpo_gym/experiments/mf_env/", required = TRUE) + reticulate::use_condaenv("/home/lschnei8/.conda/envs/env", required = TRUE) library(yahpogym) logger = lgr::get_logger("bbotk") logger$set_threshold("warn") @@ -117,6 +117,8 @@ results = reduceResultsList(done, function(x, job) { pars = job$pars target_var = pars$prob.pars$target tmp = x[, eval(target_var), with = FALSE] + colnames(tmp) = "target" + tmp[, best := cummax(target)] tmp[, method := pars$algo.pars$algorithm] tmp[, scenario := pars$prob.pars$scenario] tmp[, instance := pars$prob.pars$instance] @@ -125,10 +127,71 @@ results = reduceResultsList(done, function(x, job) { tmp }) results = rbindlist(results, fill = TRUE) -saveRDS(results, "results_yahpo_mies.rds") +saveRDS(results, "/gscratch/lschnei8/results_yahpo_mies.rds") + +mean_results = results[, .(mean_best = mean(best), se_best = sd(best) / sqrt(.N)), by = .(scenario, instance, iter)] +saveRDS(mean_results, "/gscratch/lschnei8/results_yahpo_mies_average.rds") + +library(ggplot2) +library(gridExtra) + +lm_data = copy(mean_results) +lm_data[, problem := paste0(scenario, "_", instance)] +models = map_dtr(unique(lm_data$problem), function(problem_) { + tmp = lm_data[problem == problem_] + values = tail(unique(tmp$mean_best), 2L) + dat = map_dtr(values, function(value) { + tmp[mean_best == value][.N, ] + }) + model = lm(mean_best ~ iter, data = dat) + coefs = coef(model) + if (coefs[2L] < .Machine$double.eps) { + stop("Almost constant linear model") + } + max_mean_best = max(tmp$mean_best) + max_iter = max(tmp$iter) + estimate_iter = function(mean_best, correct = TRUE) { + stopifnot(mean_best >= max_mean_best) + iter = as.integer(ceiling((mean_best - coefs[1L]) / coefs[2L])) + if (correct) { + if (mean_best > max_mean_best && iter <= max_iter) { + iter = max_iter + 1L + } + } + iter + } + env = new.env() + environment(estimate_iter) = env + assign("max_mean_best", value = max_mean_best, envir = env) + assign("max_iter", value = max_iter, envir = env) + if (estimate_iter(1.00001 * max(tmp$mean_best), correct = FALSE) < max(tmp$iter)) { + # marginal improvements should require more iter than max iter + stop("Model does not interpolate latest iter well.") + } + tmp_p = data.table(iter = ((NROW(tmp) - 1L):ceiling(1.5 * NROW(tmp)))) + p = predict(model, tmp_p) + tmp_p[, mean_best := p] + tmp_p[, method := "interpolation"] + tmp_plot = copy(tmp)[, c("iter", "mean_best")] + tmp_plot[, method := "real"] + tmp_plot = rbind(tmp_plot, tmp_p) + g = ggplot(aes(x = iter, y = mean_best, colour = method), data = tmp_plot[ceiling(0.9 * NROW(tmp)):.N , ]) + + scale_y_log10() + + geom_step(direction = "vh") + + geom_hline(yintercept = max(tmp$mean_best), linetype = 2L) + + labs(title = problem_) + + theme_minimal() + + theme(legend.position = "bottom") + info = strsplit(problem_, "_")[[1L]] + if (length(info) == 3L) { + info = c(paste0(info[1L], "_", info[2L]), info[3L]) # rbv2_ + } + list(model = list(estimate_iter), plot = list(g), scenario = info[1L], instance = info[2L]) +}) + +g = do.call("grid.arrange", c(models$plot, ncol = 8L)) + +ggsave("mies_extrapolation.png", plot = g, width = 32, height = 12) -mean_results_lcbench = results[grepl("lcbench", x = scenario), .(mean_val_accuracy = mean(val_accuracy), se_val_accuracy = sd(val_accuracy) / sqrt(.N)), by = .(scenario, instance, iter)] -mean_results_rbv2 = results[grepl("rbv2", x = scenario), .(mean_acc = mean(acc), se_acc = sd(acc) / sqrt(.N)), by = .(scenario, instance, iter)] -mean_results = rbind(mean_results_lcbench, mean_results_rbv2, fill = TRUE) -saveRDS(mean_results, "results_yahpo_mies_average.rds") +saveRDS(models[, - "plot"], "mies_extrapolation.rds") From ad412aa541fa27f787d12de72a2a4b6f5ebb3b40 Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Fri, 20 Jan 2023 15:56:20 +0100 Subject: [PATCH 040/126] some changes to char to fct handling, ... --- DESCRIPTION | 2 +- R/SurrogateLearner.R | 3 +- R/SurrogateLearnerCollection.R | 13 +-- R/helper.R | 18 ---- R/mbo_defaults.R | 16 +++- attic/so_config/OptimizerCoordinateDescent.R | 6 +- attic/so_config/coordinate_descent.R | 99 ++++++++++++++------ attic/so_config/run_yahpo.R | 5 +- man/default_surrogate.Rd | 1 + man/mlr_acqfunctions_aei.Rd | 1 + man/mlr_acqfunctions_ttei.Rd | 1 + man/mlr_loop_functions_ego_log.Rd | 2 +- tests/testthat/test_mbo_defaults.R | 8 +- 13 files changed, 107 insertions(+), 68 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 6be083d7..c2201290 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -70,7 +70,7 @@ Config/testthat/edition: 3 Config/testthat/parallel: false NeedsCompilation: yes Roxygen: list(markdown = TRUE, r6 = TRUE) -RoxygenNote: 7.2.2 +RoxygenNote: 7.2.3 Collate: 'mlr_acqfunctions.R' 'AcqFunction.R' diff --git a/R/SurrogateLearner.R b/R/SurrogateLearner.R index db2f80ec..5b4fd56e 100644 --- a/R/SurrogateLearner.R +++ b/R/SurrogateLearner.R @@ -110,7 +110,6 @@ SurrogateLearner = R6Class("SurrogateLearner", predict = function(xdt) { assert_xdt(xdt) xdt = fix_xdt_missing(xdt, x_cols = self$x_cols, archive = self$archive) - xdt = char_to_fct(xdt) pred = self$model$predict_newdata(newdata = xdt) if (self$model$predict_type == "se") { @@ -197,7 +196,7 @@ SurrogateLearner = R6Class("SurrogateLearner", # Also calculates the insample performance based on the `perf_measure` hyperparameter if `assert_insample_perf = TRUE`. .update = function() { xydt = self$archive$data[, c(self$x_cols, self$y_cols), with = FALSE] - task = TaskRegr$new(id = "surrogate_task", backend = char_to_fct(xydt), target = self$y_cols) + task = TaskRegr$new(id = "surrogate_task", backend = xydt, target = self$y_cols) assert_learnable(task, learner = self$model) self$model$train(task) diff --git a/R/SurrogateLearnerCollection.R b/R/SurrogateLearnerCollection.R index 14f5ec39..747478e7 100644 --- a/R/SurrogateLearnerCollection.R +++ b/R/SurrogateLearnerCollection.R @@ -128,7 +128,6 @@ SurrogateLearnerCollection = R6Class("SurrogateLearnerCollection", predict = function(xdt) { assert_xdt(xdt) xdt = fix_xdt_missing(xdt, x_cols = self$x_cols, archive = self$archive) - xdt = char_to_fct(xdt) preds = lapply(self$model, function(model) { pred = model$predict_newdata(newdata = xdt) @@ -224,17 +223,11 @@ SurrogateLearnerCollection = R6Class("SurrogateLearnerCollection", # Train model with new data. # Also calculates the insample performance based on the `perf_measures` hyperparameter if `assert_insample_perf = TRUE`. .update = function() { - xydt = char_to_fct(self$archive$data[, c(self$x_cols, self$y_cols), with = FALSE]) - backend = as_data_backend(xydt) # we do this here to save time in the lapply below + xydt = self$archive$data[, c(self$x_cols, self$y_cols), with = FALSE] features = setdiff(names(xydt), self$y_cols) - tasks = lapply(self$y_cols, function(y_col) { - # If this turns out to be a bottleneck, we can also operate on a single task here. - task = TaskRegr$new( - id = paste0("surrogate_task_", y_col), - backend = backend, - target = y_col) - task$col_roles$feature = features + # if this turns out to be a bottleneck, we can also operate on a single task here + task = TaskRegr$new(id = paste0("surrogate_task_", y_col), backend = xydt[, c(features, y_col), with = FALSE], target = y_col) task }) pmap(list(model = self$model, task = tasks), .f = function(model, task) { diff --git a/R/helper.R b/R/helper.R index cf484735..3035a6d8 100644 --- a/R/helper.R +++ b/R/helper.R @@ -23,24 +23,6 @@ archive_x = function(archive) { archive$data[, archive$cols_x, with = FALSE] } -# convert character params to factors -char_to_fct = function(xydt) { - chr_cols = names(xydt)[map_chr(xydt, class) == "character"] - if (length(chr_cols)) { - xydt[, (chr_cols) := map(.SD, as.factor), .SDcols = chr_cols] - } - xydt -} - -# convert factor params to character -fct_to_char = function(xydt) { - fct_cols = names(xydt)[map_chr(xydt, class) %in% c("factor", "ordered")] - if (length(fct_cols)) { - xydt[, (fct_cols) := map(.SD, as.character), .SDcols = fct_cols] - } - xydt -} - # during surrogate prediction it may have happened that whole columns where dropped (e.g., during focussearch if the search space was shrunk) fix_xdt_missing = function(xdt, x_cols, archive) { missing = x_cols[x_cols %nin% colnames(xdt)] diff --git a/R/mbo_defaults.R b/R/mbo_defaults.R index e9a71019..f5d84eae 100644 --- a/R/mbo_defaults.R +++ b/R/mbo_defaults.R @@ -75,6 +75,7 @@ default_loopfun = function(instance) { #' We simply handle those with an imputation method, added to the ranger random forest, more #' concretely we use \code{po("imputesample")} (for logicals) and \code{po("imputeoor")} (for anything else) from #' package \CRANpkg{mlr3pipelines}. +#' Characters are always encoded as factors via \code{po("colapply")}. #' Out of range imputation makes sense for tree-based methods and is usually hard to beat, see Ding et al. (2010). #' In the case of dependencies, the following learner is used as a fallback: #' \code{lrn("regr.featureless")}. @@ -136,9 +137,20 @@ default_surrogate = function(instance, learner = NULL, n_learner = NULL) { if (has_deps) { require_namespaces("mlr3pipelines") - learner = mlr3pipelines::GraphLearner$new(mlr3pipelines::"%>>%"(mlr3pipelines::"%>>%"(mlr3pipelines::po("imputesample", affect_columns = mlr3pipelines::selector_type("logical")), mlr3pipelines::po("imputeoor", multiplier = 2)), learner)) + learner = mlr3pipelines::GraphLearner$new( + mlr3pipelines::"%>>%"( + mlr3pipelines::"%>>%"( + mlr3pipelines::po("imputesample", affect_columns = mlr3pipelines::selector_type("logical")), + mlr3pipelines::"%>>%"( + mlr3pipelines::po("imputeoor", multiplier = 3, affect_columns = mlr3pipelines::selector_type(c("integer", "numeric", "character", "factor", "ordered"))), + mlr3pipelines::po("colapply", applicator = as.factor, affect_columns = mlr3pipelines::selector_type("character")) + ) + ), + learner + ) + ) learner$encapsulate[c("train", "predict")] = "evaluate" - learner$fallback = lrn("regr.featureless") + learner$fallback = LearnerRegrFeatureless$new() } } diff --git a/attic/so_config/OptimizerCoordinateDescent.R b/attic/so_config/OptimizerCoordinateDescent.R index 67e50408..8f4db58a 100644 --- a/attic/so_config/OptimizerCoordinateDescent.R +++ b/attic/so_config/OptimizerCoordinateDescent.R @@ -46,7 +46,11 @@ OptimizerCoordinateDescent = R6Class("OptimizerCoordinateDescent", inherit = bbo deps = inst$search_space$deps[on == param_id, ] for (i in seq_len(nrow(deps))) { to_replace = which(map_lgl(xdt[[param_id]], function(x) deps$cond[[i]]$test(x))) - set(xdt, i = to_replace, j = deps$id[[i]], value = sample_random(inst$search_space$params[[deps$id[[i]]]], n = length(to_replace))) + if (test_r6(inst$search_space$params[[deps$id[[i]]]]$default, classes = "NoDefault")) { + set(xdt, i = to_replace, j = deps$id[[i]], value = sample_random(inst$search_space$params[[deps$id[[i]]]], n = length(to_replace))) # random + } else { + set(xdt, i = to_replace, j = deps$id[[i]], value = inst$search_space$params[[deps$id[[i]]]]$default) # default + } } } xdt = Design$new(inst$search_space, data = xdt, remove_dupl = TRUE)$data # fixes potentially broken dependencies diff --git a/attic/so_config/coordinate_descent.R b/attic/so_config/coordinate_descent.R index ea086379..3069ae41 100644 --- a/attic/so_config/coordinate_descent.R +++ b/attic/so_config/coordinate_descent.R @@ -12,7 +12,8 @@ library(miesmuschel) # @mlr3mbo_config library(R6) library(checkmate) -reticulate::use_condaenv("/home/lschnei8/.conda/envs/env", required = TRUE) +#reticulate::use_condaenv("/home/lschnei8/.conda/envs/env", required = TRUE) +reticulate::use_virtualenv("/home/lps/.local/share/virtualenvs/yahpo_gym-4ygV7ggv/", required = TRUE) library(reticulate) library(yahpogym) library("future") @@ -29,7 +30,7 @@ search_space = ps( init = p_fct(c("random", "lhs", "sobol"), default = "random"), init_size_fraction = p_fct(c("0.0625", "0.125", "0.25"), default = "0.25"), random_interleave = p_lgl(default = FALSE), - random_interleave_iter = p_fct(c("2", "5", "10"), default = "10", depends = random_interleave == TRUE), + random_interleave_iter = p_fct(c("2", "5", "10"), depends = random_interleave == TRUE, default = "10"), rf_type = p_fct(c("standard", "extratrees", "smaclike_boot", "smaclike_no_boot"), default = "standard"), acqf = p_fct(c("EI", "CB", "PI", "Mean"), default = "EI"), acqf_ei_log = p_lgl(depends = loop_function == "ego_log" && acqf == "EI", default = FALSE), @@ -48,13 +49,33 @@ instances = setup = data.table(scenario = rep(c("lcbench", paste0("rbv2_", c("ak "40984", "40979", "40966", "28"), target = rep(c("val_accuracy", "acc"), c(4L, 28L)), budget = rep(c(126L, 118L, 90L, 134L, 110L, 267L, 118L, 170L), each = 4L)) +instances[, problem := paste0(scenario, "_", instance)] +setorderv(instances, col = "budget", order = -1L) -mies_average = readRDS("/gscratch/lschnei8/results_yahpo_mies_average.rds") -mies_extrapolation readRDS("/gscratch/lschnei8/mies_extrapolation.rds") +#mies_average = readRDS("/gscratch/lschnei8/results_yahpo_mies_average.rds") +mies_average = readRDS("results_yahpo_mies_average.rds") +#mies_extrapolation = readRDS("/gscratch/lschnei8/mies_extrapolation.rds") +mies_extrapolation = readRDS("mies_extrapolation.rds") + +get_k = function(best, scenario_, instance_, budget_) { + # assumes maximization + if (best > max(mies_average[scenario == scenario_ & instance == instance_][["mean_best"]])) { + extrapolate = TRUE + k = mies_extrapolation[scenario == scenario_ & instance == instance_][["model"]][[1L]](best) + } else { + extrapolate = FALSE + k = min(mies_average[scenario == scenario_ & instance == instance_ & mean_best >= best]$iter) # min k so that mean_best_mies[k] >= best_mbo[final] + } + k = k / budget_ # sample efficiency compared to mies + attr(k, "extrapolate") = extrapolate + k +} evaluate = function(xdt, instance) { id = xdt$id + repl = xdt$repl xdt$id = NULL + xdt$repl = NULL library(data.table) library(mlr3) @@ -67,7 +88,8 @@ evaluate = function(xdt, instance) { library(miesmuschel) # @mlr3mbo_config library(R6) library(checkmate) - reticulate::use_condaenv("/home/lschnei8/.conda/envs/env", required = TRUE) + #reticulate::use_condaenv("/home/lschnei8/.conda/envs/env", required = TRUE) + reticulate::use_virtualenv("/home/lps/.local/share/virtualenvs/yahpo_gym-4ygV7ggv/", required = TRUE) library(reticulate) library(yahpogym) @@ -137,7 +159,11 @@ evaluate = function(xdt, instance) { learner$param_set$values$mtry.ratio = 1 } - surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor", multiplier = 3) %>>% learner)) + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% + po("imputeoor", multiplier = 3, affect_columns = selector_type(c("integer", "numeric", "character", "factor", "ordered"))) %>>% + po("colapply", applicator = as.factor, affect_columns = selector_type("character")) %>>% + learner)) + surrogate$param_set$values$catch_errors = FALSE acq_optimizer = if (xdt$acqopt == "RS_1000") { AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 1000L)) @@ -155,6 +181,7 @@ evaluate = function(xdt, instance) { acq_optimizer$param_set$values$warmstart_size = "all" acq_optimizer } + acq_optimizer$param_set$values$catch_errors = FALSE acq_function = if (xdt$acqf == "EI") { if (isTRUE(xdt$acqf_ei_log)) { @@ -177,37 +204,48 @@ evaluate = function(xdt, instance) { } best = optim_instance$archive$best()[[instance$target]] - scenario_ = instance$scenario - instance_ = instance$instance - target_ = paste0("mean_", instance$target) - # assumes maximization - if (best > max(mies_average[scenario == scenario_ & instance == instance_][["mean_best"]])) { - extrapolate = TRUE - k = mies_extrapolation[scenario == scenario_ & instance == instance_][["model"]][[1L]](best) - } else { - extrapolate = FALSE - k = min(mies_average[scenario == scenario_ & instance == instance_ & mean_best >= best]$iter) # minimum k so that mean_best_mies[k] >= best_mbo[final] - } - k = k / instance$budget # sample efficiency compared to mies - - data.table(k = k, extrapolate = extrapolate, id = id, instance = paste0(instance$scenario, "_", instance$instance)) + data.table(best = best, scenario = instance$scenario, instance = instance$instance, target = instance$target, id = id, repl = repl) } objective = ObjectiveRFunDt$new( fun = function(xdt) { + n_repl = 3L xdt[, id := seq_len(.N)] - xdt_tmp = do.call("rbind", replicate(nrow(instances), xdt, simplify = FALSE)) + xdt_tmp = map_dtr(seq_len(nrow(instances)), function(i) copy(xdt)) setorderv(xdt_tmp, col = "id") - instances_tmp = do.call("rbind", replicate(nrow(xdt), instances, simplify = FALSE)) - # FIXME: shuffle instances_tmp or maybe make sure that expensive are sheduled first but be careful that xdt matches - res = future_mapply(FUN = evaluate, transpose_list(xdt_tmp), transpose_list(instances_tmp), SIMPLIFY = FALSE, future.seed = TRUE) + xdt_tmp = map_dtr(seq_len(n_repl), function(i) { + tmp = copy(xdt_tmp) + tmp[, repl := i] + tmp + }) + instances_tmp = map_dtr(seq_len(nrow(xdt) * n_repl), function(i) copy(instances)) + #res = future_mapply(FUN = evaluate, transpose_list(xdt_tmp), transpose_list(instances_tmp), SIMPLIFY = FALSE, future.seed = TRUE) + res = mapply(FUN = evaluate, transpose_list(xdt_tmp), transpose_list(instances_tmp), SIMPLIFY = FALSE) res = rbindlist(res) setorderv(res, col = "instance") setorderv(res, col = "id") - agg = res[, .(mean_k = exp(mean(log(k))), raw_k = list(k), n_na = sum(is.na(k)), raw_extrapolate = list(extrapolate) n = .N), by = .(id)] - agg[n < nrow(instances), mean_k := 0] - agg + setorderv(res, col = "repl") + res[, problem := paste0(scenario, "_", instance)] + + # average best over replications and determine ks + agg = res[, .(mean_best = mean(best), raw_best = list(best), n_na = sum(is.na(best)), n = .N), by = .(id, problem, scenario, instance)] + ks = map_dbl(seq_len(nrow(agg)), function(i) { + if (agg[i, ][["n"]] < n_repl) { + 0 + } else { + get_k(agg[i, ][["mean_best"]], + scenario_ = agg[i, ][["scenario"]], + instance_ = agg[i, ][["instance"]], + budget_ = instances[problem == agg[i, ][["problem"]]][["budget"]]) + } + }) + agg[, k := ks] + + # average k over instances and determine mean_k + agg_k = agg[, .(mean_k = exp(mean(log(k))), raw_k = list(k), n_na = sum(is.na(k)), n = .N, raw_mean_best = list(mean_best)), by = .(id)] + agg_k[n < nrow(instances), mean_k := 0] + agg_k }, domain = search_space, codomain = ps(mean_k = p_dbl(tags = "maximize")), @@ -224,8 +262,9 @@ optimizer = OptimizerCoordinateDescent$new() optimizer$param_set$values$max_gen = 1L init = data.table(loop_function = "ego", init = "random", init_size_fraction = "0.25", random_interleave = FALSE, random_interleave_iter = NA_character_, rf_type = "standard", acqf = "EI", acqf_ei_log = NA, lambda = NA_character_, acqopt = "RS_1000") -set.seed(2906) -plan("batchtools_slurm", resources = list(walltime = 3600L * 12L, ncpus = 1L, memory = 4000L), template = "slurm_wyoming.tmpl") +set.seed(2906, kind = "L'Ecuyer-CMRG") +#plan("batchtools_slurm", resources = list(walltime = 3600L * 12L, ncpus = 1L, memory = 4000L), template = "slurm_wyoming.tmpl") +plan(sequential) cd_instance$eval_batch(init) -optimizer$optimize(cd_instance) +#optimizer$optimize(cd_instance) diff --git a/attic/so_config/run_yahpo.R b/attic/so_config/run_yahpo.R index 8fd58530..1a62ded3 100644 --- a/attic/so_config/run_yahpo.R +++ b/attic/so_config/run_yahpo.R @@ -90,7 +90,10 @@ mlr3mbo_wrapper = function(job, data, instance, ...) { learner$param_set$values$mtry.ratio = 1 } - surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% po("imputeoor", multiplier = 3) %>>% learner)) + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% + po("imputeoor", multiplier = 3, affect_columns = selector_type(c("integer", "numeric", "character", "factor", "ordered"))) %>>% + po("colapply", applicator = as.factor, affect_columns = selector_type("character")) %>>% + learner)) acq_optimizer = if (xdt$acqopt == "RS_1000") { AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 1000L)) diff --git a/man/default_surrogate.Rd b/man/default_surrogate.Rd index 4302133f..24f734b6 100644 --- a/man/default_surrogate.Rd +++ b/man/default_surrogate.Rd @@ -57,6 +57,7 @@ are represented by missing \code{NA} values in the training design data. We simply handle those with an imputation method, added to the ranger random forest, more concretely we use \code{po("imputesample")} (for logicals) and \code{po("imputeoor")} (for anything else) from package \CRANpkg{mlr3pipelines}. +Characters are always encoded as factors via \code{po("colapply")}. Out of range imputation makes sense for tree-based methods and is usually hard to beat, see Ding et al. (2010). In the case of dependencies, the following learner is used as a fallback: \code{lrn("regr.featureless")}. diff --git a/man/mlr_acqfunctions_aei.Rd b/man/mlr_acqfunctions_aei.Rd index 5c58ec9b..0d874ee0 100644 --- a/man/mlr_acqfunctions_aei.Rd +++ b/man/mlr_acqfunctions_aei.Rd @@ -85,6 +85,7 @@ Other Acquisition Function: \code{\link{mlr_acqfunctions_mean}}, \code{\link{mlr_acqfunctions_pi}}, \code{\link{mlr_acqfunctions_smsego}}, +\code{\link{mlr_acqfunctions_ttei}}, \code{\link{mlr_acqfunctions}} } \concept{Acquisition Function} diff --git a/man/mlr_acqfunctions_ttei.Rd b/man/mlr_acqfunctions_ttei.Rd index d61bc0cd..fa08eeb7 100644 --- a/man/mlr_acqfunctions_ttei.Rd +++ b/man/mlr_acqfunctions_ttei.Rd @@ -71,6 +71,7 @@ if (requireNamespace("mlr3learners") & \seealso{ Other Acquisition Function: \code{\link{AcqFunction}}, +\code{\link{mlr_acqfunctions_aei}}, \code{\link{mlr_acqfunctions_cb}}, \code{\link{mlr_acqfunctions_eips}}, \code{\link{mlr_acqfunctions_ei}}, diff --git a/man/mlr_loop_functions_ego_log.Rd b/man/mlr_loop_functions_ego_log.Rd index efe1eaaf..dcd56fcd 100644 --- a/man/mlr_loop_functions_ego_log.Rd +++ b/man/mlr_loop_functions_ego_log.Rd @@ -85,7 +85,7 @@ if (requireNamespace("mlr3learners") & list(y = xs$x ^ 2) } domain = ps(x = p_dbl(lower = -10, upper = 10)) - codomain = ps(y = p_dbl(tags = "minimize")) + codomain = ps(y = p_dbl(tags = "maximize")) objective = ObjectiveRFun$new(fun = fun, domain = domain, codomain = codomain) instance = OptimInstanceSingleCrit$new( diff --git a/tests/testthat/test_mbo_defaults.R b/tests/testthat/test_mbo_defaults.R index a9d9cd77..f677e33a 100644 --- a/tests/testthat/test_mbo_defaults.R +++ b/tests/testthat/test_mbo_defaults.R @@ -1,5 +1,6 @@ test_that("default_surrogate", { skip_if_not_installed("mlr3learners") + skip_if_not_installed("mlr3pipelines") skip_if_not_installed("DiceKriging") skip_if_not_installed("ranger") @@ -70,12 +71,15 @@ test_that("default_surrogate", { surrogate = default_surrogate(MAKE_INST(OBJ_1D_MIXED, search_space = PS_1D_MIXED_DEPS)) expect_r6(surrogate, "SurrogateLearner") expect_r6(surrogate$model, "GraphLearner") - expect_equal(surrogate$model$graph$ids(), c("imputesample", "imputeoor", "regr.ranger")) + expect_equal(surrogate$model$graph$ids(), c("imputesample", "imputeoor", "colapply", "regr.ranger")) expect_equal(surrogate$model$param_set$values, list(imputesample.affect_columns = mlr3pipelines::selector_type("logical"), imputeoor.min = TRUE, imputeoor.offset = 1, - imputeoor.multiplier = 2, + imputeoor.multiplier = 3, + imputeoor.affect_columns = mlr3pipelines::selector_type(c("integer", "numeric", "character", "factor", "ordered")), + colapply.applicator = as.factor, + colapply.affect_columns = mlr3pipelines::selector_type("character"), regr.ranger.num.threads = 1L, regr.ranger.num.trees = 500L, regr.ranger.keep.inbag = TRUE, From 42f318cccad96daed2a11aa8f2803018f022c950 Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Sat, 21 Jan 2023 14:21:43 +0100 Subject: [PATCH 041/126] .. --- attic/so_config/coordinate_descent.R | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/attic/so_config/coordinate_descent.R b/attic/so_config/coordinate_descent.R index 627f0923..b3dc947e 100644 --- a/attic/so_config/coordinate_descent.R +++ b/attic/so_config/coordinate_descent.R @@ -11,8 +11,7 @@ library(paradox) library(R6) library(checkmate) -#reticulate::use_condaenv("/home/lschnei8/.conda/envs/env", required = TRUE) -reticulate::use_virtualenv("/home/lps/.local/share/virtualenvs/yahpo_gym-4ygV7ggv/", required = TRUE) +reticulate::use_condaenv("/home/lschnei8/.conda/envs/env", required = TRUE) library(reticulate) library(yahpogym) library("future") @@ -51,21 +50,19 @@ instances = setup = data.table(scenario = rep(c("lcbench", paste0("rbv2_", c("ak instances[, problem := paste0(scenario, "_", instance)] setorderv(instances, col = "budget", order = -1L) -#mies_average = readRDS("/gscratch/lschnei8/results_yahpo_mies_average.rds") -mies_average = readRDS("results_yahpo_mies_average.rds") -#mies_extrapolation = readRDS("/gscratch/lschnei8/mies_extrapolation.rds") -mies_extrapolation = readRDS("mies_extrapolation.rds") +fs_average = readRDS("/gscratch/lschnei8/results_yahpo_fs_average.rds") +fs_extrapolation = readRDS("/gscratch/lschnei8/fs_extrapolation.rds") get_k = function(best, scenario_, instance_, budget_) { # assumes maximization - if (best > max(mies_average[scenario == scenario_ & instance == instance_][["mean_best"]])) { + if (best > max(fs_average[scenario == scenario_ & instance == instance_][["mean_best"]])) { extrapolate = TRUE - k = mies_extrapolation[scenario == scenario_ & instance == instance_][["model"]][[1L]](best) + k = fs_extrapolation[scenario == scenario_ & instance == instance_][["model"]][[1L]](best) } else { extrapolate = FALSE - k = min(mies_average[scenario == scenario_ & instance == instance_ & mean_best >= best]$iter) # min k so that mean_best_mies[k] >= best_mbo[final] + k = min(fs_average[scenario == scenario_ & instance == instance_ & mean_best >= best]$iter) # min k so that mean_best_fs[k] >= best_mbo[final] } - k = k / budget_ # sample efficiency compared to mies + k = k / budget_ # sample efficiency compared to fs attr(k, "extrapolate") = extrapolate k } @@ -86,8 +83,7 @@ evaluate = function(xdt, instance) { library(paradox) library(R6) library(checkmate) - #reticulate::use_condaenv("/home/lschnei8/.conda/envs/env", required = TRUE) - reticulate::use_virtualenv("/home/lps/.local/share/virtualenvs/yahpo_gym-4ygV7ggv/", required = TRUE) + reticulate::use_condaenv("/home/lschnei8/.conda/envs/env", required = TRUE) library(reticulate) library(yahpogym) @@ -218,8 +214,7 @@ objective = ObjectiveRFunDt$new( tmp }) instances_tmp = map_dtr(seq_len(nrow(xdt) * n_repl), function(i) copy(instances)) - #res = future_mapply(FUN = evaluate, transpose_list(xdt_tmp), transpose_list(instances_tmp), SIMPLIFY = FALSE, future.seed = TRUE) - res = mapply(FUN = evaluate, transpose_list(xdt_tmp), transpose_list(instances_tmp), SIMPLIFY = FALSE) + res = future_mapply(FUN = evaluate, transpose_list(xdt_tmp), transpose_list(instances_tmp), SIMPLIFY = FALSE, future.seed = TRUE) res = rbindlist(res) setorderv(res, col = "instance") setorderv(res, col = "id") @@ -261,8 +256,7 @@ optimizer$param_set$values$max_gen = 1L init = data.table(loop_function = "ego", init = "random", init_size_fraction = "0.25", random_interleave = FALSE, random_interleave_iter = NA_character_, rf_type = "standard", acqf = "EI", acqf_ei_log = NA, lambda = NA_character_, acqopt = "RS_1000") set.seed(2906, kind = "L'Ecuyer-CMRG") -#plan("batchtools_slurm", resources = list(walltime = 3600L * 12L, ncpus = 1L, memory = 4000L), template = "slurm_wyoming.tmpl") -plan(sequential) +plan("batchtools_slurm", resources = list(walltime = 3600L * 12L, ncpus = 1L, memory = 4000L), template = "slurm_wyoming.tmpl") cd_instance$eval_batch(init) -#optimizer$optimize(cd_instance) +optimizer$optimize(cd_instance) From a961d0c8fc777269d9ce649d5d8afc150a64a89a Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Mon, 23 Jan 2023 17:04:41 +0100 Subject: [PATCH 042/126] ... --- attic/so_config/coordinate_descent.R | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/attic/so_config/coordinate_descent.R b/attic/so_config/coordinate_descent.R index b3dc947e..932a03c0 100644 --- a/attic/so_config/coordinate_descent.R +++ b/attic/so_config/coordinate_descent.R @@ -204,6 +204,7 @@ evaluate = function(xdt, instance) { objective = ObjectiveRFunDt$new( fun = function(xdt) { + future_label = paste0("future_mapply_", if(is.null(xdt[1L, ][[".gen"]])) "" else xdt[1L, ][[".gen"]], "_", if(is.null(xdt[1L, ][[".param"]])) "" else xdt[1L, ][[".param"]], "-%d") n_repl = 3L xdt[, id := seq_len(.N)] xdt_tmp = map_dtr(seq_len(nrow(instances)), function(i) copy(xdt)) @@ -214,7 +215,14 @@ objective = ObjectiveRFunDt$new( tmp }) instances_tmp = map_dtr(seq_len(nrow(xdt) * n_repl), function(i) copy(instances)) - res = future_mapply(FUN = evaluate, transpose_list(xdt_tmp), transpose_list(instances_tmp), SIMPLIFY = FALSE, future.seed = TRUE) + current_seed = .Random.seed + res = tryCatch(future_mapply(FUN = evaluate, transpose_list(xdt_tmp), transpose_list(instances_tmp), SIMPLIFY = FALSE, future.seed = TRUE, future.scheduling = FALSE, future.label = future_label), error = function(ec) ec) + if (inherits(res, "error")) { + browser() # last manual fallback resort to get things working again + # cleanup future stuff + # reset the current seed and continue the eval + # .Random.seed = current_seed + } res = rbindlist(res) setorderv(res, col = "instance") setorderv(res, col = "id") @@ -252,11 +260,13 @@ cd_instance = OptimInstanceSingleCrit$new( ) optimizer = OptimizerCoordinateDescent$new() -optimizer$param_set$values$max_gen = 1L +optimizer$param_set$values$max_gen = 3L init = data.table(loop_function = "ego", init = "random", init_size_fraction = "0.25", random_interleave = FALSE, random_interleave_iter = NA_character_, rf_type = "standard", acqf = "EI", acqf_ei_log = NA, lambda = NA_character_, acqopt = "RS_1000") +options(future.cache.path = "/gscratch/lschnei8/coordinate_descent/future") set.seed(2906, kind = "L'Ecuyer-CMRG") -plan("batchtools_slurm", resources = list(walltime = 3600L * 12L, ncpus = 1L, memory = 4000L), template = "slurm_wyoming.tmpl") +# currently we evaluate at most 4 * 32 * 3 jobs in parallel so 400 workers is enough +plan("batchtools_slurm", template = "slurm_wyoming_cd.tmpl", resources = list(walltime = 3600L * 12L, ncpus = 1L, memory = 4000L), workers = 400L) cd_instance$eval_batch(init) optimizer$optimize(cd_instance) From 11860b9072280e0ba8d5cefaf80a1ff6f1512f8f Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Tue, 24 Jan 2023 22:30:24 +0100 Subject: [PATCH 043/126] .. --- attic/so_config/coordinate_descent.R | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/attic/so_config/coordinate_descent.R b/attic/so_config/coordinate_descent.R index 932a03c0..00568f62 100644 --- a/attic/so_config/coordinate_descent.R +++ b/attic/so_config/coordinate_descent.R @@ -11,6 +11,9 @@ library(paradox) library(R6) library(checkmate) +# FIXME: error handling +# how to continue after one job is finished (i.e. from an intermediate gen) save seed? + reticulate::use_condaenv("/home/lschnei8/.conda/envs/env", required = TRUE) library(reticulate) library(yahpogym) @@ -204,7 +207,6 @@ evaluate = function(xdt, instance) { objective = ObjectiveRFunDt$new( fun = function(xdt) { - future_label = paste0("future_mapply_", if(is.null(xdt[1L, ][[".gen"]])) "" else xdt[1L, ][[".gen"]], "_", if(is.null(xdt[1L, ][[".param"]])) "" else xdt[1L, ][[".param"]], "-%d") n_repl = 3L xdt[, id := seq_len(.N)] xdt_tmp = map_dtr(seq_len(nrow(instances)), function(i) copy(xdt)) @@ -216,12 +218,13 @@ objective = ObjectiveRFunDt$new( }) instances_tmp = map_dtr(seq_len(nrow(xdt) * n_repl), function(i) copy(instances)) current_seed = .Random.seed - res = tryCatch(future_mapply(FUN = evaluate, transpose_list(xdt_tmp), transpose_list(instances_tmp), SIMPLIFY = FALSE, future.seed = TRUE, future.scheduling = FALSE, future.label = future_label), error = function(ec) ec) + res = tryCatch(future_mapply(FUN = evaluate, transpose_list(xdt_tmp), transpose_list(instances_tmp), SIMPLIFY = FALSE, future.seed = TRUE, future.scheduling = FALSE), error = function(ec) ec) if (inherits(res, "error")) { browser() # last manual fallback resort to get things working again # cleanup future stuff # reset the current seed and continue the eval # .Random.seed = current_seed + # res = future_mapply(FUN = evaluate, transpose_list(xdt_tmp), transpose_list(instances_tmp), SIMPLIFY = FALSE, future.seed = TRUE, future.scheduling = FALSE) } res = rbindlist(res) setorderv(res, col = "instance") From c0d5c73262f71cfe926cbf5413f0e70bc5f4cb4f Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Thu, 26 Jan 2023 13:43:12 +0100 Subject: [PATCH 044/126] .. --- attic/so_config/OptimizerCoordinateDescent.R | 1 + attic/so_config/coordinate_descent.R | 22 ++++++++++++++------ attic/so_config/run_yahpo_fs.R | 16 ++++++-------- 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/attic/so_config/OptimizerCoordinateDescent.R b/attic/so_config/OptimizerCoordinateDescent.R index 8f4db58a..fd285726 100644 --- a/attic/so_config/OptimizerCoordinateDescent.R +++ b/attic/so_config/OptimizerCoordinateDescent.R @@ -36,6 +36,7 @@ OptimizerCoordinateDescent = R6Class("OptimizerCoordinateDescent", inherit = bbo } else { 0L } + set(inst$archive$data, j = ".gen", value = gen) # 0 for first batch repeat { gen = gen + 1L for (param_id in shuffle(inst$search_space$ids())) { diff --git a/attic/so_config/coordinate_descent.R b/attic/so_config/coordinate_descent.R index 00568f62..5b55fd83 100644 --- a/attic/so_config/coordinate_descent.R +++ b/attic/so_config/coordinate_descent.R @@ -238,10 +238,13 @@ objective = ObjectiveRFunDt$new( if (agg[i, ][["n"]] < n_repl) { 0 } else { - get_k(agg[i, ][["mean_best"]], - scenario_ = agg[i, ][["scenario"]], - instance_ = agg[i, ][["instance"]], - budget_ = instances[problem == agg[i, ][["problem"]]][["budget"]]) + tryCatch( + get_k(agg[i, ][["mean_best"]], + scenario_ = agg[i, ][["scenario"]], + instance_ = agg[i, ][["instance"]], + budget_ = instances[problem == agg[i, ][["problem"]]][["budget"]]), + error = function(ec) 0 + ) } }) agg[, k := ks] @@ -266,10 +269,17 @@ optimizer = OptimizerCoordinateDescent$new() optimizer$param_set$values$max_gen = 3L init = data.table(loop_function = "ego", init = "random", init_size_fraction = "0.25", random_interleave = FALSE, random_interleave_iter = NA_character_, rf_type = "standard", acqf = "EI", acqf_ei_log = NA, lambda = NA_character_, acqopt = "RS_1000") -options(future.cache.path = "/gscratch/lschnei8/coordinate_descent/future") +#options(future.cache.path = "/gscratch/lschnei8/coordinate_descent/future") +options(future.cache.path = "/home/lschnei8/mlr3mbo_config/future") set.seed(2906, kind = "L'Ecuyer-CMRG") # currently we evaluate at most 4 * 32 * 3 jobs in parallel so 400 workers is enough -plan("batchtools_slurm", template = "slurm_wyoming_cd.tmpl", resources = list(walltime = 3600L * 12L, ncpus = 1L, memory = 4000L), workers = 400L) +plan("batchtools_slurm", template = "slurm_wyoming_cd.tmpl", resources = list(walltime = 3600L * 9L, ncpus = 1L, memory = 4000L), workers = 400L) cd_instance$eval_batch(init) optimizer$optimize(cd_instance) +# NOTE: +# cd_instance_gen1.rds first gen through +# then load, subset data to gen1 +# set.seed(2409, kind = "L'Ecuyer-CMRG") +# and rerun + diff --git a/attic/so_config/run_yahpo_fs.R b/attic/so_config/run_yahpo_fs.R index 07540f6a..5612b1d1 100644 --- a/attic/so_config/run_yahpo_fs.R +++ b/attic/so_config/run_yahpo_fs.R @@ -134,23 +134,19 @@ models = map_dtr(unique(lm_data$problem), function(problem_) { if (coefs[2L] < .Machine$double.eps) { stop("Almost constant linear model") } - max_mean_best = max(tmp$mean_best) max_iter = max(tmp$iter) - estimate_iter = function(mean_best, correct = TRUE) { - stopifnot(mean_best >= max_mean_best) - iter = as.integer(ceiling((mean_best - coefs[1L]) / coefs[2L])) - if (correct) { - if (mean_best > max_mean_best && iter <= max_iter) { - iter = max_iter + 1L - } + estimate_iter = function(mean_best) { + iter = ceiling((mean_best - coefs[1L]) / coefs[2L]) + if (!isTRUE(iter > max_iter)) { + iter = max_iter + 1 } iter } env = new.env() environment(estimate_iter) = env - assign("max_mean_best", value = max_mean_best, envir = env) assign("max_iter", value = max_iter, envir = env) - if (estimate_iter(1.00001 * max(tmp$mean_best), correct = FALSE) < max(tmp$iter)) { + assign("coefs", value = coefs, envir = env) + if (estimate_iter(1.00001 * max(tmp$mean_best)) < max(tmp$iter)) { # marginal improvements should require more iter than max iter stop("Model does not interpolate latest iter well.") } From 4d3f68b765369b9d3b2cba7b6044924d7a341217 Mon Sep 17 00:00:00 2001 From: pfistfl Date: Wed, 1 Feb 2023 10:56:09 +0100 Subject: [PATCH 045/126] add learner --- R/LearnerR2C.R | 109 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 R/LearnerR2C.R diff --git a/R/LearnerR2C.R b/R/LearnerR2C.R new file mode 100644 index 00000000..cf299e76 --- /dev/null +++ b/R/LearnerR2C.R @@ -0,0 +1,109 @@ +LearnerR2C = R6Class("LearnerR2C", + inherit = mlr3::LearnerRegr, + public = list( + #' classifier + classifier = NULL, + #' wt + wt = NULL, + #' gamma + gamma = NULL, + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + initialize = function(learner, gamma = 0.9, wt = "ei") { + assert_class(learner, "LearnerClassif") + assert_true(learner$predict_type == "prob") + assert_true("weights" %in% learner$properties) + l = learner + self$classifier = learner + self$gamma = assert_number(gamma, lower = 0, upper = 1) + self$wt = assert_choice(wt, choices = c("ei", "pi")) + props = setdiff(l$properties, c("twoclass", "multiclass")) + super$initialize( + id = paste0(l$id, "2regr"), + param_set = l$param_set, feature_types = l$feature_types, + predict_types = c("response", "se"), + properties = props, data_formats = l$data_formats, packages = l$packages, + label = l$label, man = l$man) + }, + train = function(task, row_ids = NULL) { + # create df with new data. + tn = private$.to_weighted_task(task, row_ids) + self$classifier$train(tn, row_ids) + self$state$train_task = mlr3:::task_rm_backend(task$clone(deep = TRUE)) + }, + predict = function(task, row_ids = NULL) { + fts = task$data(rows = row_ids, cols = task$feature_names) + colnames(fts) = paste0("X.", colnames(fts)) + pd = self$classifier$predict_newdata(fts)$data + pdata = as_prediction_data( + list(response = pd$prob[, 1L]), + task = task, + row_ids = pd$row_ids + ) + as_prediction(pdata) + } + ), + private = list( + .to_weighted_task = function(task, row_ids) { + dt = do.call("cbind", private$.regr_to_weights( + task$data(rows = row_ids, cols = task$feature_names), + task$data(rows = row_ids, cols = task$target_names)[[1]] + )) + tn = TaskClassif$new("weighted_classif", dt, target = "z") + tn$set_col_roles("w", "weight") + }, + .regr_to_weights = function(X,y, gamma = .9, wt = "ei") { + tau = quantile(y, p=self$gamma) + z = y < tau + + if (self$wt == "ei") { + w1 = tau - y[z] + w1 = w1 / mean(w1) + Xn = rbind(X, X[z,]) + zn = c(z[z], rep(0, length(z))) + wn = c(w1 * sum(z), rep(1, length(z)) / length(z)) + wn = wn / mean(wn) + } else if (self$wt == "pi") { + Xn = X + wn = rep(1, nrow(X)) + zn = z + } + return(list(X=Xn,w=wn,z=as.factor(zn))) + } + ) +) + +library(mlr3) +library(data.table) +library(checkmate) +library(R6) +library(mlr3mbo) +library(bbotk) +library(paradox) +library(mlr3learners) +fun = function(xs) { + list(y = xs$x ^ 2) +} +domain = ps(x = p_dbl(lower = -10, upper = 10)) +codomain = ps(y = p_dbl(tags = "minimize")) +objective = ObjectiveRFun$new(fun = fun, domain = domain, codomain = codomain) +instance = OptimInstanceSingleCrit$new( + objective = objective, + terminator = trm("evals", n_evals = 20) +) +surrogate = SurrogateLearner$new(LearnerR2C$new(lrn("classif.rpart", predict_type = "prob"))) +acq_function = acqf("mean") +acq_optimizer = acqo( + optimizer = opt("random_search"), + terminator = trm("evals", n_evals = 100)) +optimizer = opt("mbo", + loop_function = bayesopt_ego, + surrogate = surrogate, + acq_function = acq_function, + acq_optimizer = acq_optimizer +) +optimizer$optimize(instance) + +dt = instance$archive +ggplot(dt$data) + geom_point(aes(x = x, y = acq_mean), size = 5) + theme_minimal() \ No newline at end of file From 02af5655a62a57a676e02d6843c75c68e683d40c Mon Sep 17 00:00:00 2001 From: pfistfl Date: Wed, 1 Feb 2023 14:46:20 +0100 Subject: [PATCH 046/126] fix impl errors, add docs --- R/LearnerLFBO.R | 99 ++++++++++++++++++++++++++ R/LearnerR2C.R | 109 ----------------------------- tests/testthat/test_learner_lfbo.R | 54 ++++++++++++++ 3 files changed, 153 insertions(+), 109 deletions(-) create mode 100644 R/LearnerLFBO.R delete mode 100644 R/LearnerR2C.R create mode 100644 tests/testthat/test_learner_lfbo.R diff --git a/R/LearnerLFBO.R b/R/LearnerLFBO.R new file mode 100644 index 00000000..93050329 --- /dev/null +++ b/R/LearnerLFBO.R @@ -0,0 +1,99 @@ +#' @title Likelihood Free Bayesian Optimization Learner +#' +#' @description +#' Wraps a classification learner to be used as a regression learner for +#' Likelihood-free Optimization. +#' +#' @details +#' The regression data is internally converted to a weighted classification task. +#' The new objective is a weighted version of the prediction problem, where the goal +#' is to predict whether an instance is smaller than the `gamma` quantile of the target +#' distribution. +#' +#' @export +LearnerLFBO = R6Class("LearnerLFBO", + inherit = mlr3::LearnerRegr, + public = list( + + #' @field clf ([mlr3::LearnerClassif])\cr + #' Classifier to be used for LFBO. + clf = NULL, + #' @field wt (character)\cr + #' Weighting scheme for LFBO. Currently allows "ei" and "pi". + wt = NULL, + #' @field gamma (numeric)\cr + #' Quantile of the target distribution. Defaults to `0.33`. + gamma = NULL, + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + #' + #' @param clf ([mlr3::LearnerClassif])\cr + #' Classifier to be used for LFBO. Needs to have predict_type "prob". + #' @param wt (character)\cr + #' Weighting scheme for LFBO. Currently allows "ei" (expected improvement) and "pi" (probability of improvement). + #' @param gamma (numeric)\cr + #' Quantile of the target distribution. Defaults to `0.33`. + initialize = function(clf, gamma = 0.33, wt = "ei") { + assert_class(clf, "LearnerClassif") + assert_true(clf$predict_type == "prob") + assert_true("weights" %in% clf$properties) + self$clf = clf + self$gamma = assert_number(gamma, lower = 0, upper = 1) + self$wt = assert_choice(wt, choices = c("ei", "pi")) + props = setdiff(clf$properties, c("twoclass", "multiclass")) + super$initialize( + id = paste0(clf$id, "2regr"), + param_set = clf$param_set, feature_types = clf$feature_types, + predict_types = c("response"), + properties = props, data_formats = clf$data_formats, packages = clf$packages, + label = clf$label, man = clf$man + ) + }, + train = function(task, row_ids = NULL) { + # create df with new data. + tn = private$.to_weighted_task(task, row_ids) + self$clf$train(tn, row_ids) + self$state$train_task = mlr3:::task_rm_backend(task$clone(deep = TRUE)) + invisible(NULL) + }, + predict = function(task, row_ids = NULL) { + fts = task$data(rows = row_ids, cols = task$feature_names) + colnames(fts) = paste0("X.", colnames(fts)) + pd = self$clf$predict_newdata(fts)$data + pdata = as_prediction_data( + list(response = - pd$prob[, 2L]), + task = task, + row_ids = pd$row_ids + ) + as_prediction(pdata) + } + ), + private = list( + .to_weighted_task = function(task, row_ids) { + dt = do.call("cbind", private$.regr_to_weights( + task$data(rows = row_ids, cols = task$feature_names), + task$data(rows = row_ids, cols = task$target_names)[[1]] + )) + tn = TaskClassif$new("weighted_classif", dt, target = "z") + tn$set_col_roles("w", "weight") + }, + .regr_to_weights = function(X,y, gamma = .8, wt = "ei") { + tau = quantile(y, probs = self$gamma) + z = y < tau + if (self$wt == "ei") { + w1 = tau - y[z] + w1 = w1 / mean(w1) + Xn = rbind(X, X[z,]) + zn = c(rep(0, length(z)), z[z]) + wn = c(rep(1, length(z)) / length(z), w1 * sum(z)) + wn = wn / mean(wn) + } else if (self$wt == "pi") { + Xn = X + wn = rep(1, nrow(X)) + zn = z + } + return(list(X=Xn,w=wn,z=as.factor(zn))) + } + ) +) diff --git a/R/LearnerR2C.R b/R/LearnerR2C.R deleted file mode 100644 index cf299e76..00000000 --- a/R/LearnerR2C.R +++ /dev/null @@ -1,109 +0,0 @@ -LearnerR2C = R6Class("LearnerR2C", - inherit = mlr3::LearnerRegr, - public = list( - #' classifier - classifier = NULL, - #' wt - wt = NULL, - #' gamma - gamma = NULL, - - #' @description - #' Creates a new instance of this [R6][R6::R6Class] class. - initialize = function(learner, gamma = 0.9, wt = "ei") { - assert_class(learner, "LearnerClassif") - assert_true(learner$predict_type == "prob") - assert_true("weights" %in% learner$properties) - l = learner - self$classifier = learner - self$gamma = assert_number(gamma, lower = 0, upper = 1) - self$wt = assert_choice(wt, choices = c("ei", "pi")) - props = setdiff(l$properties, c("twoclass", "multiclass")) - super$initialize( - id = paste0(l$id, "2regr"), - param_set = l$param_set, feature_types = l$feature_types, - predict_types = c("response", "se"), - properties = props, data_formats = l$data_formats, packages = l$packages, - label = l$label, man = l$man) - }, - train = function(task, row_ids = NULL) { - # create df with new data. - tn = private$.to_weighted_task(task, row_ids) - self$classifier$train(tn, row_ids) - self$state$train_task = mlr3:::task_rm_backend(task$clone(deep = TRUE)) - }, - predict = function(task, row_ids = NULL) { - fts = task$data(rows = row_ids, cols = task$feature_names) - colnames(fts) = paste0("X.", colnames(fts)) - pd = self$classifier$predict_newdata(fts)$data - pdata = as_prediction_data( - list(response = pd$prob[, 1L]), - task = task, - row_ids = pd$row_ids - ) - as_prediction(pdata) - } - ), - private = list( - .to_weighted_task = function(task, row_ids) { - dt = do.call("cbind", private$.regr_to_weights( - task$data(rows = row_ids, cols = task$feature_names), - task$data(rows = row_ids, cols = task$target_names)[[1]] - )) - tn = TaskClassif$new("weighted_classif", dt, target = "z") - tn$set_col_roles("w", "weight") - }, - .regr_to_weights = function(X,y, gamma = .9, wt = "ei") { - tau = quantile(y, p=self$gamma) - z = y < tau - - if (self$wt == "ei") { - w1 = tau - y[z] - w1 = w1 / mean(w1) - Xn = rbind(X, X[z,]) - zn = c(z[z], rep(0, length(z))) - wn = c(w1 * sum(z), rep(1, length(z)) / length(z)) - wn = wn / mean(wn) - } else if (self$wt == "pi") { - Xn = X - wn = rep(1, nrow(X)) - zn = z - } - return(list(X=Xn,w=wn,z=as.factor(zn))) - } - ) -) - -library(mlr3) -library(data.table) -library(checkmate) -library(R6) -library(mlr3mbo) -library(bbotk) -library(paradox) -library(mlr3learners) -fun = function(xs) { - list(y = xs$x ^ 2) -} -domain = ps(x = p_dbl(lower = -10, upper = 10)) -codomain = ps(y = p_dbl(tags = "minimize")) -objective = ObjectiveRFun$new(fun = fun, domain = domain, codomain = codomain) -instance = OptimInstanceSingleCrit$new( - objective = objective, - terminator = trm("evals", n_evals = 20) -) -surrogate = SurrogateLearner$new(LearnerR2C$new(lrn("classif.rpart", predict_type = "prob"))) -acq_function = acqf("mean") -acq_optimizer = acqo( - optimizer = opt("random_search"), - terminator = trm("evals", n_evals = 100)) -optimizer = opt("mbo", - loop_function = bayesopt_ego, - surrogate = surrogate, - acq_function = acq_function, - acq_optimizer = acq_optimizer -) -optimizer$optimize(instance) - -dt = instance$archive -ggplot(dt$data) + geom_point(aes(x = x, y = acq_mean), size = 5) + theme_minimal() \ No newline at end of file diff --git a/tests/testthat/test_learner_lfbo.R b/tests/testthat/test_learner_lfbo.R new file mode 100644 index 00000000..294fa004 --- /dev/null +++ b/tests/testthat/test_learner_lfbo.R @@ -0,0 +1,54 @@ +test_that("LFBO Learner can be constructed and trained", { + l1 = lrn("classif.rpart", predict_type = "prob") + expect_class(l1, "LearnerClassif") + l2 = LearnerLFBO$new(l1) + expect_class(l2, "LearnerRegr") + s1 = SurrogateLearner$new(l2) + expect_class(s1, "SurrogateLearner") + + t = tsk("boston_housing") + l2$train(t) + expect_true(!is.null(l2$state)) + prd = l2$predict(t) + # Note: Truth value does not make sense! + expect_class(prd, "PredictionRegr") +}) + +test_that("LFBO Learner works for optim", { + set.seed(4321L) + funs = list( + fun = function(xs) { + list(y = xs$x ^ 2) + }, + fun = function(xs) { + list(y = - xs$x ^ 2) + } + ) + codomains = list( + ps(y = p_dbl(tags = "minimize")), + ps(y = p_dbl(tags = "maximize")) + ) + for (i in seq_len(2)) { + domain = ps(x = p_dbl(lower = -10, upper = 10)) + fun = funs[[i]] + codomain = codomains[[i]] + objective = ObjectiveRFun$new(fun = fun, domain = domain, codomain = codomain) + instance = OptimInstanceSingleCrit$new( + objective = objective, + terminator = trm("evals", n_evals = 20) + ) + surrogate = SurrogateLearner$new(LearnerLFBO$new(lrn("classif.ranger", predict_type = "prob", num.trees=2L))) + acq_function = acqf("mean") + acq_optimizer = acqo( + optimizer = opt("random_search", batch_size = 100), + terminator = trm("evals", n_evals = 100)) + optimizer = opt("mbo", + loop_function = bayesopt_ego, + surrogate = surrogate, + acq_function = acq_function, + acq_optimizer = acq_optimizer + ) + optimizer$optimize(instance) + expect_true(abs(instance$result$y) < .1) + } +}) From 774cd43dd2dd2a45f1b3cad69ede4c38757093a2 Mon Sep 17 00:00:00 2001 From: pfistfl Date: Fri, 3 Feb 2023 11:57:51 +0100 Subject: [PATCH 047/126] update --- R/{LearnerLFBO.R => LearnerRegrLFBO.R} | 73 +++++++++++--------- R/bibentries.R | 10 ++- tests/testthat/test_learner_lfbo.R | 93 +++++++++++++------------- 3 files changed, 96 insertions(+), 80 deletions(-) rename R/{LearnerLFBO.R => LearnerRegrLFBO.R} (66%) diff --git a/R/LearnerLFBO.R b/R/LearnerRegrLFBO.R similarity index 66% rename from R/LearnerLFBO.R rename to R/LearnerRegrLFBO.R index 93050329..17f02d30 100644 --- a/R/LearnerLFBO.R +++ b/R/LearnerRegrLFBO.R @@ -1,17 +1,20 @@ #' @title Likelihood Free Bayesian Optimization Learner -#' -#' @description +#' +#' @description #' Wraps a classification learner to be used as a regression learner for -#' Likelihood-free Optimization. -#' -#' @details +#' Likelihood-free Optimization. +#' +#' @details #' The regression data is internally converted to a weighted classification task. -#' The new objective is a weighted version of the prediction problem, where the goal -#' is to predict whether an instance is smaller than the `gamma` quantile of the target +#' The new objective is a weighted version of a prediction problem, where the goal +#' to predict whether an instance is smaller than the `gamma` quantile of the target #' distribution. #' +#' @references +#' * `r format_bib("song_2022")` +#' #' @export -LearnerLFBO = R6Class("LearnerLFBO", +LearnerRegrLFBO = R6Class("LearnerRegrLFBO", inherit = mlr3::LearnerRegr, public = list( @@ -22,7 +25,7 @@ LearnerLFBO = R6Class("LearnerLFBO", #' Weighting scheme for LFBO. Currently allows "ei" and "pi". wt = NULL, #' @field gamma (numeric)\cr - #' Quantile of the target distribution. Defaults to `0.33`. + #' Quantile of the target distribution. Defaults to `0.33`. gamma = NULL, #' @description @@ -33,11 +36,12 @@ LearnerLFBO = R6Class("LearnerLFBO", #' @param wt (character)\cr #' Weighting scheme for LFBO. Currently allows "ei" (expected improvement) and "pi" (probability of improvement). #' @param gamma (numeric)\cr - #' Quantile of the target distribution. Defaults to `0.33`. + #' Quantile of the target distribution. Defaults to `0.33`. initialize = function(clf, gamma = 0.33, wt = "ei") { assert_class(clf, "LearnerClassif") assert_true(clf$predict_type == "prob") assert_true("weights" %in% clf$properties) + # FIXME: Should we expose gamma, wt as params, and if: how without making things ugly. self$clf = clf self$gamma = assert_number(gamma, lower = 0, upper = 1) self$wt = assert_choice(wt, choices = c("ei", "pi")) @@ -63,7 +67,7 @@ LearnerLFBO = R6Class("LearnerLFBO", pd = self$clf$predict_newdata(fts)$data pdata = as_prediction_data( list(response = - pd$prob[, 2L]), - task = task, + task = task, row_ids = pd$row_ids ) as_prediction(pdata) @@ -71,29 +75,32 @@ LearnerLFBO = R6Class("LearnerLFBO", ), private = list( .to_weighted_task = function(task, row_ids) { - dt = do.call("cbind", private$.regr_to_weights( - task$data(rows = row_ids, cols = task$feature_names), - task$data(rows = row_ids, cols = task$target_names)[[1]] - )) - tn = TaskClassif$new("weighted_classif", dt, target = "z") - tn$set_col_roles("w", "weight") + dt = do.call("cbind", private$.regr_to_weights( + task$data(rows = row_ids, cols = task$feature_names), + task$data(rows = row_ids, cols = task$target_names)[[1]] + )) + tn = TaskClassif$new("weighted_classif", dt, target = "z") + tn$set_col_roles("w", "weight") }, .regr_to_weights = function(X,y, gamma = .8, wt = "ei") { - tau = quantile(y, probs = self$gamma) - z = y < tau - if (self$wt == "ei") { - w1 = tau - y[z] - w1 = w1 / mean(w1) - Xn = rbind(X, X[z,]) - zn = c(rep(0, length(z)), z[z]) - wn = c(rep(1, length(z)) / length(z), w1 * sum(z)) - wn = wn / mean(wn) - } else if (self$wt == "pi") { - Xn = X - wn = rep(1, nrow(X)) - zn = z - } - return(list(X=Xn,w=wn,z=as.factor(zn))) - } + # Adapted from https://github.com/lfbo-ml/lfbo + tau = quantile(y, probs = self$gamma) + z = y < tau + if (self$wt == "ei") { + w1 = tau - y[z] + w1 = w1 / mean(w1) + Xn = rbind(X, X[z,]) + zn = c(rep(0, length(z)), z[z]) + wn = c(rep(1, length(z)) / length(z), w1 * sum(z)) + wn = wn / mean(wn) + } else if (self$wt == "pi") { + Xn = X + wn = rep(1, nrow(X)) + zn = z + } + return(list(X=Xn,w=wn,z=as.factor(zn))) + } ) ) + +mlr_learners$add("regr.lfbo", function() LearnerRegrLFBO$new()) diff --git a/R/bibentries.R b/R/bibentries.R index b7418ec4..c3df374b 100644 --- a/R/bibentries.R +++ b/R/bibentries.R @@ -105,6 +105,14 @@ bibentries = c( title = "A Multi-Points Criterion for Deterministic Parallel Global Optimization Based on Gaussian Processes", author = "Ginsbourger, David and Le Riche, Rodolphe and Carraro, Laurent", year = "2008" + ), + + song_2022 = bibentry("article", + title = "A General Recipe for Likelihood-free {B}ayesian Optimization", + author = "Song, Jiaming and Yu, Lantao and Neiswanger, Willie and Ermon, Stefano", + year = "2022", + volume = "162", + pages = "20384--20404", + journal = "Proceedings of Machine Learning Research" ) ) - diff --git a/tests/testthat/test_learner_lfbo.R b/tests/testthat/test_learner_lfbo.R index 294fa004..d6afa15b 100644 --- a/tests/testthat/test_learner_lfbo.R +++ b/tests/testthat/test_learner_lfbo.R @@ -1,54 +1,55 @@ test_that("LFBO Learner can be constructed and trained", { - l1 = lrn("classif.rpart", predict_type = "prob") - expect_class(l1, "LearnerClassif") - l2 = LearnerLFBO$new(l1) - expect_class(l2, "LearnerRegr") - s1 = SurrogateLearner$new(l2) - expect_class(s1, "SurrogateLearner") + l1 = lrn("classif.rpart", predict_type = "prob") + expect_class(l1, "LearnerClassif") + l2 = LearnerLFBO$new(l1) + expect_class(l2, "LearnerRegr") + expect_true(l2$id == "classif.rpart2regr") + s1 = SurrogateLearner$new(l2) + expect_class(s1, "SurrogateLearner") - t = tsk("boston_housing") - l2$train(t) - expect_true(!is.null(l2$state)) - prd = l2$predict(t) - # Note: Truth value does not make sense! - expect_class(prd, "PredictionRegr") + t = tsk("boston_housing") + l2$train(t) + expect_true(!is.null(l2$state)) + prd = l2$predict(t) + # Note: Truth value does not make sense! + expect_class(prd, "PredictionRegr") }) test_that("LFBO Learner works for optim", { - set.seed(4321L) - funs = list( - fun = function(xs) { - list(y = xs$x ^ 2) - }, - fun = function(xs) { - list(y = - xs$x ^ 2) - } + set.seed(4321L) + funs = list( + fun = function(xs) { + list(y = xs$x ^ 2) + }, + fun = function(xs) { + list(y = - xs$x ^ 2) + } + ) + codomains = list( + ps(y = p_dbl(tags = "minimize")), + ps(y = p_dbl(tags = "maximize")) + ) + for (i in seq_len(2)) { + domain = ps(x = p_dbl(lower = -10, upper = 10)) + fun = funs[[i]] + codomain = codomains[[i]] + objective = ObjectiveRFun$new(fun = fun, domain = domain, codomain = codomain) + instance = OptimInstanceSingleCrit$new( + objective = objective, + terminator = trm("evals", n_evals = 20) ) - codomains = list( - ps(y = p_dbl(tags = "minimize")), - ps(y = p_dbl(tags = "maximize")) + surrogate = SurrogateLearner$new(LearnerLFBO$new(lrn("classif.ranger", predict_type = "prob", num.trees=2L))) + acq_function = acqf("mean") + acq_optimizer = acqo( + optimizer = opt("random_search", batch_size = 100), + terminator = trm("evals", n_evals = 100)) + optimizer = opt("mbo", + loop_function = bayesopt_ego, + surrogate = surrogate, + acq_function = acq_function, + acq_optimizer = acq_optimizer ) - for (i in seq_len(2)) { - domain = ps(x = p_dbl(lower = -10, upper = 10)) - fun = funs[[i]] - codomain = codomains[[i]] - objective = ObjectiveRFun$new(fun = fun, domain = domain, codomain = codomain) - instance = OptimInstanceSingleCrit$new( - objective = objective, - terminator = trm("evals", n_evals = 20) - ) - surrogate = SurrogateLearner$new(LearnerLFBO$new(lrn("classif.ranger", predict_type = "prob", num.trees=2L))) - acq_function = acqf("mean") - acq_optimizer = acqo( - optimizer = opt("random_search", batch_size = 100), - terminator = trm("evals", n_evals = 100)) - optimizer = opt("mbo", - loop_function = bayesopt_ego, - surrogate = surrogate, - acq_function = acq_function, - acq_optimizer = acq_optimizer - ) - optimizer$optimize(instance) - expect_true(abs(instance$result$y) < .1) - } + optimizer$optimize(instance) + expect_true(abs(instance$result$y) < .1) + } }) From 235785056295a5a69ec2db79872b3545e635f9fe Mon Sep 17 00:00:00 2001 From: pfistfl Date: Fri, 3 Feb 2023 11:58:14 +0100 Subject: [PATCH 048/126] add to dict? --- R/LearnerRegrLFBO.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/LearnerRegrLFBO.R b/R/LearnerRegrLFBO.R index 17f02d30..079e0065 100644 --- a/R/LearnerRegrLFBO.R +++ b/R/LearnerRegrLFBO.R @@ -103,4 +103,4 @@ LearnerRegrLFBO = R6Class("LearnerRegrLFBO", ) ) -mlr_learners$add("regr.lfbo", function() LearnerRegrLFBO$new()) +# mlr_learners$add("regr.lfbo", function() LearnerRegrLFBO$new()) From 910707078253c797ac0ca46e771fbe18d978df8a Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Mon, 13 Feb 2023 15:14:42 +0100 Subject: [PATCH 049/126] .. --- attic/so_config/analyze_cd.R | 37 +++++++++++++++++++++++++- attic/so_config/noisyness_of_k.R | 45 ++++++++++++++++++++++++++++++++ attic/so_config/run_yahpo.R | 8 +++--- 3 files changed, 84 insertions(+), 6 deletions(-) create mode 100644 attic/so_config/noisyness_of_k.R diff --git a/attic/so_config/analyze_cd.R b/attic/so_config/analyze_cd.R index 37dd2307..8f4ac2d9 100644 --- a/attic/so_config/analyze_cd.R +++ b/attic/so_config/analyze_cd.R @@ -3,5 +3,40 @@ library(ggplot2) library(pammtools) library(mlr3misc) -dat = readRDS("cd_instance.rds") +instance = readRDS("cd_instance.rds") +archive = instance$archive +data = instance$archive$data +cols = c(archive$cols_x, archive$cols_y, ".param") + +for (i in seq_along(idx)) { + print(data[idx[[i]], cols, with = FALSE]) + cat("\n") + cat("\n") +} + +png("hist1_k.png") +hist(data[1L, ]$raw_k[[1L]], main = "k, geom mean = 4.265765", xlab = "") +dev.off() + +tmp = data[1L, ]$raw_mean_best[[1L]] +tmp[tmp > 1] = tmp[tmp > 1] / 100 +png("hist1_target.png") +hist(tmp, main = "Mean target, mean = 0.9433329", xlab = "") +dev.off() + +tmp = data[4L, ]$raw_mean_best[[1L]] +tmp[tmp > 1] = tmp[tmp > 1] / 100 + +png("hist2_k.png") +hist(data[5L, ]$raw_k[[1L]], main = "k, geom mean = 4.894954", xlab = "") +dev.off() + +tmp = data[5L, ]$raw_mean_best[[1L]] +tmp[tmp > 1] = tmp[tmp > 1] / 100 +png("hist2_target.png") +hist(tmp, main = "Mean target, mean = 0.9452976", xlab = "") +dev.off() + + + diff --git a/attic/so_config/noisyness_of_k.R b/attic/so_config/noisyness_of_k.R new file mode 100644 index 00000000..5c2373d9 --- /dev/null +++ b/attic/so_config/noisyness_of_k.R @@ -0,0 +1,45 @@ +library(data.table) +library(mlr3misc) + +fs_average = readRDS("results_yahpo_fs_average.rds") +fs = readRDS("results_yahpo_fs.rds")[scenario == "lcbench" & instance == 167152] + + +get_k = function(best, scenario_, instance_, budget_) { + # assumes maximization + if (best > max(fs_average[scenario == scenario_ & instance == instance_][["mean_best"]])) { + extrapolate = TRUE + k = fs_extrapolation[scenario == scenario_ & instance == instance_][["model"]][[1L]](best) + } else { + extrapolate = FALSE + k = min(fs_average[scenario == scenario_ & instance == instance_ & mean_best >= best]$iter) # min k so that mean_best_fs[k] >= best_mbo[final] + } + k = k / budget_ # sample efficiency compared to fs + attr(k, "extrapolate") = extrapolate + k +} + +k = get_k(97.543, scenario_ = "lcbench", instance_ = 167152, budget_ = 126) + +perfs = 97.543 + rnorm(100L, mean = 0, sd = 0.01) +ks = map_dbl(perfs, function(perf) get_k(perf, scenario_ = "lcbench", instance_ = 167152, budget_ = 126)) + +png("k_noise.png") +plot(perfs, ks, xlab = "Validation Accuracy", ylab = "k") +dev.off() + +ks_alt = map_dtr(perfs, function(perf) { + tmp = map_dbl(1:30, function(i) { + min(fs[scenario == "lcbench" & instance == 167152 & repl == i & best >= perf]$iter) / 126 + }) + data.table(mean = mean(tmp), geom = exp(mean(log(tmp)))) +}) + +png("k_noise_mean.png") +plot(perfs, ks_alt$mean, xlab = "Validation Accuracy", ylab = "k") +dev.off() + +png("k_noise_geom.png") +plot(perfs, ks_alt$geom, xlab = "Validation Accuracy", ylab = "k") +dev.off() + diff --git a/attic/so_config/run_yahpo.R b/attic/so_config/run_yahpo.R index 1a62ded3..3d1f5f6c 100644 --- a/attic/so_config/run_yahpo.R +++ b/attic/so_config/run_yahpo.R @@ -7,7 +7,6 @@ library(mlr3misc) library(mlr3mbo) # @so_config library(bbotk) # @localsearch library(paradox) -library(miesmuschel) # @mlr3mbo_config library(R6) library(checkmate) @@ -15,7 +14,7 @@ reticulate::use_condaenv("/home/lschnei8/.conda/envs/env", required = TRUE) library(reticulate) yahpo_gym = import("yahpo_gym") -packages = c("data.table", "mlr3", "mlr3learners", "mlr3pipelines", "mlr3misc", "mlr3mbo", "bbotk", "paradox", "miesmuschel", "R6", "checkmate") +packages = c("data.table", "mlr3", "mlr3learners", "mlr3pipelines", "mlr3misc", "mlr3mbo", "bbotk", "paradox", "R6", "checkmate") #RhpcBLASctl::blas_set_num_threads(1L) #RhpcBLASctl::omp_set_num_threads(1L) @@ -41,9 +40,8 @@ mlr3mbo_wrapper = function(job, data, instance, ...) { optim_instance = make_optim_instance(instance) - xdt = data.table(loop_function = "ego_log", init = "sobol", init_size_fraction = "0.25", random_interleave = TRUE, random_interleave_iter = "5", rf_type = "smaclike_boot", acqf = "EI", acqf_ei_log = TRUE, lambda = NA_character_, acqopt = "LS") + xdt = data.table(loop_function = "ego_log", init = "random", init_size_fraction = "0.25", random_interleave = TRUE, random_interleave_iter = "10", rf_type = "smaclike_no_boot", acqf = "CB", acqf_ei_log = NA, lambda = "1", acqopt = "LS") - d = optim_instance$search_space$length init_design_size = ceiling(as.numeric(xdt$init_size_fraction) * optim_instance$terminator$param_set$values$n_evals) init_design = if (xdt$init == "random") { generate_design_random(optim_instance$search_space, n = init_design_size)$data @@ -222,7 +220,7 @@ for (i in seq_len(nrow(optimizers))) { } jobs = getJobTable() -resources.default = list(walltime = 3600 * 12L, memory = 2048L, ntasks = 1L, ncpus = 1L, nodes = 1L, clusters = "beartooth", max.concurrent.jobs = 9999L) +resources.default = list(walltime = 3600 * 8L, memory = 2048L, ntasks = 1L, ncpus = 1L, nodes = 1L, clusters = "beartooth", max.concurrent.jobs = 9999L) submitJobs(jobs, resources = resources.default) done = findDone() From f218465c51ebe706d736f71f052bf82e144d5cd3 Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Wed, 15 Feb 2023 18:29:45 +0100 Subject: [PATCH 050/126] reiterate lfbo --- DESCRIPTION | 2 + NAMESPACE | 2 + R/AcqFunctionLFBO.R | 117 +++++++++++++++ R/LearnerRegrLFBO.R | 208 ++++++++++++++++++-------- R/helper.R | 11 ++ R/zzz.R | 4 + man/AcqFunction.Rd | 1 + man/mlr_acqfunctions.Rd | 1 + man/mlr_acqfunctions_aei.Rd | 1 + man/mlr_acqfunctions_cb.Rd | 1 + man/mlr_acqfunctions_ei.Rd | 1 + man/mlr_acqfunctions_eips.Rd | 1 + man/mlr_acqfunctions_lfbo.Rd | 142 ++++++++++++++++++ man/mlr_acqfunctions_mean.Rd | 1 + man/mlr_acqfunctions_pi.Rd | 1 + man/mlr_acqfunctions_smsego.Rd | 1 + man/mlr_learners_regr_lfbo.Rd | 194 ++++++++++++++++++++++++ tests/testthat/test_AcqFunctionLFBO.R | 31 ++++ tests/testthat/test_learner_lfbo.R | 55 ------- tests/testthat/test_regr_lfbo.R | 71 +++++++++ 20 files changed, 732 insertions(+), 114 deletions(-) create mode 100644 R/AcqFunctionLFBO.R create mode 100644 man/mlr_acqfunctions_lfbo.Rd create mode 100644 man/mlr_learners_regr_lfbo.Rd create mode 100644 tests/testthat/test_AcqFunctionLFBO.R delete mode 100644 tests/testthat/test_learner_lfbo.R create mode 100644 tests/testthat/test_regr_lfbo.R diff --git a/DESCRIPTION b/DESCRIPTION index 15e11c92..94ccfec3 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -78,10 +78,12 @@ Collate: 'AcqFunctionCB.R' 'AcqFunctionEI.R' 'AcqFunctionEIPS.R' + 'AcqFunctionLFBO.R' 'AcqFunctionMean.R' 'AcqFunctionPI.R' 'AcqFunctionSmsEgo.R' 'AcqOptimizer.R' + 'LearnerRegrLFBO.R' 'OptimizerMbo.R' 'Surrogate.R' 'SurrogateLearner.R' diff --git a/NAMESPACE b/NAMESPACE index 0f2b9246..0a22377b 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -8,10 +8,12 @@ export(AcqFunctionAEI) export(AcqFunctionCB) export(AcqFunctionEI) export(AcqFunctionEIPS) +export(AcqFunctionLFBO) export(AcqFunctionMean) export(AcqFunctionPI) export(AcqFunctionSmsEgo) export(AcqOptimizer) +export(LearnerRegrLFBO) export(OptimizerMbo) export(Surrogate) export(SurrogateLearner) diff --git a/R/AcqFunctionLFBO.R b/R/AcqFunctionLFBO.R new file mode 100644 index 00000000..c337bc35 --- /dev/null +++ b/R/AcqFunctionLFBO.R @@ -0,0 +1,117 @@ +#' @title Acquisition Function LFBO +#' +#' @include AcqFunction.R +#' @name mlr_acqfunctions_lfbo +#' +#' @templateVar id lfbo +#' @template section_dictionary_acqfunctions +#' +#' @description +#' Likelihood free Bayesian Optimization. +#' Parameters specifying the weighting type and the gamma quantile of the target distribition have to be set via +#' the [paradox::ParamSet] of the [LearnerRegrLFBO] used within the [SurrogateLearner]. +#' +#' @references +#' * `r format_bib("song_2022")` +#' +#' @family Acquisition Function +#' @export +#' @examples +#' if (requireNamespace("mlr3learners") & +#' requireNamespace("ranger")) { +#' library(bbotk) +#' library(paradox) +#' library(mlr3learners) +#' library(data.table) +#' +#' fun = function(xs) { +#' list(y = xs$x ^ 2) +#' } +#' domain = ps(x = p_dbl(lower = -10, upper = 10)) +#' codomain = ps(y = p_dbl(tags = "minimize")) +#' objective = ObjectiveRFun$new(fun = fun, domain = domain, codomain = codomain) +#' +#' instance = OptimInstanceSingleCrit$new( +#' objective = objective, +#' terminator = trm("evals", n_evals = 5)) +#' +#' instance$eval_batch(data.table(x = c(-6, -5, 3, 9))) +#' +#' learner = LearnerRegrLFBO$new(lrn("classif.ranger")) +#' +#' surrogate = srlrn(learner, archive = instance$archive) +#' +#' acq_function = acqf("lfbo", surrogate = surrogate) +#' +#' acq_function$surrogate$update() +#' acq_function$update() +#' acq_function$eval_dt(data.table(x = c(-1, 0, 1))) +#' } +AcqFunctionLFBO = R6Class("AcqFunctionLFBO", + inherit = AcqFunction, + + public = list( + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + #' + #' @param surrogate (`NULL` | [SurrogateLearner]). + initialize = function(surrogate = NULL) { + assert_r6(surrogate, "SurrogateLearner", null.ok = TRUE) + + if (!is.null(surrogate)) { + assert_r6(surrogate$model, "LearnerRegrLFBO", null.ok = TRUE) + surrogate$model$surrogate_max_to_min = surrogate_mult_max_to_min(surrogate$archive$codomain, y_cols = surrogate$y_cols) + } + + super$initialize("acq_lfbo", surrogate = surrogate, direction = "maximize", label = "Likelihood Free Bayesian Optimization", man = "mlr3mbo::mlr_acqfunctions_lfbo") + } + ), + + active = list( + #' @field surrogate ([SurrogateLearner])\cr + #' Surrogate learner encapsulating a [LearnerRegrLFBO]. + surrogate = function(rhs) { + if (missing(rhs)) { + private$.surrogate + } else { + assert_r6(rhs, classes = "SurrogateLearner") + assert_r6(surrogate$model, "LearnerRegrLFBO") + private$.surrogate = rhs + private$.archive = assert_r6(rhs$archive, classes = "Archive") + codomain = generate_acq_codomain(rhs$archive$codomain, id = self$id, direction = self$direction) + self$surrogate_max_to_min = surrogate_mult_max_to_min(rhs$archive$codomain, y_cols = rhs$y_cols) + domain = rhs$archive$search_space$clone(deep = TRUE) + domain$trafo = NULL + self$codomain = Codomain$new(codomain$params) # lazy initialization requires this + self$domain = domain + + private$.surrogate$model$surrogate_max_to_min = self$surrogate_max_to_min + } + }, + + #' @field surrogate_max_to_min (`-1` | `1`)\cr + #' Multiplicative factor to correct for minimization or maximization of the acquisition function. + #' Changing this value will be propagated to the `$surrogate$model$surrogate_max_to_min` when possible. + surrogate_max_to_min = function(rhs) { + if (missing(rhs)) { + private$.surrogate_max_to_min + } else { + private$.surrogate_max_to_min = assert_subset(rhs, choices = c(-1L, 1L)) + if (!is.null(private$.surrogate)) { + private$.surrogate$model$surrogate_max_to_min = private$.surrogate_max_to_min + } + } + } + ), + + private = list( + .fun = function(xdt, ...) { + p = self$surrogate$predict(xdt) + data.table(acq_lfbo = p$mean) + } + ) +) + +mlr_acqfunctions$add("lfbo", AcqFunctionLFBO) + diff --git a/R/LearnerRegrLFBO.R b/R/LearnerRegrLFBO.R index 079e0065..be95fd5e 100644 --- a/R/LearnerRegrLFBO.R +++ b/R/LearnerRegrLFBO.R @@ -1,106 +1,196 @@ #' @title Likelihood Free Bayesian Optimization Learner #' +#' @name mlr_learners_regr_lfbo +#' #' @description #' Wraps a classification learner to be used as a regression learner for -#' Likelihood-free Optimization. +#' Likelihood-free Bayesian Optimization (LFBO). #' #' @details #' The regression data is internally converted to a weighted classification task. -#' The new objective is a weighted version of a prediction problem, where the goal -#' to predict whether an instance is smaller than the `gamma` quantile of the target -#' distribution. +#' The new objective is a weighted version of a prediction problem, where the goal is +#' to predict whether a target value is smaller than a gamma quantile of the target +#' distribution (assuming minimization). +#' +#' To specify the weighting type, set the value of the `lfbo.wt` parameter (`"ei"` or `"pi"`). +#' To specify the `gamma` quantile of the target distribition set the value of the `lfbo.gamma` parameter. #' #' @references #' * `r format_bib("song_2022")` #' #' @export +#' @examples +#' \donttest{ +#' if (requireNamespace("mlr3learners") & +#' requireNamespace("ranger")) { +#' +#' library(bbotk) +#' library(paradox) +#' library(mlr3learners) +#' +#' fun = function(xs) { +#' list(y = xs$x ^ 2) +#' } +#' domain = ps(x = p_dbl(lower = -10, upper = 10)) +#' codomain = ps(y = p_dbl(tags = "minimize")) +#' objective = ObjectiveRFun$new(fun = fun, domain = domain, codomain = codomain) +#' +#' instance = OptimInstanceSingleCrit$new( +#' objective = objective, +#' terminator = trm("evals", n_evals = 5)) +#' +#' surrogate = srlrn(lrn("regr.lfbo", lrn("classif.ranger"))) +#' +#' acq_function = acqf("lfbo") +#' +#' acq_optimizer = acqo( +#' optimizer = opt("random_search", batch_size = 100), +#' terminator = trm("evals", n_evals = 100)) +#' +#' optimizer = opt("mbo", +#' loop_function = bayesopt_ego, +#' surrogate = surrogate, +#' acq_function = acq_function, +#' acq_optimizer = acq_optimizer) +#' +#' optimizer$optimize(instance) +#' } +#' } LearnerRegrLFBO = R6Class("LearnerRegrLFBO", inherit = mlr3::LearnerRegr, public = list( - - #' @field clf ([mlr3::LearnerClassif])\cr - #' Classifier to be used for LFBO. - clf = NULL, - #' @field wt (character)\cr - #' Weighting scheme for LFBO. Currently allows "ei" and "pi". - wt = NULL, - #' @field gamma (numeric)\cr - #' Quantile of the target distribution. Defaults to `0.33`. - gamma = NULL, + #' @field learner_classif ([mlr3::LearnerClassif])\cr + #' Classification learner to be used for LFBO. + learner_classif = NULL, #' @description #' Creates a new instance of this [R6][R6::R6Class] class. #' - #' @param clf ([mlr3::LearnerClassif])\cr - #' Classifier to be used for LFBO. Needs to have predict_type "prob". - #' @param wt (character)\cr - #' Weighting scheme for LFBO. Currently allows "ei" (expected improvement) and "pi" (probability of improvement). - #' @param gamma (numeric)\cr - #' Quantile of the target distribution. Defaults to `0.33`. - initialize = function(clf, gamma = 0.33, wt = "ei") { - assert_class(clf, "LearnerClassif") - assert_true(clf$predict_type == "prob") - assert_true("weights" %in% clf$properties) - # FIXME: Should we expose gamma, wt as params, and if: how without making things ugly. - self$clf = clf - self$gamma = assert_number(gamma, lower = 0, upper = 1) - self$wt = assert_choice(wt, choices = c("ei", "pi")) - props = setdiff(clf$properties, c("twoclass", "multiclass")) + #' @param learner_classif ([mlr3::LearnerClassif])\cr + #' Classifcation learner to be used for LFBO. + #' Requires `predict_type = "prob"` which will be set automatically during construction. + #' Also requires the learner to be able to handle case weights, see `learner$properties`. + initialize = function(learner_classif) { + assert_class(learner_classif, "LearnerClassif") + assert_true("prob" %in% learner_classif$predict_types) + learner_classif$predict_type = "prob" + assert_true("weights" %in% learner_classif$properties) + self$learner_classif = learner_classif + + lfbo_param_set = ps(wt = p_fct(levels = c("ei", "pi"), default = "ei", tags = "train"), gamma = p_dbl(lower = 0, upper = 1, default = 1/3, tags = "train")) + lfbo_param_set$set_id = "lfbo" + super$initialize( - id = paste0(clf$id, "2regr"), - param_set = clf$param_set, feature_types = clf$feature_types, - predict_types = c("response"), - properties = props, data_formats = clf$data_formats, packages = clf$packages, - label = clf$label, man = clf$man + id = "regr.lfbo", + param_set = ParamSetCollection$new(list(lfbo_param_set, self$learner_classif$param_set)), + feature_types = learner_classif$feature_types, + predict_types = "response", + properties = setdiff(learner_classif$properties, c("twoclass", "multiclass")), + data_formats = learner_classif$data_formats, + packages = learner_classif$packages, + label = learner_classif$label, + man = "mlr3mbo::mlr_learners_regr_lfbo" ) }, + + #' @description + #' Train the learner on a set of observations of the provided `task`. + #' Mutates the learner by reference, i.e. stores the model alongside other information in field `$state`. + #' + #' @param task ([TaskRegr]). + #' + #' @param row_ids (`integer()`)\cr + #' Vector of training indices as subset of `task$row_ids`. + #' For a simple split into training and test set, see [mlr3::partition()]. + #' + #' @return + #' Returns the object itself, but modified **by reference**. + #' You need to explicitly `$clone()` the object beforehand if you want to keeps + #' the object in its previous state. train = function(task, row_ids = NULL) { - # create df with new data. - tn = private$.to_weighted_task(task, row_ids) - self$clf$train(tn, row_ids) - self$state$train_task = mlr3:::task_rm_backend(task$clone(deep = TRUE)) - invisible(NULL) + # create df with new data + if (is.null(self$surrogate_max_to_min)) { + stop("$surrogate_max_to_min must be set prior to training.") + } + row_ids = assert_row_ids(row_ids, null.ok = TRUE) + task_new = private$.to_weighted_task(task, row_ids) + self$learner_classif$train(task_new, row_ids) + self$state$train_task = task_rm_backend(task$clone(deep = TRUE)) + invisible(NULL) }, + + #' @description + #' Uses the information stored during `$train()` in `$learner$classif$state` to create a new [mlr3::PredictionRegr] + #' for a set of observations of the provided `task`. + #' + #' @param task ([mlr3::TaskRegr]). + #' + #' @param row_ids (`integer()`)\cr + #' Vector of test indices as subset of `task$row_ids`. + #' For a simple split into training and test set, see [mlr3::partition()]. + #' + #' @return [PredictionRegr]. predict = function(task, row_ids = NULL) { - fts = task$data(rows = row_ids, cols = task$feature_names) - colnames(fts) = paste0("X.", colnames(fts)) - pd = self$clf$predict_newdata(fts)$data - pdata = as_prediction_data( - list(response = - pd$prob[, 2L]), - task = task, - row_ids = pd$row_ids - ) - as_prediction(pdata) + row_ids = assert_row_ids(row_ids, null.ok = TRUE) + features = task$data(rows = row_ids, cols = task$feature_names) + colnames(features) = paste0("X.", colnames(features)) + pred = self$learner_classif$predict_newdata(features)$data + prediction_data = as_prediction_data(list(response = pred$prob[, 2L]), task = task, row_ids = pred$row_ids) + as_prediction(prediction_data) + } + ), + + active = list( + #' @field surrogate_max_to_min (`-1` | `1`)\cr + #' Multiplicative factor to correct for minimization or maximization. + surrogate_max_to_min = function(rhs) { + if (missing(rhs)) { + private$.surrogate_max_to_min + } else { + private$.surrogate_max_to_min = assert_subset(rhs, choices = c(-1L, 1L)) + } } ), + private = list( + .surrogate_max_to_min = NULL, + .to_weighted_task = function(task, row_ids) { - dt = do.call("cbind", private$.regr_to_weights( + data = do.call("cbind", private$.regr_to_weights( task$data(rows = row_ids, cols = task$feature_names), task$data(rows = row_ids, cols = task$target_names)[[1]] )) - tn = TaskClassif$new("weighted_classif", dt, target = "z") - tn$set_col_roles("w", "weight") + task_new = TaskClassif$new("weighted_classif", data, target = "z") + task_new$set_col_roles("w", "weight") }, - .regr_to_weights = function(X,y, gamma = .8, wt = "ei") { - # Adapted from https://github.com/lfbo-ml/lfbo - tau = quantile(y, probs = self$gamma) + + .regr_to_weights = function(X, y) { + # wt and gamma are inferred from the param_set + # surrogate_max_to_min is inferred from the active bindings + # adapted from https://github.com/lfbo-ml/lfbo + wt = if (is.null(self$param_set$values$lfbo.wt)) "ei" else self$param_set$values$lfbo.wt + gamma = if (is.null(self$param_set$values$lfbo.gamma)) 1/3 else self$param_set$values$lfbo.gamma + y = self$surrogate_max_to_min * y + tau = quantile(y, probs = gamma) z = y < tau - if (self$wt == "ei") { + if (wt == "ei") { w1 = tau - y[z] w1 = w1 / mean(w1) Xn = rbind(X, X[z,]) zn = c(rep(0, length(z)), z[z]) wn = c(rep(1, length(z)) / length(z), w1 * sum(z)) wn = wn / mean(wn) - } else if (self$wt == "pi") { + } else if (wt == "pi") { Xn = X wn = rep(1, nrow(X)) zn = z } - return(list(X=Xn,w=wn,z=as.factor(zn))) - } + return(list(X = Xn, w = wn, z = as.factor(zn))) + }, + + .train = function(task) {}, + + .predict = function(task) {} ) ) -# mlr_learners$add("regr.lfbo", function() LearnerRegrLFBO$new()) diff --git a/R/helper.R b/R/helper.R index 3035a6d8..fa0e24f2 100644 --- a/R/helper.R +++ b/R/helper.R @@ -120,3 +120,14 @@ assert_xdt = function(xdt) { assert_data_table(xdt) } +task_rm_backend = function(task) { + # fix task hash + ee = get_private(task) + ee$.hash = force(task$hash) + ee$.col_hashes = force(task$col_hashes) + + # NULL backend + task$backend = NULL + + task +} diff --git a/R/zzz.R b/R/zzz.R index a2d67ae4..02c0d542 100644 --- a/R/zzz.R +++ b/R/zzz.R @@ -21,6 +21,10 @@ x = utils::getFromNamespace("mlr_optimizers", ns = "bbotk") x$add("mbo", OptimizerMbo) + # add regr.lfbo to learner dictionary + x = utils::getFromNamespace("mlr_learners", ns = "mlr3") + x$add("regr.lfbo", LearnerRegrLFBO) + # setup logger assign("lg", lgr::get_logger("bbotk"), envir = parent.env(environment())) diff --git a/man/AcqFunction.Rd b/man/AcqFunction.Rd index 7af441c5..88586afa 100644 --- a/man/AcqFunction.Rd +++ b/man/AcqFunction.Rd @@ -14,6 +14,7 @@ Other Acquisition Function: \code{\link{mlr_acqfunctions_cb}}, \code{\link{mlr_acqfunctions_eips}}, \code{\link{mlr_acqfunctions_ei}}, +\code{\link{mlr_acqfunctions_lfbo}}, \code{\link{mlr_acqfunctions_mean}}, \code{\link{mlr_acqfunctions_pi}}, \code{\link{mlr_acqfunctions_smsego}}, diff --git a/man/mlr_acqfunctions.Rd b/man/mlr_acqfunctions.Rd index 96e58c69..7a2b9e50 100644 --- a/man/mlr_acqfunctions.Rd +++ b/man/mlr_acqfunctions.Rd @@ -34,6 +34,7 @@ Other Acquisition Function: \code{\link{mlr_acqfunctions_cb}}, \code{\link{mlr_acqfunctions_eips}}, \code{\link{mlr_acqfunctions_ei}}, +\code{\link{mlr_acqfunctions_lfbo}}, \code{\link{mlr_acqfunctions_mean}}, \code{\link{mlr_acqfunctions_pi}}, \code{\link{mlr_acqfunctions_smsego}} diff --git a/man/mlr_acqfunctions_aei.Rd b/man/mlr_acqfunctions_aei.Rd index 5c58ec9b..de1a8cbe 100644 --- a/man/mlr_acqfunctions_aei.Rd +++ b/man/mlr_acqfunctions_aei.Rd @@ -82,6 +82,7 @@ Other Acquisition Function: \code{\link{mlr_acqfunctions_cb}}, \code{\link{mlr_acqfunctions_eips}}, \code{\link{mlr_acqfunctions_ei}}, +\code{\link{mlr_acqfunctions_lfbo}}, \code{\link{mlr_acqfunctions_mean}}, \code{\link{mlr_acqfunctions_pi}}, \code{\link{mlr_acqfunctions_smsego}}, diff --git a/man/mlr_acqfunctions_cb.Rd b/man/mlr_acqfunctions_cb.Rd index fb764de5..2bac14b7 100644 --- a/man/mlr_acqfunctions_cb.Rd +++ b/man/mlr_acqfunctions_cb.Rd @@ -75,6 +75,7 @@ Other Acquisition Function: \code{\link{mlr_acqfunctions_aei}}, \code{\link{mlr_acqfunctions_eips}}, \code{\link{mlr_acqfunctions_ei}}, +\code{\link{mlr_acqfunctions_lfbo}}, \code{\link{mlr_acqfunctions_mean}}, \code{\link{mlr_acqfunctions_pi}}, \code{\link{mlr_acqfunctions_smsego}}, diff --git a/man/mlr_acqfunctions_ei.Rd b/man/mlr_acqfunctions_ei.Rd index 863a1822..4659648f 100644 --- a/man/mlr_acqfunctions_ei.Rd +++ b/man/mlr_acqfunctions_ei.Rd @@ -67,6 +67,7 @@ Other Acquisition Function: \code{\link{mlr_acqfunctions_aei}}, \code{\link{mlr_acqfunctions_cb}}, \code{\link{mlr_acqfunctions_eips}}, +\code{\link{mlr_acqfunctions_lfbo}}, \code{\link{mlr_acqfunctions_mean}}, \code{\link{mlr_acqfunctions_pi}}, \code{\link{mlr_acqfunctions_smsego}}, diff --git a/man/mlr_acqfunctions_eips.Rd b/man/mlr_acqfunctions_eips.Rd index 3455d5ad..db8fad44 100644 --- a/man/mlr_acqfunctions_eips.Rd +++ b/man/mlr_acqfunctions_eips.Rd @@ -75,6 +75,7 @@ Other Acquisition Function: \code{\link{mlr_acqfunctions_aei}}, \code{\link{mlr_acqfunctions_cb}}, \code{\link{mlr_acqfunctions_ei}}, +\code{\link{mlr_acqfunctions_lfbo}}, \code{\link{mlr_acqfunctions_mean}}, \code{\link{mlr_acqfunctions_pi}}, \code{\link{mlr_acqfunctions_smsego}}, diff --git a/man/mlr_acqfunctions_lfbo.Rd b/man/mlr_acqfunctions_lfbo.Rd new file mode 100644 index 00000000..cba397ff --- /dev/null +++ b/man/mlr_acqfunctions_lfbo.Rd @@ -0,0 +1,142 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/AcqFunctionLFBO.R +\name{mlr_acqfunctions_lfbo} +\alias{mlr_acqfunctions_lfbo} +\alias{AcqFunctionLFBO} +\title{Acquisition Function LFBO} +\description{ +Likelihood free Bayesian Optimization. +Parameters specifying the weighting type and the gamma quantile of the target distribition have to be set via +the \link[paradox:ParamSet]{paradox::ParamSet} of the \link{LearnerRegrLFBO} used within the \link{SurrogateLearner}. +} +\section{Dictionary}{ + +This \link{AcqFunction} can be instantiated via the \link[mlr3misc:Dictionary]{dictionary} +\link{mlr_acqfunctions} or with the associated sugar function \code{\link[=acqf]{acqf()}}: + +\if{html}{\out{
}}\preformatted{mlr_acqfunctions$get("lfbo") +acqf("lfbo") +}\if{html}{\out{
}} +} + +\examples{ +if (requireNamespace("mlr3learners") & + requireNamespace("ranger")) { + library(bbotk) + library(paradox) + library(mlr3learners) + library(data.table) + + fun = function(xs) { + list(y = xs$x ^ 2) + } + domain = ps(x = p_dbl(lower = -10, upper = 10)) + codomain = ps(y = p_dbl(tags = "minimize")) + objective = ObjectiveRFun$new(fun = fun, domain = domain, codomain = codomain) + + instance = OptimInstanceSingleCrit$new( + objective = objective, + terminator = trm("evals", n_evals = 5)) + + instance$eval_batch(data.table(x = c(-6, -5, 3, 9))) + + learner = LearnerRegrLFBO$new(lrn("classif.ranger")) + + surrogate = srlrn(learner, archive = instance$archive) + + acq_function = acqf("lfbo", surrogate = surrogate) + + acq_function$surrogate$update() + acq_function$update() + acq_function$eval_dt(data.table(x = c(-1, 0, 1))) +} +} +\references{ +\itemize{ +\item Song, Jiaming, Yu, Lantao, Neiswanger, Willie, Ermon, Stefano (2022). +\dQuote{A General Recipe for Likelihood-free Bayesian Optimization.} +\emph{Proceedings of Machine Learning Research}, \bold{162}, 20384--20404. +} +} +\seealso{ +Other Acquisition Function: +\code{\link{AcqFunction}}, +\code{\link{mlr_acqfunctions_aei}}, +\code{\link{mlr_acqfunctions_cb}}, +\code{\link{mlr_acqfunctions_eips}}, +\code{\link{mlr_acqfunctions_ei}}, +\code{\link{mlr_acqfunctions_mean}}, +\code{\link{mlr_acqfunctions_pi}}, +\code{\link{mlr_acqfunctions_smsego}}, +\code{\link{mlr_acqfunctions}} +} +\concept{Acquisition Function} +\section{Super classes}{ +\code{\link[bbotk:Objective]{bbotk::Objective}} -> \code{\link[mlr3mbo:AcqFunction]{mlr3mbo::AcqFunction}} -> \code{AcqFunctionLFBO} +} +\section{Active bindings}{ +\if{html}{\out{
}} +\describe{ +\item{\code{surrogate}}{(\link{SurrogateLearner})\cr +Surrogate learner encapsulating a \link{LearnerRegrLFBO}.} + +\item{\code{surrogate_max_to_min}}{(\code{-1} | \code{1})\cr +Multiplicative factor to correct for minimization or maximization of the acquisition function. +Changing this value will be propagated to the \verb{$surrogate$model$surrogate_max_to_min} when possible.} +} +\if{html}{\out{
}} +} +\section{Methods}{ +\subsection{Public methods}{ +\itemize{ +\item \href{#method-AcqFunctionLFBO-new}{\code{AcqFunctionLFBO$new()}} +\item \href{#method-AcqFunctionLFBO-clone}{\code{AcqFunctionLFBO$clone()}} +} +} +\if{html}{\out{ +
Inherited methods + +
+}} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqFunctionLFBO-new}{}}} +\subsection{Method \code{new()}}{ +Creates a new instance of this \link[R6:R6Class]{R6} class. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{AcqFunctionLFBO$new(surrogate = NULL)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{surrogate}}{(\code{NULL} | \link{SurrogateLearner}).} +} +\if{html}{\out{
}} +} +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqFunctionLFBO-clone}{}}} +\subsection{Method \code{clone()}}{ +The objects of this class are cloneable with this method. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{AcqFunctionLFBO$clone(deep = FALSE)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{deep}}{Whether to make a deep clone.} +} +\if{html}{\out{
}} +} +} +} diff --git a/man/mlr_acqfunctions_mean.Rd b/man/mlr_acqfunctions_mean.Rd index d0d01569..783a2cb2 100644 --- a/man/mlr_acqfunctions_mean.Rd +++ b/man/mlr_acqfunctions_mean.Rd @@ -61,6 +61,7 @@ Other Acquisition Function: \code{\link{mlr_acqfunctions_cb}}, \code{\link{mlr_acqfunctions_eips}}, \code{\link{mlr_acqfunctions_ei}}, +\code{\link{mlr_acqfunctions_lfbo}}, \code{\link{mlr_acqfunctions_pi}}, \code{\link{mlr_acqfunctions_smsego}}, \code{\link{mlr_acqfunctions}} diff --git a/man/mlr_acqfunctions_pi.Rd b/man/mlr_acqfunctions_pi.Rd index dfb1ef61..0569eb92 100644 --- a/man/mlr_acqfunctions_pi.Rd +++ b/man/mlr_acqfunctions_pi.Rd @@ -68,6 +68,7 @@ Other Acquisition Function: \code{\link{mlr_acqfunctions_cb}}, \code{\link{mlr_acqfunctions_eips}}, \code{\link{mlr_acqfunctions_ei}}, +\code{\link{mlr_acqfunctions_lfbo}}, \code{\link{mlr_acqfunctions_mean}}, \code{\link{mlr_acqfunctions_smsego}}, \code{\link{mlr_acqfunctions}} diff --git a/man/mlr_acqfunctions_smsego.Rd b/man/mlr_acqfunctions_smsego.Rd index 46167233..117af773 100644 --- a/man/mlr_acqfunctions_smsego.Rd +++ b/man/mlr_acqfunctions_smsego.Rd @@ -78,6 +78,7 @@ Other Acquisition Function: \code{\link{mlr_acqfunctions_cb}}, \code{\link{mlr_acqfunctions_eips}}, \code{\link{mlr_acqfunctions_ei}}, +\code{\link{mlr_acqfunctions_lfbo}}, \code{\link{mlr_acqfunctions_mean}}, \code{\link{mlr_acqfunctions_pi}}, \code{\link{mlr_acqfunctions}} diff --git a/man/mlr_learners_regr_lfbo.Rd b/man/mlr_learners_regr_lfbo.Rd new file mode 100644 index 00000000..eb1f5521 --- /dev/null +++ b/man/mlr_learners_regr_lfbo.Rd @@ -0,0 +1,194 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/LearnerRegrLFBO.R +\name{mlr_learners_regr_lfbo} +\alias{mlr_learners_regr_lfbo} +\alias{LearnerRegrLFBO} +\title{Likelihood Free Bayesian Optimization Learner} +\description{ +Wraps a classification learner to be used as a regression learner for +Likelihood-free Bayesian Optimization (LFBO). +} +\details{ +The regression data is internally converted to a weighted classification task. +The new objective is a weighted version of a prediction problem, where the goal is +to predict whether a target value is smaller than a gamma quantile of the target +distribution (assuming minimization). + +To specify the weighting type, set the value of the \code{lfbo.wt} parameter (\code{"ei"} or \code{"pi"}). +To specify the \code{gamma} quantile of the target distribition set the value of the \code{lfbo.gamma} parameter. +} +\examples{ +\donttest{ +if (requireNamespace("mlr3learners") & + requireNamespace("ranger")) { + + library(bbotk) + library(paradox) + library(mlr3learners) + + fun = function(xs) { + list(y = xs$x ^ 2) + } + domain = ps(x = p_dbl(lower = -10, upper = 10)) + codomain = ps(y = p_dbl(tags = "minimize")) + objective = ObjectiveRFun$new(fun = fun, domain = domain, codomain = codomain) + + instance = OptimInstanceSingleCrit$new( + objective = objective, + terminator = trm("evals", n_evals = 5)) + + surrogate = srlrn(lrn("regr.lfbo", lrn("classif.ranger"))) + + acq_function = acqf("lfbo") + + acq_optimizer = acqo( + optimizer = opt("random_search", batch_size = 100), + terminator = trm("evals", n_evals = 100)) + + optimizer = opt("mbo", + loop_function = bayesopt_ego, + surrogate = surrogate, + acq_function = acq_function, + acq_optimizer = acq_optimizer) + + optimizer$optimize(instance) +} +} +} +\references{ +\itemize{ +\item Song, Jiaming, Yu, Lantao, Neiswanger, Willie, Ermon, Stefano (2022). +\dQuote{A General Recipe for Likelihood-free Bayesian Optimization.} +\emph{Proceedings of Machine Learning Research}, \bold{162}, 20384--20404. +} +} +\section{Super classes}{ +\code{\link[mlr3:Learner]{mlr3::Learner}} -> \code{\link[mlr3:LearnerRegr]{mlr3::LearnerRegr}} -> \code{LearnerRegrLFBO} +} +\section{Public fields}{ +\if{html}{\out{
}} +\describe{ +\item{\code{learner_classif}}{(\link[mlr3:LearnerClassif]{mlr3::LearnerClassif})\cr +Classification learner to be used for LFBO.} +} +\if{html}{\out{
}} +} +\section{Active bindings}{ +\if{html}{\out{
}} +\describe{ +\item{\code{surrogate_max_to_min}}{(\code{-1} | \code{1})\cr +Multiplicative factor to correct for minimization or maximization.} +} +\if{html}{\out{
}} +} +\section{Methods}{ +\subsection{Public methods}{ +\itemize{ +\item \href{#method-LearnerRegrLFBO-new}{\code{LearnerRegrLFBO$new()}} +\item \href{#method-LearnerRegrLFBO-train}{\code{LearnerRegrLFBO$train()}} +\item \href{#method-LearnerRegrLFBO-predict}{\code{LearnerRegrLFBO$predict()}} +\item \href{#method-LearnerRegrLFBO-clone}{\code{LearnerRegrLFBO$clone()}} +} +} +\if{html}{\out{ +
Inherited methods + +
+}} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-LearnerRegrLFBO-new}{}}} +\subsection{Method \code{new()}}{ +Creates a new instance of this \link[R6:R6Class]{R6} class. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{LearnerRegrLFBO$new(learner_classif)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{learner_classif}}{(\link[mlr3:LearnerClassif]{mlr3::LearnerClassif})\cr +Classifcation learner to be used for LFBO. +Requires \code{predict_type = "prob"} which will be set automatically during construction. +Also requires the learner to be able to handle case weights, see \code{learner$properties}.} +} +\if{html}{\out{
}} +} +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-LearnerRegrLFBO-train}{}}} +\subsection{Method \code{train()}}{ +Train the learner on a set of observations of the provided \code{task}. +Mutates the learner by reference, i.e. stores the model alongside other information in field \verb{$state}. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{LearnerRegrLFBO$train(task, row_ids = NULL)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{task}}{(\link{TaskRegr}).} + +\item{\code{row_ids}}{(\code{integer()})\cr +Vector of training indices as subset of \code{task$row_ids}. +For a simple split into training and test set, see \code{\link[mlr3:partition]{mlr3::partition()}}.} +} +\if{html}{\out{
}} +} +\subsection{Returns}{ +Returns the object itself, but modified \strong{by reference}. +You need to explicitly \verb{$clone()} the object beforehand if you want to keeps +the object in its previous state. +} +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-LearnerRegrLFBO-predict}{}}} +\subsection{Method \code{predict()}}{ +Uses the information stored during \verb{$train()} in \verb{$learner$classif$state} to create a new \link[mlr3:PredictionRegr]{mlr3::PredictionRegr} +for a set of observations of the provided \code{task}. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{LearnerRegrLFBO$predict(task, row_ids = NULL)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{task}}{(\link[mlr3:TaskRegr]{mlr3::TaskRegr}).} + +\item{\code{row_ids}}{(\code{integer()})\cr +Vector of test indices as subset of \code{task$row_ids}. +For a simple split into training and test set, see \code{\link[mlr3:partition]{mlr3::partition()}}.} +} +\if{html}{\out{
}} +} +\subsection{Returns}{ +\link{PredictionRegr}. +} +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-LearnerRegrLFBO-clone}{}}} +\subsection{Method \code{clone()}}{ +The objects of this class are cloneable with this method. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{LearnerRegrLFBO$clone(deep = FALSE)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{deep}}{Whether to make a deep clone.} +} +\if{html}{\out{
}} +} +} +} diff --git a/tests/testthat/test_AcqFunctionLFBO.R b/tests/testthat/test_AcqFunctionLFBO.R new file mode 100644 index 00000000..976646bc --- /dev/null +++ b/tests/testthat/test_AcqFunctionLFBO.R @@ -0,0 +1,31 @@ +test_that("AcqFunctionLFBO works", { + inst = MAKE_INST_1D() + surrogate = SurrogateLearner$new(lrn("regr.lfbo", learner_classif = lrn("classif.rpart")), archive = inst$archive) + acqf = AcqFunctionLFBO$new(surrogate = surrogate) + expect_acqfunction(acqf) + + expect_r6(acqf$codomain, "ParamSet") + expect_equal(acqf$codomain$ids(), acqf$id) + expect_equal(acqf$surrogate_max_to_min, c(y = 1)) + expect_equal(acqf$direction, "maximize") + expect_equal(acqf$domain, inst$search_space) + expect_learner(acqf$surrogate$model) + expect_learner(acqf$surrogate$model$learner_classif) + + design = MAKE_DESIGN(inst) + inst$eval_batch(design) + + acqf$surrogate$update() + xdt = data.table(x = seq(-1, 1, length.out = 5L)) + res = acqf$eval_dt(xdt) + expect_data_table(res, ncols = 1L, nrows = 5L, any.missing = FALSE) + expect_named(res, acqf$id) + + expect_true(acqf$surrogate_max_to_min == 1) + expect_true(acqf$surrogate$model$surrogate_max_to_min == 1) + + acqf$surrogate_max_to_min = -1 + expect_true(acqf$surrogate_max_to_min == -1) + expect_true(acqf$surrogate$model$surrogate_max_to_min == -1) +}) + diff --git a/tests/testthat/test_learner_lfbo.R b/tests/testthat/test_learner_lfbo.R deleted file mode 100644 index d6afa15b..00000000 --- a/tests/testthat/test_learner_lfbo.R +++ /dev/null @@ -1,55 +0,0 @@ -test_that("LFBO Learner can be constructed and trained", { - l1 = lrn("classif.rpart", predict_type = "prob") - expect_class(l1, "LearnerClassif") - l2 = LearnerLFBO$new(l1) - expect_class(l2, "LearnerRegr") - expect_true(l2$id == "classif.rpart2regr") - s1 = SurrogateLearner$new(l2) - expect_class(s1, "SurrogateLearner") - - t = tsk("boston_housing") - l2$train(t) - expect_true(!is.null(l2$state)) - prd = l2$predict(t) - # Note: Truth value does not make sense! - expect_class(prd, "PredictionRegr") -}) - -test_that("LFBO Learner works for optim", { - set.seed(4321L) - funs = list( - fun = function(xs) { - list(y = xs$x ^ 2) - }, - fun = function(xs) { - list(y = - xs$x ^ 2) - } - ) - codomains = list( - ps(y = p_dbl(tags = "minimize")), - ps(y = p_dbl(tags = "maximize")) - ) - for (i in seq_len(2)) { - domain = ps(x = p_dbl(lower = -10, upper = 10)) - fun = funs[[i]] - codomain = codomains[[i]] - objective = ObjectiveRFun$new(fun = fun, domain = domain, codomain = codomain) - instance = OptimInstanceSingleCrit$new( - objective = objective, - terminator = trm("evals", n_evals = 20) - ) - surrogate = SurrogateLearner$new(LearnerLFBO$new(lrn("classif.ranger", predict_type = "prob", num.trees=2L))) - acq_function = acqf("mean") - acq_optimizer = acqo( - optimizer = opt("random_search", batch_size = 100), - terminator = trm("evals", n_evals = 100)) - optimizer = opt("mbo", - loop_function = bayesopt_ego, - surrogate = surrogate, - acq_function = acq_function, - acq_optimizer = acq_optimizer - ) - optimizer$optimize(instance) - expect_true(abs(instance$result$y) < .1) - } -}) diff --git a/tests/testthat/test_regr_lfbo.R b/tests/testthat/test_regr_lfbo.R new file mode 100644 index 00000000..8ebf5faf --- /dev/null +++ b/tests/testthat/test_regr_lfbo.R @@ -0,0 +1,71 @@ +test_that("LearnerRegrLFBO can be constructed and trained and setting hyperparameters works", { + learner_classif = lrn("classif.rpart", predict_type = "prob") + expect_class(learner_classif, "LearnerClassif") + learner = LearnerRegrLFBO$new(learner_classif) + expect_class(learner, "LearnerRegr") + expect_learner(learner) + expect_true(learner$id == "regr.lfbo") + surrogate = SurrogateLearner$new(learner) + expect_class(surrogate, "SurrogateLearner") + + task = tsk("mtcars") + expect_error(learner$train(task), regexp = "surrogate_max_to_min must be set") + expect_error({learner$surrogate_max_to_min = 10}, regexp = "Must be a subset") + learner$surrogate_max_to_min = 1 + learner$train(task) + expect_true(!is.null(learner$state)) + prediction = learner$predict(task) + expect_class(prediction, "PredictionRegr") # Note: Truth value does not make sense! + expect_true(cor(prediction$truth, prediction$response) < - 0.5) + + learner$surrogate_max_to_min = -1 + learner$train(task) + expect_true(!is.null(learner$state)) + prediction = learner$predict(task) + expect_class(prediction, "PredictionRegr") # Note: Truth value does not make sense! + expect_true(cor(prediction$truth, prediction$response) > 0.5) + + lfbo_param_set = ps(wt = p_fct(levels = c("ei", "pi"), default = "ei", tags = "train"), gamma = p_dbl(lower = 0, upper = 1, default = 1/3, tags = "train")) + lfbo_param_set$set_id = "lfbo" + param_set = ParamSetCollection$new(list(lfbo_param_set, learner_classif$param_set)) + expect_equal(param_set, learner$param_set) + + + learner$param_set$values$lfbo.wt = "pi" + learner$param_set$values$lfbo.gamma = 0.5 + learner$param_set$values$maxdepth = 20L + + expect_true(learner_classif$param_set$values$maxdepth == 20L) + + learner$train(task) + expect_true(!is.null(learner$state)) + prediction = learner$predict(task) + expect_class(prediction, "PredictionRegr") # Note: Truth value does not make sense! +}) + +test_that("Likelihood Free Bayesian Optimization", { + with_seed(2906, { + instance = MAKE_INST_1D(terminator = trm("evals", n_evals = 10L)) + surrogate = SurrogateLearner$new(lrn("regr.lfbo", lrn("classif.ranger", predict_type = "prob", num.trees = 10L))) + acq_function = acqf("lfbo") + acq_optimizer = acqo(optimizer = opt("random_search", batch_size = 100L), terminator = trm("evals", n_evals = 100L)) + optimizer = opt("mbo", + loop_function = bayesopt_ego, + surrogate = surrogate, + acq_function = acq_function, + acq_optimizer = acq_optimizer + ) + optimizer$optimize(instance) + expect_true(abs(instance$result$y) < 0.1) + + # maximization of - y + fun = function(xs) { + list(y = - as.numeric(xs)^2) + } + codomain = ParamSet$new(list(ParamDbl$new("y", tags = "maximize"))) + objective = ObjectiveRFun$new(fun = fun, domain = PS_1D, codomain = codomain, properties = "single-crit") + instance = MAKE_INST(objective = objective, search_space = PS_1D, terminator = trm("evals", n_evals = 10L)) + optimizer$optimize(instance) + expect_true(abs(instance$result$y) < 0.1) + }) +}) From 564d0d78bf6ccd5158fb8ecb2f4124dbdb8a4abc Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Wed, 15 Feb 2023 18:44:14 +0100 Subject: [PATCH 051/126] rerun docs --- NAMESPACE | 1 + R/AcqFunctionLFBO.R | 8 ++++---- R/LearnerRegrLFBO.R | 6 +++--- R/bibentries.R | 19 ++++++++++--------- R/zzz.R | 2 +- man/mlr_acqfunctions_lfbo.Rd | 8 ++++---- man/mlr_learners_regr_lfbo.Rd | 6 +++--- 7 files changed, 26 insertions(+), 24 deletions(-) diff --git a/NAMESPACE b/NAMESPACE index 0a22377b..e8bb8b35 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -46,6 +46,7 @@ import(paradox) importFrom(R6,R6Class) importFrom(stats,dnorm) importFrom(stats,pnorm) +importFrom(stats,quantile) importFrom(stats,runif) importFrom(stats,setNames) importFrom(utils,bibentry) diff --git a/R/AcqFunctionLFBO.R b/R/AcqFunctionLFBO.R index c337bc35..18d3e9ca 100644 --- a/R/AcqFunctionLFBO.R +++ b/R/AcqFunctionLFBO.R @@ -7,8 +7,8 @@ #' @template section_dictionary_acqfunctions #' #' @description -#' Likelihood free Bayesian Optimization. -#' Parameters specifying the weighting type and the gamma quantile of the target distribition have to be set via +#' Likelihood-Free Bayesian Optimization. +#' Parameters specifying the weighting type and the gamma quantile of the target distribution have to be set via #' the [paradox::ParamSet] of the [LearnerRegrLFBO] used within the [SurrogateLearner]. #' #' @references @@ -64,7 +64,7 @@ AcqFunctionLFBO = R6Class("AcqFunctionLFBO", surrogate$model$surrogate_max_to_min = surrogate_mult_max_to_min(surrogate$archive$codomain, y_cols = surrogate$y_cols) } - super$initialize("acq_lfbo", surrogate = surrogate, direction = "maximize", label = "Likelihood Free Bayesian Optimization", man = "mlr3mbo::mlr_acqfunctions_lfbo") + super$initialize("acq_lfbo", surrogate = surrogate, direction = "maximize", label = "Likelihood-Free Bayesian Optimization", man = "mlr3mbo::mlr_acqfunctions_lfbo") } ), @@ -76,7 +76,7 @@ AcqFunctionLFBO = R6Class("AcqFunctionLFBO", private$.surrogate } else { assert_r6(rhs, classes = "SurrogateLearner") - assert_r6(surrogate$model, "LearnerRegrLFBO") + assert_r6(rhs$model, "LearnerRegrLFBO") private$.surrogate = rhs private$.archive = assert_r6(rhs$archive, classes = "Archive") codomain = generate_acq_codomain(rhs$archive$codomain, id = self$id, direction = self$direction) diff --git a/R/LearnerRegrLFBO.R b/R/LearnerRegrLFBO.R index be95fd5e..4eb78e66 100644 --- a/R/LearnerRegrLFBO.R +++ b/R/LearnerRegrLFBO.R @@ -13,11 +13,11 @@ #' distribution (assuming minimization). #' #' To specify the weighting type, set the value of the `lfbo.wt` parameter (`"ei"` or `"pi"`). -#' To specify the `gamma` quantile of the target distribition set the value of the `lfbo.gamma` parameter. -#' +#' To specify the `gamma` quantile of the target distribution set the value of the `lfbo.gamma` parameter. +#' #' @references #' * `r format_bib("song_2022")` -#' +#' #' @export #' @examples #' \donttest{ diff --git a/R/bibentries.R b/R/bibentries.R index c3df374b..983f3cf0 100644 --- a/R/bibentries.R +++ b/R/bibentries.R @@ -46,7 +46,7 @@ bibentries = c( ), snoek_2012 = bibentry("inproceedings", - title = "Practical Bayesian Optimization of Machine Learning Algorithms", + title = "Practical {B}ayesian Optimization of Machine Learning Algorithms", author = "Snoek, Jasper and Larochelle, Hugo and Adams, Ryan P", year = "2012", booktitle = "Advances in Neural Information Processing Systems", @@ -74,7 +74,7 @@ bibentries = c( ), wang_2020 = bibentry("article", - title = "Parallel Bayesian Global Optimization of Expensive Functions", + title = "Parallel {B}ayesian Global Optimization of Expensive Functions", author = "Wang, Jialei and Clark, Scott C. and Liu, Eric and Frazier, Peter I.", year = "2020", journal = "Operations Research", @@ -107,12 +107,13 @@ bibentries = c( year = "2008" ), - song_2022 = bibentry("article", - title = "A General Recipe for Likelihood-free {B}ayesian Optimization", - author = "Song, Jiaming and Yu, Lantao and Neiswanger, Willie and Ermon, Stefano", - year = "2022", - volume = "162", - pages = "20384--20404", - journal = "Proceedings of Machine Learning Research" + song_2022 = bibentry("inproceedings", + title = "A General Recipe for Likelihood-Free {B}ayesian Optimization", + author = "Song, Jiaming and Yu, Lantao and Neiswanger, Willie and Ermon, Stefano", + year = "2022", + booktitle = "Proceedings of the 39th International Conference on Machine Learning", + editor = "K. Chaudhuri and S. Jegelka and L. Song and C. Szepesvari and G. Niu and S. Sabato", + pages = "20384--20404", + volume = "162" ) ) diff --git a/R/zzz.R b/R/zzz.R index 02c0d542..9416fcf9 100644 --- a/R/zzz.R +++ b/R/zzz.R @@ -7,7 +7,7 @@ #' @import lgr #' @import mlr3 #' @import mlr3tuning -#' @importFrom stats setNames runif dnorm pnorm +#' @importFrom stats setNames runif dnorm pnorm quantile #' @useDynLib mlr3mbo c_sms_indicator c_eps_indicator "_PACKAGE" diff --git a/man/mlr_acqfunctions_lfbo.Rd b/man/mlr_acqfunctions_lfbo.Rd index cba397ff..5734c00a 100644 --- a/man/mlr_acqfunctions_lfbo.Rd +++ b/man/mlr_acqfunctions_lfbo.Rd @@ -5,8 +5,8 @@ \alias{AcqFunctionLFBO} \title{Acquisition Function LFBO} \description{ -Likelihood free Bayesian Optimization. -Parameters specifying the weighting type and the gamma quantile of the target distribition have to be set via +Likelihood-Free Bayesian Optimization. +Parameters specifying the weighting type and the gamma quantile of the target distribution have to be set via the \link[paradox:ParamSet]{paradox::ParamSet} of the \link{LearnerRegrLFBO} used within the \link{SurrogateLearner}. } \section{Dictionary}{ @@ -54,8 +54,8 @@ if (requireNamespace("mlr3learners") & \references{ \itemize{ \item Song, Jiaming, Yu, Lantao, Neiswanger, Willie, Ermon, Stefano (2022). -\dQuote{A General Recipe for Likelihood-free Bayesian Optimization.} -\emph{Proceedings of Machine Learning Research}, \bold{162}, 20384--20404. +\dQuote{A General Recipe for Likelihood-Free Bayesian Optimization.} +In Chaudhuri K, Jegelka S, Song L, Szepesvari C, Niu G, Sabato S (eds.), \emph{Proceedings of the 39th International Conference on Machine Learning}, volume 162, 20384--20404. } } \seealso{ diff --git a/man/mlr_learners_regr_lfbo.Rd b/man/mlr_learners_regr_lfbo.Rd index eb1f5521..9f8e0a82 100644 --- a/man/mlr_learners_regr_lfbo.Rd +++ b/man/mlr_learners_regr_lfbo.Rd @@ -15,7 +15,7 @@ to predict whether a target value is smaller than a gamma quantile of the target distribution (assuming minimization). To specify the weighting type, set the value of the \code{lfbo.wt} parameter (\code{"ei"} or \code{"pi"}). -To specify the \code{gamma} quantile of the target distribition set the value of the \code{lfbo.gamma} parameter. +To specify the \code{gamma} quantile of the target distribution set the value of the \code{lfbo.gamma} parameter. } \examples{ \donttest{ @@ -58,8 +58,8 @@ if (requireNamespace("mlr3learners") & \references{ \itemize{ \item Song, Jiaming, Yu, Lantao, Neiswanger, Willie, Ermon, Stefano (2022). -\dQuote{A General Recipe for Likelihood-free Bayesian Optimization.} -\emph{Proceedings of Machine Learning Research}, \bold{162}, 20384--20404. +\dQuote{A General Recipe for Likelihood-Free Bayesian Optimization.} +In Chaudhuri K, Jegelka S, Song L, Szepesvari C, Niu G, Sabato S (eds.), \emph{Proceedings of the 39th International Conference on Machine Learning}, volume 162, 20384--20404. } } \section{Super classes}{ From 3e746e7068a8bfabb6a3b0b262ff33e90b19b120 Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Wed, 15 Feb 2023 19:10:17 +0100 Subject: [PATCH 052/126] .. --- attic/so_config/run_yahpo.R | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/attic/so_config/run_yahpo.R b/attic/so_config/run_yahpo.R index 3d1f5f6c..ca6ff771 100644 --- a/attic/so_config/run_yahpo.R +++ b/attic/so_config/run_yahpo.R @@ -133,8 +133,45 @@ mlr3mbo_wrapper = function(job, data, instance, ...) { optim_instance } +lfbo_wrapper = function(job, data, instance, ...) { + reticulate::use_condaenv("/home/lschnei8/.conda/envs/env", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + optim_instance = make_optim_instance(instance) + + init_design_size = ceiling(0.25 * optim_instance$terminator$param_set$values$n_evals) + init_design = generate_design_random(optim_instance$search_space, n = init_design_size)$data + + optim_instance$eval_batch(init_design) + + random_interleave_iter = 0L + + learner = lrn("regr.lfbo", lrn("classif.ranger")) + learner$param_set$values$keep.inbag = TRUE + + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% + po("imputeoor", multiplier = 3, affect_columns = selector_type(c("integer", "numeric", "character", "factor", "ordered"))) %>>% + po("colapply", applicator = as.factor, affect_columns = selector_type("character")) %>>% + learner)) + + optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10000L), trm("evals", n_evals = 10000L))) + acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20000L)) + acq_optimizer$param_set$values$warmstart = TRUE + acq_optimizer$param_set$values$warmstart_size = "all" + + acq_function = AcqFunctionLFBO$new() + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + + optim_instance +} + + # add algorithms addAlgorithm("mlr3mbo", fun = mlr3mbo_wrapper) +addAlgorithm("lfbo", fun = lfbo_wrapper) # setup scenarios and instances get_nb301_setup = function(budget_factor = 40L) { From 7332fba576b5dc38cc1303e650549ebf4d18eb9d Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Wed, 15 Feb 2023 19:54:33 +0100 Subject: [PATCH 053/126] manually require setting the lfbo direction --- R/AcqFunctionLFBO.R | 53 ++++----------------------- R/LearnerRegrLFBO.R | 41 ++++++++++----------- man/mlr_acqfunctions_lfbo.Rd | 20 +++------- man/mlr_learners_regr_lfbo.Rd | 18 ++++----- tests/testthat/test_AcqFunctionLFBO.R | 9 +---- tests/testthat/test_regr_lfbo.R | 24 +++++++----- 6 files changed, 57 insertions(+), 108 deletions(-) diff --git a/R/AcqFunctionLFBO.R b/R/AcqFunctionLFBO.R index 18d3e9ca..645b2c2f 100644 --- a/R/AcqFunctionLFBO.R +++ b/R/AcqFunctionLFBO.R @@ -8,9 +8,13 @@ #' #' @description #' Likelihood-Free Bayesian Optimization. -#' Parameters specifying the weighting type and the gamma quantile of the target distribution have to be set via +#' Parameters specifying the weighting type and the gamma quantile of the target distribution have to be set in #' the [paradox::ParamSet] of the [LearnerRegrLFBO] used within the [SurrogateLearner]. #' +#' By default, it is assumed that the optimization problem is a minimization problem! +#' If maximization is required, change the value of the direction parameter of the [LearnerRegrLFBO] used within the +#' [SurrogateLearner]. +#' #' @references #' * `r format_bib("song_2022")` #' @@ -37,7 +41,7 @@ #' #' instance$eval_batch(data.table(x = c(-6, -5, 3, 9))) #' -#' learner = LearnerRegrLFBO$new(lrn("classif.ranger")) +#' learner = lrn("regr.lfbo", learner_classif = lrn("classif.ranger"), lfbo.direction = "minimize") #' #' surrogate = srlrn(learner, archive = instance$archive) #' @@ -58,55 +62,12 @@ AcqFunctionLFBO = R6Class("AcqFunctionLFBO", #' @param surrogate (`NULL` | [SurrogateLearner]). initialize = function(surrogate = NULL) { assert_r6(surrogate, "SurrogateLearner", null.ok = TRUE) - - if (!is.null(surrogate)) { - assert_r6(surrogate$model, "LearnerRegrLFBO", null.ok = TRUE) - surrogate$model$surrogate_max_to_min = surrogate_mult_max_to_min(surrogate$archive$codomain, y_cols = surrogate$y_cols) - } - super$initialize("acq_lfbo", surrogate = surrogate, direction = "maximize", label = "Likelihood-Free Bayesian Optimization", man = "mlr3mbo::mlr_acqfunctions_lfbo") } ), - active = list( - #' @field surrogate ([SurrogateLearner])\cr - #' Surrogate learner encapsulating a [LearnerRegrLFBO]. - surrogate = function(rhs) { - if (missing(rhs)) { - private$.surrogate - } else { - assert_r6(rhs, classes = "SurrogateLearner") - assert_r6(rhs$model, "LearnerRegrLFBO") - private$.surrogate = rhs - private$.archive = assert_r6(rhs$archive, classes = "Archive") - codomain = generate_acq_codomain(rhs$archive$codomain, id = self$id, direction = self$direction) - self$surrogate_max_to_min = surrogate_mult_max_to_min(rhs$archive$codomain, y_cols = rhs$y_cols) - domain = rhs$archive$search_space$clone(deep = TRUE) - domain$trafo = NULL - self$codomain = Codomain$new(codomain$params) # lazy initialization requires this - self$domain = domain - - private$.surrogate$model$surrogate_max_to_min = self$surrogate_max_to_min - } - }, - - #' @field surrogate_max_to_min (`-1` | `1`)\cr - #' Multiplicative factor to correct for minimization or maximization of the acquisition function. - #' Changing this value will be propagated to the `$surrogate$model$surrogate_max_to_min` when possible. - surrogate_max_to_min = function(rhs) { - if (missing(rhs)) { - private$.surrogate_max_to_min - } else { - private$.surrogate_max_to_min = assert_subset(rhs, choices = c(-1L, 1L)) - if (!is.null(private$.surrogate)) { - private$.surrogate$model$surrogate_max_to_min = private$.surrogate_max_to_min - } - } - } - ), - private = list( - .fun = function(xdt, ...) { + .fun = function(xdt) { p = self$surrogate$predict(xdt) data.table(acq_lfbo = p$mean) } diff --git a/R/LearnerRegrLFBO.R b/R/LearnerRegrLFBO.R index 4eb78e66..3300bb4d 100644 --- a/R/LearnerRegrLFBO.R +++ b/R/LearnerRegrLFBO.R @@ -11,10 +11,19 @@ #' The new objective is a weighted version of a prediction problem, where the goal is #' to predict whether a target value is smaller than a gamma quantile of the target #' distribution (assuming minimization). +# +#' Note that the optimization direction must be specified explicitly! +#' If minimization is required, set the value of the `lfbo.direction` parameter to `"minimize"`. +#' If maximization is required, set the value of the `lfbo.direction` parameter to `"maximize"`. #' #' To specify the weighting type, set the value of the `lfbo.wt` parameter (`"ei"` or `"pi"`). #' To specify the `gamma` quantile of the target distribution set the value of the `lfbo.gamma` parameter. #' +#' This learner should only be used for Bayesian Optimization in combination with an [AcqFunctionLFBO]. +#' +#' The `$param_set` consists of a [paradox::ParamSetCollection] of the [paradox::ParamSet] of the `$learner_classif` and +#' an `"lfbo"` [paradox::ParamSet] including the parameters `"lfbo.direction"`, `"lfbo.wt"`, and `"lfbo.gamma"`. +#' #' @references #' * `r format_bib("song_2022")` #' @@ -39,7 +48,7 @@ #' objective = objective, #' terminator = trm("evals", n_evals = 5)) #' -#' surrogate = srlrn(lrn("regr.lfbo", lrn("classif.ranger"))) +#' surrogate = srlrn(lrn("regr.lfbo", lrn("classif.ranger"), lfbo.direction = "minimize")) #' #' acq_function = acqf("lfbo") #' @@ -77,7 +86,9 @@ LearnerRegrLFBO = R6Class("LearnerRegrLFBO", assert_true("weights" %in% learner_classif$properties) self$learner_classif = learner_classif - lfbo_param_set = ps(wt = p_fct(levels = c("ei", "pi"), default = "ei", tags = "train"), gamma = p_dbl(lower = 0, upper = 1, default = 1/3, tags = "train")) + lfbo_param_set = ps(direction = p_fct(levels = c("minimize", "maximize"), tags = c("required", "train")), + wt = p_fct(levels = c("ei", "pi"), default = "ei", tags = "train"), + gamma = p_dbl(lower = 0, upper = 1, default = 1/3, tags = "train")) lfbo_param_set$set_id = "lfbo" super$initialize( @@ -109,8 +120,8 @@ LearnerRegrLFBO = R6Class("LearnerRegrLFBO", #' the object in its previous state. train = function(task, row_ids = NULL) { # create df with new data - if (is.null(self$surrogate_max_to_min)) { - stop("$surrogate_max_to_min must be set prior to training.") + if (is.null(self$param_set$values$lfbo.direction)) { + stopf("Learner '%s' requires the specification of the 'lfbo.direction' hyperparameter prior to training.", self$id) } row_ids = assert_row_ids(row_ids, null.ok = TRUE) task_new = private$.to_weighted_task(task, row_ids) @@ -140,21 +151,7 @@ LearnerRegrLFBO = R6Class("LearnerRegrLFBO", } ), - active = list( - #' @field surrogate_max_to_min (`-1` | `1`)\cr - #' Multiplicative factor to correct for minimization or maximization. - surrogate_max_to_min = function(rhs) { - if (missing(rhs)) { - private$.surrogate_max_to_min - } else { - private$.surrogate_max_to_min = assert_subset(rhs, choices = c(-1L, 1L)) - } - } - ), - private = list( - .surrogate_max_to_min = NULL, - .to_weighted_task = function(task, row_ids) { data = do.call("cbind", private$.regr_to_weights( task$data(rows = row_ids, cols = task$feature_names), @@ -165,12 +162,14 @@ LearnerRegrLFBO = R6Class("LearnerRegrLFBO", }, .regr_to_weights = function(X, y) { - # wt and gamma are inferred from the param_set - # surrogate_max_to_min is inferred from the active bindings + # direction, wt and gamma are inferred from the param_set # adapted from https://github.com/lfbo-ml/lfbo + direction = self$param_set$values$lfbo.direction wt = if (is.null(self$param_set$values$lfbo.wt)) "ei" else self$param_set$values$lfbo.wt gamma = if (is.null(self$param_set$values$lfbo.gamma)) 1/3 else self$param_set$values$lfbo.gamma - y = self$surrogate_max_to_min * y + if (direction == "maximize") { + y = -1L * y + } tau = quantile(y, probs = gamma) z = y < tau if (wt == "ei") { diff --git a/man/mlr_acqfunctions_lfbo.Rd b/man/mlr_acqfunctions_lfbo.Rd index 5734c00a..e78f93a8 100644 --- a/man/mlr_acqfunctions_lfbo.Rd +++ b/man/mlr_acqfunctions_lfbo.Rd @@ -6,8 +6,12 @@ \title{Acquisition Function LFBO} \description{ Likelihood-Free Bayesian Optimization. -Parameters specifying the weighting type and the gamma quantile of the target distribution have to be set via +Parameters specifying the weighting type and the gamma quantile of the target distribution have to be set in the \link[paradox:ParamSet]{paradox::ParamSet} of the \link{LearnerRegrLFBO} used within the \link{SurrogateLearner}. + +By default, it is assumed that the optimization problem is a minimization problem! +If maximization is required, change the value of the direction parameter of the \link{LearnerRegrLFBO} used within the +\link{SurrogateLearner}. } \section{Dictionary}{ @@ -40,7 +44,7 @@ if (requireNamespace("mlr3learners") & instance$eval_batch(data.table(x = c(-6, -5, 3, 9))) - learner = LearnerRegrLFBO$new(lrn("classif.ranger")) + learner = lrn("regr.lfbo", learner_classif = lrn("classif.ranger"), lfbo.direction = "minimize") surrogate = srlrn(learner, archive = instance$archive) @@ -74,18 +78,6 @@ Other Acquisition Function: \section{Super classes}{ \code{\link[bbotk:Objective]{bbotk::Objective}} -> \code{\link[mlr3mbo:AcqFunction]{mlr3mbo::AcqFunction}} -> \code{AcqFunctionLFBO} } -\section{Active bindings}{ -\if{html}{\out{
}} -\describe{ -\item{\code{surrogate}}{(\link{SurrogateLearner})\cr -Surrogate learner encapsulating a \link{LearnerRegrLFBO}.} - -\item{\code{surrogate_max_to_min}}{(\code{-1} | \code{1})\cr -Multiplicative factor to correct for minimization or maximization of the acquisition function. -Changing this value will be propagated to the \verb{$surrogate$model$surrogate_max_to_min} when possible.} -} -\if{html}{\out{
}} -} \section{Methods}{ \subsection{Public methods}{ \itemize{ diff --git a/man/mlr_learners_regr_lfbo.Rd b/man/mlr_learners_regr_lfbo.Rd index 9f8e0a82..e261cb66 100644 --- a/man/mlr_learners_regr_lfbo.Rd +++ b/man/mlr_learners_regr_lfbo.Rd @@ -13,9 +13,17 @@ The regression data is internally converted to a weighted classification task. The new objective is a weighted version of a prediction problem, where the goal is to predict whether a target value is smaller than a gamma quantile of the target distribution (assuming minimization). +Note that the optimization direction must be specified explicitly! +If minimization is required, set the value of the \code{lfbo.direction} parameter to \code{"minimize"}. +If maximization is required, set the value of the \code{lfbo.direction} parameter to \code{"maximize"}. To specify the weighting type, set the value of the \code{lfbo.wt} parameter (\code{"ei"} or \code{"pi"}). To specify the \code{gamma} quantile of the target distribution set the value of the \code{lfbo.gamma} parameter. + +This learner should only be used for Bayesian Optimization in combination with an \link{AcqFunctionLFBO}. + +The \verb{$param_set} consists of a \link[paradox:ParamSetCollection]{paradox::ParamSetCollection} of the \link[paradox:ParamSet]{paradox::ParamSet} of the \verb{$learner_classif} and +an \code{"lfbo"} \link[paradox:ParamSet]{paradox::ParamSet} including the parameters \code{"lfbo.direction"}, \code{"lfbo.wt"}, and \code{"lfbo.gamma"}. } \examples{ \donttest{ @@ -37,7 +45,7 @@ if (requireNamespace("mlr3learners") & objective = objective, terminator = trm("evals", n_evals = 5)) - surrogate = srlrn(lrn("regr.lfbo", lrn("classif.ranger"))) + surrogate = srlrn(lrn("regr.lfbo", lrn("classif.ranger"), lfbo.direction = "minimize")) acq_function = acqf("lfbo") @@ -73,14 +81,6 @@ Classification learner to be used for LFBO.} } \if{html}{\out{}} } -\section{Active bindings}{ -\if{html}{\out{
}} -\describe{ -\item{\code{surrogate_max_to_min}}{(\code{-1} | \code{1})\cr -Multiplicative factor to correct for minimization or maximization.} -} -\if{html}{\out{
}} -} \section{Methods}{ \subsection{Public methods}{ \itemize{ diff --git a/tests/testthat/test_AcqFunctionLFBO.R b/tests/testthat/test_AcqFunctionLFBO.R index 976646bc..e9572097 100644 --- a/tests/testthat/test_AcqFunctionLFBO.R +++ b/tests/testthat/test_AcqFunctionLFBO.R @@ -1,6 +1,6 @@ test_that("AcqFunctionLFBO works", { inst = MAKE_INST_1D() - surrogate = SurrogateLearner$new(lrn("regr.lfbo", learner_classif = lrn("classif.rpart")), archive = inst$archive) + surrogate = SurrogateLearner$new(lrn("regr.lfbo", learner_classif = lrn("classif.rpart"), lfbo.direction = "minimize"), archive = inst$archive) acqf = AcqFunctionLFBO$new(surrogate = surrogate) expect_acqfunction(acqf) @@ -20,12 +20,5 @@ test_that("AcqFunctionLFBO works", { res = acqf$eval_dt(xdt) expect_data_table(res, ncols = 1L, nrows = 5L, any.missing = FALSE) expect_named(res, acqf$id) - - expect_true(acqf$surrogate_max_to_min == 1) - expect_true(acqf$surrogate$model$surrogate_max_to_min == 1) - - acqf$surrogate_max_to_min = -1 - expect_true(acqf$surrogate_max_to_min == -1) - expect_true(acqf$surrogate$model$surrogate_max_to_min == -1) }) diff --git a/tests/testthat/test_regr_lfbo.R b/tests/testthat/test_regr_lfbo.R index 8ebf5faf..301a009c 100644 --- a/tests/testthat/test_regr_lfbo.R +++ b/tests/testthat/test_regr_lfbo.R @@ -8,29 +8,30 @@ test_that("LearnerRegrLFBO can be constructed and trained and setting hyperparam surrogate = SurrogateLearner$new(learner) expect_class(surrogate, "SurrogateLearner") + lfbo_param_set = ps(direction = p_fct(levels = c("minimize", "maximize"), tags = c("required", "train")), + wt = p_fct(levels = c("ei", "pi"), default = "ei", tags = "train"), + gamma = p_dbl(lower = 0, upper = 1, default = 1/3, tags = "train")) + lfbo_param_set$set_id = "lfbo" + param_set = ParamSetCollection$new(list(lfbo_param_set, learner_classif$param_set)) + expect_equal(param_set, learner$param_set) + task = tsk("mtcars") - expect_error(learner$train(task), regexp = "surrogate_max_to_min must be set") - expect_error({learner$surrogate_max_to_min = 10}, regexp = "Must be a subset") - learner$surrogate_max_to_min = 1 + expect_error(learner$train(task), "requires the specification of the 'lfbo.direction' hyperparameter") + expect_true(is.null(learner$state)) + learner$param_set$values$lfbo.direction = "minimize" learner$train(task) expect_true(!is.null(learner$state)) prediction = learner$predict(task) expect_class(prediction, "PredictionRegr") # Note: Truth value does not make sense! expect_true(cor(prediction$truth, prediction$response) < - 0.5) - learner$surrogate_max_to_min = -1 + learner$param_set$values$lfbo.direction = "maximize" learner$train(task) expect_true(!is.null(learner$state)) prediction = learner$predict(task) expect_class(prediction, "PredictionRegr") # Note: Truth value does not make sense! expect_true(cor(prediction$truth, prediction$response) > 0.5) - lfbo_param_set = ps(wt = p_fct(levels = c("ei", "pi"), default = "ei", tags = "train"), gamma = p_dbl(lower = 0, upper = 1, default = 1/3, tags = "train")) - lfbo_param_set$set_id = "lfbo" - param_set = ParamSetCollection$new(list(lfbo_param_set, learner_classif$param_set)) - expect_equal(param_set, learner$param_set) - - learner$param_set$values$lfbo.wt = "pi" learner$param_set$values$lfbo.gamma = 0.5 learner$param_set$values$maxdepth = 20L @@ -47,6 +48,7 @@ test_that("Likelihood Free Bayesian Optimization", { with_seed(2906, { instance = MAKE_INST_1D(terminator = trm("evals", n_evals = 10L)) surrogate = SurrogateLearner$new(lrn("regr.lfbo", lrn("classif.ranger", predict_type = "prob", num.trees = 10L))) + surrogate$model$param_set$values$lfbo.direction = "minimize" acq_function = acqf("lfbo") acq_optimizer = acqo(optimizer = opt("random_search", batch_size = 100L), terminator = trm("evals", n_evals = 100L)) optimizer = opt("mbo", @@ -65,6 +67,8 @@ test_that("Likelihood Free Bayesian Optimization", { codomain = ParamSet$new(list(ParamDbl$new("y", tags = "maximize"))) objective = ObjectiveRFun$new(fun = fun, domain = PS_1D, codomain = codomain, properties = "single-crit") instance = MAKE_INST(objective = objective, search_space = PS_1D, terminator = trm("evals", n_evals = 10L)) + surrogate$model$param_set$values$lfbo.direction = "maximize" + expect_true(optimizer$surrogate$model$param_set$values$lfbo.direction == "maximize") optimizer$optimize(instance) expect_true(abs(instance$result$y) < 0.1) }) From 967ab126d125780f848ffef2626ad147d6947bcf Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Wed, 15 Feb 2023 22:58:38 +0100 Subject: [PATCH 054/126] .. --- R/SurrogateLearner.R | 6 +++--- attic/so_config/run_yahpo.R | 2 +- man/mlr_acqfunctions_lfbo.Rd | 1 + man/mlr_acqfunctions_ttei.Rd | 1 + 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/R/SurrogateLearner.R b/R/SurrogateLearner.R index 5b4fd56e..e4f9b217 100644 --- a/R/SurrogateLearner.R +++ b/R/SurrogateLearner.R @@ -78,9 +78,9 @@ SurrogateLearner = R6Class("SurrogateLearner", #' @template param_y_col_surrogate initialize = function(learner, archive = NULL, x_cols = NULL, y_col = NULL) { assert_learner(learner) - if (learner$predict_type != "se" && "se" %in% learner$predict_types) { - learner$predict_type = "se" - } + #if (learner$predict_type != "se" && "se" %in% learner$predict_types) { + # learner$predict_type = "se" + #} assert_r6(archive, classes = "Archive", null.ok = TRUE) diff --git a/attic/so_config/run_yahpo.R b/attic/so_config/run_yahpo.R index ca6ff771..74bea41b 100644 --- a/attic/so_config/run_yahpo.R +++ b/attic/so_config/run_yahpo.R @@ -149,7 +149,7 @@ lfbo_wrapper = function(job, data, instance, ...) { random_interleave_iter = 0L - learner = lrn("regr.lfbo", lrn("classif.ranger")) + learner = lrn("regr.lfbo", lrn("classif.ranger"), lfbo.direction = optim_instance$objective$codomain$tags[[optim_instance$archive$cols_y]]) learner$param_set$values$keep.inbag = TRUE surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% diff --git a/man/mlr_acqfunctions_lfbo.Rd b/man/mlr_acqfunctions_lfbo.Rd index e78f93a8..1ccf5443 100644 --- a/man/mlr_acqfunctions_lfbo.Rd +++ b/man/mlr_acqfunctions_lfbo.Rd @@ -72,6 +72,7 @@ Other Acquisition Function: \code{\link{mlr_acqfunctions_mean}}, \code{\link{mlr_acqfunctions_pi}}, \code{\link{mlr_acqfunctions_smsego}}, +\code{\link{mlr_acqfunctions_ttei}}, \code{\link{mlr_acqfunctions}} } \concept{Acquisition Function} diff --git a/man/mlr_acqfunctions_ttei.Rd b/man/mlr_acqfunctions_ttei.Rd index fa08eeb7..9163da59 100644 --- a/man/mlr_acqfunctions_ttei.Rd +++ b/man/mlr_acqfunctions_ttei.Rd @@ -75,6 +75,7 @@ Other Acquisition Function: \code{\link{mlr_acqfunctions_cb}}, \code{\link{mlr_acqfunctions_eips}}, \code{\link{mlr_acqfunctions_ei}}, +\code{\link{mlr_acqfunctions_lfbo}}, \code{\link{mlr_acqfunctions_mean}}, \code{\link{mlr_acqfunctions_pi}}, \code{\link{mlr_acqfunctions_smsego}}, From 4ac01d4981594cbac2f7709f05eee47142dd869a Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Sat, 18 Feb 2023 20:23:15 +0100 Subject: [PATCH 055/126] .. --- attic/so_config/analyze_yahpo.R | 2 +- attic/so_config/run_yahpo.R | 40 +++++++++++++++++++++++++++++---- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/attic/so_config/analyze_yahpo.R b/attic/so_config/analyze_yahpo.R index 235f8a25..994957d0 100644 --- a/attic/so_config/analyze_yahpo.R +++ b/attic/so_config/analyze_yahpo.R @@ -3,7 +3,7 @@ library(ggplot2) library(pammtools) library(mlr3misc) -dat = rbind(readRDS("results_yahpo.rds"), readRDS("results_yahpo_mlr3mbo.rds")) +dat = rbind(readRDS("results_yahpo.rds"), readRDS("results_yahpo_lfbo.rds"), readRDS("results_yahpo_mlr3mbo.rds"), readRDS("results_yahpo_lfbox.rds")) dat[, cumbudget := cumsum(budget), by = .(method, scenario, instance, repl)] dat[, cumbudget_scaled := cumbudget / max(cumbudget), by = .(method, scenario, instance, repl)] dat[, normalized_regret := (target - min(target)) / (max(target) - min(target)), by = .(scenario, instance)] diff --git a/attic/so_config/run_yahpo.R b/attic/so_config/run_yahpo.R index 74bea41b..9db3bcb8 100644 --- a/attic/so_config/run_yahpo.R +++ b/attic/so_config/run_yahpo.R @@ -27,7 +27,7 @@ for (sf in source_files) { source(sf) } -reg = makeExperimentRegistry(file.dir = "/gscratch/lschnei8/registry_mlr3mbo_so_config", packages = packages, source = source_files) +reg = makeExperimentRegistry(file.dir = "/gscratch/lschnei8/registry_mlr3mbo_lfbox", packages = packages, source = source_files) #reg = makeExperimentRegistry(file.dir = NA, conf.file = NA, packages = packages, source = source_files) # interactive session saveRegistry(reg) @@ -168,10 +168,42 @@ lfbo_wrapper = function(job, data, instance, ...) { optim_instance } +lfbox_wrapper = function(job, data, instance, ...) { + reticulate::use_condaenv("/home/lschnei8/.conda/envs/env", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + optim_instance = make_optim_instance(instance) + + init_design_size = ceiling(0.25 * optim_instance$terminator$param_set$values$n_evals) + init_design = generate_design_random(optim_instance$search_space, n = init_design_size)$data + + optim_instance$eval_batch(init_design) + + random_interleave_iter = 0L + + learner = lrn("regr.lfbo", lrn("classif.ranger", num.trees = 1000, min.node.size = 1), lfbo.direction = optim_instance$objective$codomain$tags[[optim_instance$archive$cols_y]]) + learner$param_set$values$keep.inbag = TRUE + + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% + po("imputeoor", multiplier = 3, affect_columns = selector_type(c("integer", "numeric", "character", "factor", "ordered"))) %>>% + po("colapply", applicator = as.factor, affect_columns = selector_type("character")) %>>% + learner)) + + acq_optimizer = AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 1000L)) + + acq_function = AcqFunctionLFBO$new() + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + + optim_instance +} # add algorithms addAlgorithm("mlr3mbo", fun = mlr3mbo_wrapper) addAlgorithm("lfbo", fun = lfbo_wrapper) +addAlgorithm("lfbox", fun = lfbox_wrapper) # setup scenarios and instances get_nb301_setup = function(budget_factor = 40L) { @@ -243,7 +275,7 @@ prob_designs = unlist(prob_designs, recursive = FALSE, use.names = FALSE) names(prob_designs) = nn # add jobs for optimizers -optimizers = data.table(algorithm = "mlr3mbo") +optimizers = data.table(algorithm = "lfbox") for (i in seq_len(nrow(optimizers))) { algo_designs = setNames(list(optimizers[i, ]), nm = optimizers[i, ]$algorithm) @@ -257,7 +289,7 @@ for (i in seq_len(nrow(optimizers))) { } jobs = getJobTable() -resources.default = list(walltime = 3600 * 8L, memory = 2048L, ntasks = 1L, ncpus = 1L, nodes = 1L, clusters = "beartooth", max.concurrent.jobs = 9999L) +resources.default = list(walltime = 3600 * 1L, memory = 2048L, ntasks = 1L, ncpus = 1L, nodes = 1L, clusters = "beartooth", max.concurrent.jobs = 9999L) submitJobs(jobs, resources = resources.default) done = findDone() @@ -280,5 +312,5 @@ results = reduceResultsList(done, function(x, job) { tmp }) results = rbindlist(results, fill = TRUE) -saveRDS(results, "/gscratch/lschnei8/results_yahpo_mlr3mbo.rds") +saveRDS(results, "/gscratch/lschnei8/results_yahpo_lfbox.rds") From 2d54d0b26b47724f2a48400cd677b407be1d3a08 Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Wed, 22 Feb 2023 16:00:02 +0100 Subject: [PATCH 056/126] .. --- attic/so_config/OptimizerCoordinateDescent.R | 21 ++++++++++++++++++-- attic/so_config/cd_example.R | 2 +- attic/so_config/coordinate_descent.R | 15 ++++---------- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/attic/so_config/OptimizerCoordinateDescent.R b/attic/so_config/OptimizerCoordinateDescent.R index fd285726..7bf935b3 100644 --- a/attic/so_config/OptimizerCoordinateDescent.R +++ b/attic/so_config/OptimizerCoordinateDescent.R @@ -29,7 +29,6 @@ OptimizerCoordinateDescent = R6Class("OptimizerCoordinateDescent", inherit = bbo xdt = generate_design_random(inst$search_space, n = 1L)$data inst$eval_batch(xdt) } - incumbent = inst$archive$best()[, inst$archive$cols_x, with = FALSE] # check if .gen is already present, if yes continue from there gen = if (inst$archive$n_evals > 0L & ".gen" %in% colnames(inst$archive$data)) { max(inst$archive$data[[".gen"]], na.rm = TRUE) @@ -37,6 +36,14 @@ OptimizerCoordinateDescent = R6Class("OptimizerCoordinateDescent", inherit = bbo 0L } set(inst$archive$data, j = ".gen", value = gen) # 0 for first batch + + y_col = inst$archive$cols_y + y_col_orig = paste0(y_col, "_orig") + + inst$archive$data[.gen == gen, (y_col_orig) := get(y_col)] + + incumbent = inst$archive$best()[, inst$archive$cols_x, with = FALSE] + repeat { gen = gen + 1L for (param_id in shuffle(inst$search_space$ids())) { @@ -62,10 +69,20 @@ OptimizerCoordinateDescent = R6Class("OptimizerCoordinateDescent", inherit = bbo set(xdt, j = "incumbent", value = list(inst$archive$best()[, c(inst$archive$cols_x, inst$archive$cols_y), with = FALSE])) inst$eval_batch(xdt) incumbent = inst$archive$best()[, inst$archive$cols_x, with = FALSE] - saveRDS(inst, file = self$param_set$values$rds_name) } } + + # after each gen, update target evaluations of incumbents that have been evaluated multiple times by their mean + tmp = copy(inst$archive$data) + hashes = as.numeric(as.factor(map_chr(seq_len(nrow(tmp)), function(i) paste0(tmp[i, inst$archive$cols_x, with = FALSE], collapse = "_")))) + tmp[, hash := hashes] + tmp[.gen == gen, (y_col_orig) := get(y_col)] + tmp[, n_incumbent_repls := .N, by = .(hash)] + tmp[, (y_col) := mean(get(y_col_orig)), by = .(hash)] + inst$archive$data = tmp + incumbent = inst$archive$best()[, inst$archive$cols_x, with = FALSE] + if (gen >= self$param_set$values$max_gen) { break } diff --git a/attic/so_config/cd_example.R b/attic/so_config/cd_example.R index 51f29a65..6917bea5 100644 --- a/attic/so_config/cd_example.R +++ b/attic/so_config/cd_example.R @@ -21,7 +21,7 @@ objective = ObjectiveRFunDt$new( x6 = xdt$x6 x6[is.na(x6)] = 0 y = x3 * xdt$x1 - x6 - data.table(y = y) + data.table(y = y + rnorm(nrow(xdt))) }, domain = domain, codomain = ps(y = p_dbl(tags = "minimize")) diff --git a/attic/so_config/coordinate_descent.R b/attic/so_config/coordinate_descent.R index 5b55fd83..70519b40 100644 --- a/attic/so_config/coordinate_descent.R +++ b/attic/so_config/coordinate_descent.R @@ -207,7 +207,7 @@ evaluate = function(xdt, instance) { objective = ObjectiveRFunDt$new( fun = function(xdt) { - n_repl = 3L + n_repl = 5L xdt[, id := seq_len(.N)] xdt_tmp = map_dtr(seq_len(nrow(instances)), function(i) copy(xdt)) setorderv(xdt_tmp, col = "id") @@ -266,20 +266,13 @@ cd_instance = OptimInstanceSingleCrit$new( ) optimizer = OptimizerCoordinateDescent$new() -optimizer$param_set$values$max_gen = 3L +optimizer$param_set$values$max_gen = 5L init = data.table(loop_function = "ego", init = "random", init_size_fraction = "0.25", random_interleave = FALSE, random_interleave_iter = NA_character_, rf_type = "standard", acqf = "EI", acqf_ei_log = NA, lambda = NA_character_, acqopt = "RS_1000") -#options(future.cache.path = "/gscratch/lschnei8/coordinate_descent/future") options(future.cache.path = "/home/lschnei8/mlr3mbo_config/future") set.seed(2906, kind = "L'Ecuyer-CMRG") -# currently we evaluate at most 4 * 32 * 3 jobs in parallel so 400 workers is enough -plan("batchtools_slurm", template = "slurm_wyoming_cd.tmpl", resources = list(walltime = 3600L * 9L, ncpus = 1L, memory = 4000L), workers = 400L) +# currently we evaluate at most 4 * 32 * 5 jobs in parallel so 650 workers is enough +plan("batchtools_slurm", template = "slurm_wyoming_cd.tmpl", resources = list(walltime = 3600L * 9L, ncpus = 1L, memory = 4000L), workers = 650L) cd_instance$eval_batch(init) optimizer$optimize(cd_instance) -# NOTE: -# cd_instance_gen1.rds first gen through -# then load, subset data to gen1 -# set.seed(2409, kind = "L'Ecuyer-CMRG") -# and rerun - From dda812a39503a8833ad6703850eae294649de02b Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Thu, 2 Mar 2023 10:40:54 +0100 Subject: [PATCH 057/126] .. --- attic/so_config/LearnerRegrGowerNNEnsemble.R | 111 +++++++++++++++++++ attic/so_config/OptimizerCoordinateDescent.R | 8 +- attic/so_config/analyze_yahpo.R | 2 +- attic/so_config/run_yahpo.R | 84 +++++++++++++- 4 files changed, 195 insertions(+), 10 deletions(-) create mode 100644 attic/so_config/LearnerRegrGowerNNEnsemble.R diff --git a/attic/so_config/LearnerRegrGowerNNEnsemble.R b/attic/so_config/LearnerRegrGowerNNEnsemble.R new file mode 100644 index 00000000..a7d9a81d --- /dev/null +++ b/attic/so_config/LearnerRegrGowerNNEnsemble.R @@ -0,0 +1,111 @@ +#' @title Gower Distance Nearest Neighbor Regression Ensemble +#' +#' @name mlr_learners_regr.gower_nn_ensemble +#' +#' @description +#' FIXME: +#' +#' @templateVar id regr.gower_nn_ensemble +#' @template learner +#' +#' @export +#' @template seealso_learner +#' @template example +LearnerRegrGowerNNEnsemble = R6Class("LearnerRegrGowerNNEnsemble", + inherit = LearnerRegr, + + public = list( + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + initialize = function() { + ps = ps( + ks = p_uty(tags = "train"), + sample_fraction = p_dbl(lower = 0, upper = 1, tags = "train"), + replace = p_lgl(tags = "train"), + nthread = p_int(lower = 1, tags = "predict") + ) + + ps$values$ks = c(1L, 3L, 5L, 7L, 10L) + ps$values$sample_fraction = 1 - exp(-1) + ps$values$replace = TRUE + ps$values$nthread = 1L + + super$initialize( + id = "regr.gower_nn_ensemble", + param_set = ps, + predict_types = c("response", "se"), + feature_types = c("logical", "integer", "numeric", "character", "factor", "ordered"), + properties = c("missings"), + packages = c("mlr3mbo", "gower"), + man = "mlr3mbo::mlr_learners_regr.gower_nn_ensemble" + ) + } + ), + + private = list( + .train = function(task) { + + pv = self$param_set$get_values(tags = "train") + if (any(pv$ks >= task$nrow)) { + index = which(pv$ks >= task$nrow) + stopf("Parameter k = %i must be smaller than the number of observations n = %i", pv$k[index], task$nrow) + } + + list( + target_name = task$target_names, + feature_names = task$feature_names, + data = task$data(), + pv = pv + ) + + }, + + .predict = function(task) { + model = self$model + newdata = mlr3learners:::ordered_features(task, self) + pv = insert_named(model$pv, self$param_set$get_values(tags = "predict")) + + data = self$model$data + X = data[, self$model$feature_names, with = FALSE] + y = data[, self$model$target_name, with = FALSE] + + # FIXME: what about ordered here + for (feature in colnames(X)) { + if ("factor" %in% class(X[[feature]])) { + factor_levels = union(levels(X[[feature]]), levels(newdata[[feature]])) + } + levels(X[[feature]]) = factor_levels + levels(newdata[[feature]]) = factor_levels + } + + n = max(ceiling(pv$sample_fraction * nrow(X)), nrow(X)) + + results = map_dtr(pv$ks, function(k) { + ids = sample(seq_len(nrow(X)), size = n, replace = pv$replace) + top_n = invoke(gower::gower_topn, x = newdata, y = X[ids, ], n = k, nthread = pv$nthread)$index + tmp = map_dtc(seq_len(k), function(i) { + setNames(y[ids, ][top_n[i, ], ], nm = paste0(self$model$target_name, "_", i)) + }) + mu = rowMeans(tmp) + sigma_2 = apply(tmp, MARGIN = 1L, FUN = var) + sigma_2[is.na(sigma_2)] = 0 + result = data.table(mu = mu, sigma_2 = sigma_2) + result[, k := k] + result[, ID := seq_len(.N)] + result + }) + + response = results[, .(response = mean(mu)), by = .(ID)] + results[, first_term := 0.001 + sigma_2 + mu ^ 2] + variance = results[, .(variance = mean(first_term)), by = .(ID)] + setorderv(response, col = "ID") + setorderv(variance, col = "ID") + stopifnot(all(response[["ID"]] == variance[["ID"]])) + variance$variance = variance$variance - response$response ^ 2 + + list(response = response$response, se = sqrt(variance$variance)) + } + ) +) + diff --git a/attic/so_config/OptimizerCoordinateDescent.R b/attic/so_config/OptimizerCoordinateDescent.R index 7bf935b3..5670d1b9 100644 --- a/attic/so_config/OptimizerCoordinateDescent.R +++ b/attic/so_config/OptimizerCoordinateDescent.R @@ -30,12 +30,12 @@ OptimizerCoordinateDescent = R6Class("OptimizerCoordinateDescent", inherit = bbo inst$eval_batch(xdt) } # check if .gen is already present, if yes continue from there - gen = if (inst$archive$n_evals > 0L & ".gen" %in% colnames(inst$archive$data)) { - max(inst$archive$data[[".gen"]], na.rm = TRUE) + if (inst$archive$n_evals > 0L & ".gen" %in% colnames(inst$archive$data)) { + gen = max(inst$archive$data[[".gen"]], na.rm = TRUE) } else { - 0L + gen = 0L + set(inst$archive$data, j = ".gen", value = gen) # 0 for first batch } - set(inst$archive$data, j = ".gen", value = gen) # 0 for first batch y_col = inst$archive$cols_y y_col_orig = paste0(y_col, "_orig") diff --git a/attic/so_config/analyze_yahpo.R b/attic/so_config/analyze_yahpo.R index 994957d0..5b8db566 100644 --- a/attic/so_config/analyze_yahpo.R +++ b/attic/so_config/analyze_yahpo.R @@ -3,7 +3,7 @@ library(ggplot2) library(pammtools) library(mlr3misc) -dat = rbind(readRDS("results_yahpo.rds"), readRDS("results_yahpo_lfbo.rds"), readRDS("results_yahpo_mlr3mbo.rds"), readRDS("results_yahpo_lfbox.rds")) +dat = rbind(readRDS("results_yahpo.rds"), readRDS("results_yahpo_mbox.rds")) dat[, cumbudget := cumsum(budget), by = .(method, scenario, instance, repl)] dat[, cumbudget_scaled := cumbudget / max(cumbudget), by = .(method, scenario, instance, repl)] dat[, normalized_regret := (target - min(target)) / (max(target) - min(target)), by = .(scenario, instance)] diff --git a/attic/so_config/run_yahpo.R b/attic/so_config/run_yahpo.R index 9db3bcb8..3dd60a4d 100644 --- a/attic/so_config/run_yahpo.R +++ b/attic/so_config/run_yahpo.R @@ -22,7 +22,7 @@ packages = c("data.table", "mlr3", "mlr3learners", "mlr3pipelines", "mlr3misc", root = here::here() experiments_dir = file.path(root) -source_files = map_chr(c("helpers.R", "AcqFunctionLogEI.R", "LearnerRegrRangerCustom.R", "OptimizerChain.R"), function(x) file.path(experiments_dir, x)) +source_files = map_chr(c("helpers.R", "AcqFunctionLogEI.R", "LearnerRegrRangerCustom.R", "LearnerRegrGowerNNEnsemble.R", "OptimizerChain.R"), function(x) file.path(experiments_dir, x)) for (sf in source_files) { source(sf) } @@ -40,7 +40,7 @@ mlr3mbo_wrapper = function(job, data, instance, ...) { optim_instance = make_optim_instance(instance) - xdt = data.table(loop_function = "ego_log", init = "random", init_size_fraction = "0.25", random_interleave = TRUE, random_interleave_iter = "10", rf_type = "smaclike_no_boot", acqf = "CB", acqf_ei_log = NA, lambda = "1", acqopt = "LS") + xdt = data.table(loop_function = "ego_log", init = "sobol", init_size_fraction = "0.25", random_interleave = FALSE, random_interleave_iter = "NA_chr", rf_type = "smaclike_boot", acqf = "EI", acqf_ei_log = FALSE, lambda = "NA_chr", acqopt = "LS") init_design_size = ceiling(as.numeric(xdt$init_size_fraction) * optim_instance$terminator$param_set$values$n_evals) init_design = if (xdt$init == "random") { @@ -133,6 +133,79 @@ mlr3mbo_wrapper = function(job, data, instance, ...) { optim_instance } +mlr3mbox_wrapper = function(job, data, instance, ...) { + reticulate::use_condaenv("/home/lschnei8/.conda/envs/env", required = TRUE) + library(yahpogym) + logger = lgr::get_logger("bbotk") + logger$set_threshold("warn") + future::plan("sequential") + + optim_instance = make_optim_instance(instance) + + xdt = data.table(loop_function = "ego_log", init = "sobol", init_size_fraction = "0.25", random_interleave = FALSE, random_interleave_iter = "NA_chr", acqf = "EI", acqf_ei_log = TRUE, lambda = "NA_chr", acqopt = "LS") + + init_design_size = ceiling(as.numeric(xdt$init_size_fraction) * optim_instance$terminator$param_set$values$n_evals) + init_design = if (xdt$init == "random") { + generate_design_random(optim_instance$search_space, n = init_design_size)$data + } else if (xdt$init == "lhs") { + generate_design_lhs(optim_instance$search_space, n = init_design_size)$data + } else if (xdt$init == "sobol") { + generate_design_sobol(optim_instance$search_space, n = init_design_size)$data + } + + optim_instance$eval_batch(init_design) + + random_interleave_iter = if(xdt$random_interleave) as.numeric(xdt$random_interleave_iter) else 0L + + learner = LearnerRegrGowerNNEnsemble$new() + learner$predict_type = "se" + learner$param_set$values$ks = rep(1, 10) + + surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% + po("imputeoor", multiplier = 10, affect_columns = selector_type(c("integer", "numeric", "character", "factor", "ordered"))) %>>% + po("colapply", applicator = as.factor, affect_columns = selector_type("character")) %>>% + learner)) + + acq_optimizer = if (xdt$acqopt == "RS_1000") { + AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 1000L)) + } else if (xdt$acqopt == "RS") { + AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 20000L)) + } else if (xdt$acqopt == "FS") { + n_repeats = 2L + maxit = 9L + batch_size = ceiling((20000L / n_repeats) / (1 + maxit)) # 1000L + AcqOptimizer$new(opt("focus_search", n_points = batch_size, maxit = maxit), terminator = trm("evals", n_evals = 20000L)) + } else if (xdt$acqopt == "LS") { + optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10000L), trm("evals", n_evals = 10000L))) + acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20000L)) + acq_optimizer$param_set$values$warmstart = TRUE + acq_optimizer$param_set$values$warmstart_size = "all" + acq_optimizer + } + + acq_function = if (xdt$acqf == "EI") { + if (isTRUE(xdt$acqf_ei_log)) { + AcqFunctionLogEI$new() + } else { + AcqFunctionEI$new() + } + } else if (xdt$acqf == "CB") { + AcqFunctionCB$new(lambda = as.numeric(xdt$lambda)) + } else if (xdt$acqf == "PI") { + AcqFunctionPI$new() + } else if (xdt$acqf == "Mean") { + AcqFunctionMean$new() + } + + if (xdt$loop_function == "ego") { + bayesopt_ego(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + } else if (xdt$loop_function == "ego_log") { + bayesopt_ego_log(optim_instance, surrogate = surrogate, acq_function = acq_function, acq_optimizer = acq_optimizer, random_interleave_iter = random_interleave_iter) + } + + optim_instance +} + lfbo_wrapper = function(job, data, instance, ...) { reticulate::use_condaenv("/home/lschnei8/.conda/envs/env", required = TRUE) library(yahpogym) @@ -202,6 +275,7 @@ lfbox_wrapper = function(job, data, instance, ...) { # add algorithms addAlgorithm("mlr3mbo", fun = mlr3mbo_wrapper) +addAlgorithm("mlr3mbox", fun = mlr3mbox_wrapper) addAlgorithm("lfbo", fun = lfbo_wrapper) addAlgorithm("lfbox", fun = lfbox_wrapper) @@ -275,7 +349,7 @@ prob_designs = unlist(prob_designs, recursive = FALSE, use.names = FALSE) names(prob_designs) = nn # add jobs for optimizers -optimizers = data.table(algorithm = "lfbox") +optimizers = data.table(algorithm = "mlr3mbox") for (i in seq_len(nrow(optimizers))) { algo_designs = setNames(list(optimizers[i, ]), nm = optimizers[i, ]$algorithm) @@ -289,7 +363,7 @@ for (i in seq_len(nrow(optimizers))) { } jobs = getJobTable() -resources.default = list(walltime = 3600 * 1L, memory = 2048L, ntasks = 1L, ncpus = 1L, nodes = 1L, clusters = "beartooth", max.concurrent.jobs = 9999L) +resources.default = list(walltime = 3600 * 9L, memory = 2048L, ntasks = 1L, ncpus = 1L, nodes = 1L, clusters = "beartooth", max.concurrent.jobs = 9999L) submitJobs(jobs, resources = resources.default) done = findDone() @@ -312,5 +386,5 @@ results = reduceResultsList(done, function(x, job) { tmp }) results = rbindlist(results, fill = TRUE) -saveRDS(results, "/gscratch/lschnei8/results_yahpo_lfbox.rds") +saveRDS(results, "/gscratch/lschnei8/results_yahpo_mbox.rds") From 11c4e5711dfa82eaa705cdd8fee2073567fcd8e0 Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Thu, 2 Mar 2023 11:03:12 +0100 Subject: [PATCH 058/126] .. --- attic/so_config/LearnerRegrGowerNNEnsemble.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/attic/so_config/LearnerRegrGowerNNEnsemble.R b/attic/so_config/LearnerRegrGowerNNEnsemble.R index a7d9a81d..ea101ae1 100644 --- a/attic/so_config/LearnerRegrGowerNNEnsemble.R +++ b/attic/so_config/LearnerRegrGowerNNEnsemble.R @@ -74,9 +74,9 @@ LearnerRegrGowerNNEnsemble = R6Class("LearnerRegrGowerNNEnsemble", for (feature in colnames(X)) { if ("factor" %in% class(X[[feature]])) { factor_levels = union(levels(X[[feature]]), levels(newdata[[feature]])) - } levels(X[[feature]]) = factor_levels levels(newdata[[feature]]) = factor_levels + } } n = max(ceiling(pv$sample_fraction * nrow(X)), nrow(X)) From 17700ac401455a72276c4e1da172cef6f54aab56 Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Thu, 2 Mar 2023 12:42:14 +0100 Subject: [PATCH 059/126] .. --- attic/so_config/LearnerRegrGowerNNEnsemble.R | 22 +++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/attic/so_config/LearnerRegrGowerNNEnsemble.R b/attic/so_config/LearnerRegrGowerNNEnsemble.R index ea101ae1..f229cb73 100644 --- a/attic/so_config/LearnerRegrGowerNNEnsemble.R +++ b/attic/so_config/LearnerRegrGowerNNEnsemble.R @@ -21,13 +21,14 @@ LearnerRegrGowerNNEnsemble = R6Class("LearnerRegrGowerNNEnsemble", initialize = function() { ps = ps( ks = p_uty(tags = "train"), - sample_fraction = p_dbl(lower = 0, upper = 1, tags = "train"), + sample.fraction = p_dbl(lower = 0, upper = 1, tags = "train"), replace = p_lgl(tags = "train"), + mtry.ratio = p_dbl(lower = 0, upper = 1, tags = "train"), nthread = p_int(lower = 1, tags = "predict") ) ps$values$ks = c(1L, 3L, 5L, 7L, 10L) - ps$values$sample_fraction = 1 - exp(-1) + ps$values$sample.fraction = 1 ps$values$replace = TRUE ps$values$nthread = 1L @@ -36,7 +37,7 @@ LearnerRegrGowerNNEnsemble = R6Class("LearnerRegrGowerNNEnsemble", param_set = ps, predict_types = c("response", "se"), feature_types = c("logical", "integer", "numeric", "character", "factor", "ordered"), - properties = c("missings"), + properties = character(0), packages = c("mlr3mbo", "gower"), man = "mlr3mbo::mlr_learners_regr.gower_nn_ensemble" ) @@ -45,8 +46,10 @@ LearnerRegrGowerNNEnsemble = R6Class("LearnerRegrGowerNNEnsemble", private = list( .train = function(task) { - pv = self$param_set$get_values(tags = "train") + if(is.null(pv$mtry.ratio)) { + pv = insert_named(pv, list(mtry.ratio = floor(sqrt(length(task$feature_names))) / length(task$feature_names))) + } if (any(pv$ks >= task$nrow)) { index = which(pv$ks >= task$nrow) stopf("Parameter k = %i must be smaller than the number of observations n = %i", pv$k[index], task$nrow) @@ -70,7 +73,11 @@ LearnerRegrGowerNNEnsemble = R6Class("LearnerRegrGowerNNEnsemble", X = data[, self$model$feature_names, with = FALSE] y = data[, self$model$target_name, with = FALSE] - # FIXME: what about ordered here + stopifnot(all(colnames(X) == colnames(newdata))) + + mtry = as.integer(round(pv$mtry.ratio * ncol(X))) + + # FIXME: what about ordered here? for (feature in colnames(X)) { if ("factor" %in% class(X[[feature]])) { factor_levels = union(levels(X[[feature]]), levels(newdata[[feature]])) @@ -79,11 +86,12 @@ LearnerRegrGowerNNEnsemble = R6Class("LearnerRegrGowerNNEnsemble", } } - n = max(ceiling(pv$sample_fraction * nrow(X)), nrow(X)) + n = as.integer(max(ceiling(pv$sample.fraction * nrow(X)), nrow(X))) results = map_dtr(pv$ks, function(k) { ids = sample(seq_len(nrow(X)), size = n, replace = pv$replace) - top_n = invoke(gower::gower_topn, x = newdata, y = X[ids, ], n = k, nthread = pv$nthread)$index + features = sample(colnames(X), size = mtry, replace = FALSE) + top_n = invoke(gower::gower_topn, x = newdata[, features, with = FALSE], y = X[ids, features, with = FALSE], n = k, nthread = pv$nthread)$index tmp = map_dtc(seq_len(k), function(i) { setNames(y[ids, ][top_n[i, ], ], nm = paste0(self$model$target_name, "_", i)) }) From b4ecd66544dabc8f8677c53902b384f0ae2d7df8 Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Tue, 14 Mar 2023 11:48:01 +0100 Subject: [PATCH 060/126] fix: fix_xdt_missing for logical NA --- R/helper.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/helper.R b/R/helper.R index fa0e24f2..c0dca34c 100644 --- a/R/helper.R +++ b/R/helper.R @@ -27,7 +27,7 @@ archive_x = function(archive) { fix_xdt_missing = function(xdt, x_cols, archive) { missing = x_cols[x_cols %nin% colnames(xdt)] types = map_chr(missing, function(x) typeof(archive$data[[x]])) - NA_types = list(double = NA_real_, integer = NA_integer_, character = NA_character_)[types] + NA_types = list(double = NA_real_, integer = NA_integer_, character = NA_character_, logical = NA)[types] for (i in seq_along(missing)) { xdt[, eval(missing[i]) := NA_types[i]] } From 6023eaf53f18a1e5b5bd1bbe2f584eec5cf9cb82 Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Mon, 1 Apr 2024 19:10:09 +0200 Subject: [PATCH 061/126] .. --- .lintr | 2 +- DESCRIPTION | 7 +- NAMESPACE | 8 +- R/AcqFunctionLFBO.R | 80 ------------ R/AcqFunctionLogEI.R | 60 +++++++++ R/AcqFunctionTTEI.R | 166 ------------------------ R/LearnerRegrLFBO.R | 195 ----------------------------- R/LearnerRegrRangerCustom.R | 145 +++++++++++++++++++++ R/OptimizerChain.R | 114 +++++++++++++++++ R/OptimizerLocalSearch.R | 140 +++++++++++++++++++++ R/SurrogateLearner.R | 16 +-- R/bayesopt_ego_log.R | 40 +++--- R/bibentries.R | 34 +---- R/zzz.R | 2 +- man/AcqFunction.Rd | 5 +- man/SurrogateLearner.Rd | 9 +- man/mlr_acqfunctions.Rd | 5 +- man/mlr_acqfunctions_aei.Rd | 5 +- man/mlr_acqfunctions_cb.Rd | 5 +- man/mlr_acqfunctions_ehvi.Rd | 5 +- man/mlr_acqfunctions_ehvigh.Rd | 5 +- man/mlr_acqfunctions_ei.Rd | 5 +- man/mlr_acqfunctions_eips.Rd | 5 +- man/mlr_acqfunctions_lfbo.Rd | 140 --------------------- man/mlr_acqfunctions_log_ei.Rd | 111 ++++++++++++++++ man/mlr_acqfunctions_mean.Rd | 5 +- man/mlr_acqfunctions_pi.Rd | 5 +- man/mlr_acqfunctions_sd.Rd | 5 +- man/mlr_acqfunctions_smsego.Rd | 5 +- man/mlr_acqfunctions_ttei.Rd | 189 ---------------------------- man/mlr_learners_regr.ranger.Rd | 93 ++++++++++++++ man/mlr_learners_regr_lfbo.Rd | 194 ---------------------------- man/mlr_loop_functions_ego_log.Rd | 16 +-- man/mlr_optimizers_chain.Rd | 94 ++++++++++++++ man/mlr_optimizers_local_search.Rd | 88 +++++++++++++ tests/testthat/helper.R | 2 +- 36 files changed, 921 insertions(+), 1084 deletions(-) delete mode 100644 R/AcqFunctionLFBO.R create mode 100644 R/AcqFunctionLogEI.R delete mode 100644 R/AcqFunctionTTEI.R delete mode 100644 R/LearnerRegrLFBO.R create mode 100644 R/LearnerRegrRangerCustom.R create mode 100644 R/OptimizerChain.R create mode 100644 R/OptimizerLocalSearch.R delete mode 100644 man/mlr_acqfunctions_lfbo.Rd create mode 100644 man/mlr_acqfunctions_log_ei.Rd delete mode 100644 man/mlr_acqfunctions_ttei.Rd create mode 100644 man/mlr_learners_regr.ranger.Rd delete mode 100644 man/mlr_learners_regr_lfbo.Rd create mode 100644 man/mlr_optimizers_chain.Rd create mode 100644 man/mlr_optimizers_local_search.Rd diff --git a/.lintr b/.lintr index 3236892f..436e1b52 100644 --- a/.lintr +++ b/.lintr @@ -1,4 +1,4 @@ -linters: with_defaults( +linters: linters_with_defaults( # lintr defaults: https://github.com/jimhester/lintr#available-linters # the following setup changes/removes certain linters assignment_linter = NULL, # do not force using <- for assignments diff --git a/DESCRIPTION b/DESCRIPTION index 0ed93c91..47372dea 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -82,14 +82,15 @@ Collate: 'AcqFunctionEHVIGH.R' 'AcqFunctionEI.R' 'AcqFunctionEIPS.R' - 'AcqFunctionLFBO.R' + 'AcqFunctionLogEI.R' 'AcqFunctionMean.R' 'AcqFunctionPI.R' 'AcqFunctionSD.R' 'AcqFunctionSmsEgo.R' - 'AcqFunctionTTEI.R' 'AcqOptimizer.R' - 'LearnerRegrLFBO.R' + 'LearnerRegrRangerCustom.R' + 'OptimizerChain.R' + 'OptimizerLocalSearch.R' 'aaa.R' 'OptimizerMbo.R' 'mlr_result_assigners.R' diff --git a/NAMESPACE b/NAMESPACE index 4f18e270..a7ef0233 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -3,6 +3,7 @@ S3method(as.data.table,DictionaryAcqFunction) S3method(as.data.table,DictionaryLoopFunction) S3method(as.data.table,DictionaryResultAssigner) +S3method(default_values,LearnerRegrRangerCustom) S3method(print,loop_function) export(AcqFunction) export(AcqFunctionAEI) @@ -11,14 +12,15 @@ export(AcqFunctionEHVI) export(AcqFunctionEHVIGH) export(AcqFunctionEI) export(AcqFunctionEIPS) -export(AcqFunctionLFBO) +export(AcqFunctionLogEI) export(AcqFunctionMean) export(AcqFunctionPI) export(AcqFunctionSD) export(AcqFunctionSmsEgo) -export(AcqFunctionTTEI) export(AcqOptimizer) -export(LearnerRegrLFBO) +export(LearnerRegrRangerCustom) +export(OptimizerChain) +export(OptimizerLocalSearch) export(OptimizerMbo) export(ResultAssigner) export(ResultAssignerArchive) diff --git a/R/AcqFunctionLFBO.R b/R/AcqFunctionLFBO.R deleted file mode 100644 index e81f58f3..00000000 --- a/R/AcqFunctionLFBO.R +++ /dev/null @@ -1,80 +0,0 @@ -#' @title Acquisition Function LFBO -#' -#' @include AcqFunction.R -#' @name mlr_acqfunctions_lfbo -#' -#' @templateVar id lfbo -#' @template section_dictionary_acqfunctions -#' -#' @description -#' Likelihood-Free Bayesian Optimization. -#' Parameters specifying the weighting type and the gamma quantile of the target distribution have to be set in -#' the [paradox::ParamSet] of the [LearnerRegrLFBO] used within the [SurrogateLearner]. -#' -#' By default, it is assumed that the optimization problem is a minimization problem! -#' If maximization is required, change the value of the direction parameter of the [LearnerRegrLFBO] used within the -#' [SurrogateLearner]. -#' -#' @references -#' * `r format_bib("song_2022")` -#' -#' @family Acquisition Function -#' @export -#' @examples -#' \donttest{ -#' if (requireNamespace("mlr3learners") & -#' requireNamespace("ranger")) { -#' library(bbotk) -#' library(paradox) -#' library(mlr3learners) -#' library(data.table) -#' -#' fun = function(xs) { -#' list(y = xs$x ^ 2) -#' } -#' domain = ps(x = p_dbl(lower = -10, upper = 10)) -#' codomain = ps(y = p_dbl(tags = "minimize")) -#' objective = ObjectiveRFun$new(fun = fun, domain = domain, codomain = codomain) -#' -#' instance = OptimInstanceSingleCrit$new( -#' objective = objective, -#' terminator = trm("evals", n_evals = 5)) -#' -#' instance$eval_batch(data.table(x = c(-6, -5, 3, 9))) -#' -#' learner = lrn("regr.lfbo", learner_classif = lrn("classif.ranger"), lfbo.direction = "minimize") -#' -#' surrogate = srlrn(learner, archive = instance$archive) -#' -#' acq_function = acqf("lfbo", surrogate = surrogate) -#' -#' acq_function$surrogate$update() -#' acq_function$update() -#' acq_function$eval_dt(data.table(x = c(-1, 0, 1))) -#' } -#' } -AcqFunctionLFBO = R6Class("AcqFunctionLFBO", - inherit = AcqFunction, - - public = list( - - #' @description - #' Creates a new instance of this [R6][R6::R6Class] class. - #' - #' @param surrogate (`NULL` | [SurrogateLearner]). - initialize = function(surrogate = NULL) { - assert_r6(surrogate, "SurrogateLearner", null.ok = TRUE) - super$initialize("acq_lfbo", surrogate = surrogate, requires_predict_type_se = FALSE, direction = "maximize", label = "Likelihood-Free Bayesian Optimization", man = "mlr3mbo::mlr_acqfunctions_lfbo") - } - ), - - private = list( - .fun = function(xdt) { - p = self$surrogate$predict(xdt) - data.table(acq_lfbo = p$mean) - } - ) -) - -mlr_acqfunctions$add("lfbo", AcqFunctionLFBO) - diff --git a/R/AcqFunctionLogEI.R b/R/AcqFunctionLogEI.R new file mode 100644 index 00000000..e416a206 --- /dev/null +++ b/R/AcqFunctionLogEI.R @@ -0,0 +1,60 @@ +#' @title Acquisition Function Log Expected Improvement +#' +#' @include AcqFunction.R +#' @name mlr_acqfunctions_log_ei +#' +#' @templateVar id log_ei +#' @template section_dictionary_acqfunctions +#' +#' @description +#' Expected Improvement assuming that the target variable is to be minimized and has been modeled on log scale. +#' +#' @family Acquisition Function +#' @export +AcqFunctionLogEI = R6Class("AcqFunctionLogEI", + inherit = AcqFunction, + public = list( + + #' @field y_best (`numeric(1)`)\cr + #' Best objective function value observed so far. + #' On log scale. + y_best = NULL, + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + #' + #' @param surrogate (`NULL` | [SurrogateLearner]). + initialize = function(surrogate = NULL) { + assert_r6(surrogate, "SurrogateLearner", null.ok = TRUE) + super$initialize("acq_log_ei", surrogate = surrogate, requires_predict_type_se = TRUE, direction = "maximize", label = "Log Expected Improvement", man = "mlr3mbo::mlr_acqfunctions_log_ei") + }, + + #' @description + #' Updates acquisition function and sets `y_best`. + update = function() { + if (self$surrogate_max_to_min != 1L) { + stop("Log EI assumes minimization of the log transformed target value.") + } + self$y_best = min(self$surrogate_max_to_min * self$archive$data[[self$surrogate$y_cols]]) + } + ), + private = list( + .fun = function(xdt) { + if (is.null(self$y_best)) { + stop("$y_best is not set. Missed to call $update()?") + } + if (self$surrogate_max_to_min != 1L) { + stop("Log EI assumes minimization of the log transformed target value.") + } + p = self$surrogate$predict(xdt) + mu = p$mean + se = p$se + d_norm = (self$y_best - mu) / se + log_ei = (exp(self$y_best) * pnorm(d_norm)) - (exp((0.5 * se^2) + mu) * pnorm(d_norm - se)) + log_ei = ifelse(se < 1e-20, 0, log_ei) + data.table(acq_log_ei = log_ei) + } + ) +) + +mlr_acqfunctions$add("log_ei", AcqFunctionLogEI) diff --git a/R/AcqFunctionTTEI.R b/R/AcqFunctionTTEI.R deleted file mode 100644 index d600f39a..00000000 --- a/R/AcqFunctionTTEI.R +++ /dev/null @@ -1,166 +0,0 @@ -#' @title Acquisition Function Top-Two Expected Improvement -#' -#' @include AcqFunction.R -#' @name mlr_acqfunctions_ttei -#' -#' @templateVar id ttei -#' @template section_dictionary_acqfunctions -#' -#' @description -#' Top-Two Expected Improvement. -#' -#' With probability `beta` simply the Expected Improvement. -#' With probability `1 - beta` the Expected Improvement over the current Expected Improvement. -#' This requires to first obtain the current argmax of the Expected Improvement to then be able to -#' formulate the Expected Improvement with respect to the posterior prediction of that argmax. -#' -#' @references -#' * `r format_bib("qin_2017")` -#' -#' @family Acquisition Function -#' @export -#' @examples -#' if (requireNamespace("mlr3learners") & -#' requireNamespace("DiceKriging") & -#' requireNamespace("rgenoud")) { -#' library(bbotk) -#' library(paradox) -#' library(mlr3learners) -#' library(data.table) -#' -#' fun = function(xs) { -#' list(y = xs$x ^ 2) -#' } -#' domain = ps(x = p_dbl(lower = -10, upper = 10)) -#' codomain = ps(y = p_dbl(tags = "minimize")) -#' objective = ObjectiveRFun$new(fun = fun, domain = domain, codomain = codomain) -#' -#' instance = OptimInstanceSingleCrit$new( -#' objective = objective, -#' terminator = trm("evals", n_evals = 5)) -#' -#' instance$eval_batch(data.table(x = c(-6, -5, 3, 9))) -#' -#' learner = lrn("regr.km", -#' covtype = "matern3_2", -#' optim.method = "gen", -#' nugget.stability = 10^-8, -#' control = list(trace = FALSE)) -#' -#' surrogate = srlrn(learner, archive = instance$archive) -#' -#' acq_function = acqf("ttei", -#' surrogate = surrogate, -#' toplvl_acq_optimizer = acqo(opt("random_search"), terminator = trm("evals", n_evals = 100))) -#' -#' acq_function$surrogate$update() -#' acq_function$update() -#' acq_function$eval_dt(data.table(x = c(-1, 0, 1))) -#' } -AcqFunctionTTEI = R6Class("AcqFunctionTTEI", - inherit = AcqFunction, - - public = list( - #' @field y_best (`numeric(1)`)\cr - #' Best objective function value observed so far. - y_best = NULL, - - #' @field ei_argmax_mean_se ([data.table::data.table()])\cr - #' data.table::data.table() containing one row with the surrogate mean and standard error - #' prediction for the current argmax of the Expected Improvement. - ei_argmax_mean_se = NULL, # FIXME: assert type and format? - - #' @field is_ei (`logical(1)`)\cr - #' Whether in the next iteration simply the Expected Improvement is to be optimized or the - #' Expected Improvement over the Expected Improvement. - is_ei = NULL, - - #' @description - #' Creates a new instance of this [R6][R6::R6Class] class. - #' - #' @param surrogate (`NULL` | [SurrogateLearner]). - #' @param beta (`numeric(1)`)\cr - #' With probability `beta` the toplevel Expected Improvement acquisition function is to be - #' optimized and with `1 - beta` the Expected Improvement over the Expected Improvement. - #' Default is `0.5`. - #' @param toplvl_acq_optimizer (`NULL` | [AcqOptimizer])\cr - #' Acquisition function optimizer for the toplevel Expected Improvement acquisition function - #' used to find the current argmax. - #' If `NULL` will be initialized to - #' `acqo(opt("random_search", batch_size = 1000), terminator = trm("evals", n_evals = 10000))`. - initialize = function(surrogate = NULL, beta = 0.5, toplvl_acq_optimizer = NULL) { - assert_r6(surrogate, "SurrogateLearner", null.ok = TRUE) - assert_number(beta, lower = 0, upper = 1) - assert_r6(toplvl_acq_optimizer, classes = "AcqOptimizer", null.ok = TRUE) - if (is.null(toplvl_acq_optimizer)) toplvl_acq_optimizer = acqo(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 10000L)) - constants = ps(beta = p_dbl(lower = 0, upper = 1, default = 0.5)) - constants$values$beta = beta - private$.toplvl_acq_function_ei = AcqFunctionEI$new(surrogate = surrogate) # FIXME: AB and read-only? - private$.toplvl_acq_optimizer = toplvl_acq_optimizer$clone(deep = TRUE) # FIXME: AB and read-only? - super$initialize("acq_ttei", constants = constants, surrogate = surrogate, requires_predict_type_se = TRUE, direction = "maximize", label = "Top-Two Expected Improvement", man = "mlr3mbo::mlr_acqfunctions_ttei") - }, - - #' @description - #' Updates acquisition function and performs the following steps: - #' * sets `y_best` - #' * updates the internal toplevel Expected Improvement acquisition function - #' * samples whether simply the Expected Improvement is to be optimized or the Expected - #' Improvement over the Expected Improvement (depending on `beta`) and sets `is_ei` accordingly - #' * if `is_ei = FALSE`, proceeds to optimize the toplevel Expected Improvement acquisition function to - #' find the current argmax and sets `ei_argmax_mean_se` - update = function() { - self$y_best = min(self$surrogate_max_to_min * self$archive$data[[self$surrogate$y_cols]]) - private$.toplvl_acq_function_ei$surrogate = self$surrogate - private$.toplvl_acq_function_ei$update() - private$.toplvl_acq_optimizer$acq_function = private$.toplvl_acq_function_ei - if (runif(1, min = 0, max = 1) <= self$constants$values$beta) { - self$is_ei = TRUE - } else { - self$is_ei = FALSE - ei_argmax = private$.toplvl_acq_optimizer$optimize() - p_ei_argmax = self$surrogate$predict(ei_argmax) - self$ei_argmax_mean_se = cbind(ei_argmax, data.table(mean = p_ei_argmax$mean, se = p_ei_argmax$se)) - } - } - ), - - private = list( - .fun = function(xdt, ...) { - constants = list(...) - if (is.null(self$y_best)) { - stop("$y_best is not set. Missed to call $update()?") - } - if (is.null(self$is_ei)) { - stop("$is_ei is not set. Missed to call $update()?") - } - # can xdt_ei_argmax be NULL? - if (self$is_ei) { - p = self$surrogate$predict(xdt) - mu = p$mean - se = p$se - d = self$y_best - self$surrogate_max_to_min * mu - d_norm = d / se - ei = d * pnorm(d_norm) + se * dnorm(d_norm) - ei = ifelse(se < 1e-20, 0, ei) - data.table(acq_ttei = ei, acq_ttei_is_ei = self$is_ei) - } else { - p = self$surrogate$predict(xdt) - mu = p$mean - se = p$se - se_overall = sqrt((self$ei_argmax_mean_se$se ^ 2) + (se ^ 2)) - d = (self$surrogate_max_to_min * self$ei_argmax_mean_se$mean - self$surrogate_max_to_min * mu) - d_norm = d / se_overall - ttei = se_overall * (d_norm * pnorm(d_norm) + dnorm(d_norm)) - ttei = ifelse(se < 1e-20, 0, ttei) # FIXME: what about se_overall? - data.table(acq_ttei = ttei, acq_ttei_is_ei = self$is_ei) - } - }, - - .toplvl_acq_function_ei = NULL, - - .toplvl_acq_optimizer = NULL - ) -) - -mlr_acqfunctions$add("ttei", AcqFunctionTTEI) - diff --git a/R/LearnerRegrLFBO.R b/R/LearnerRegrLFBO.R deleted file mode 100644 index 3300bb4d..00000000 --- a/R/LearnerRegrLFBO.R +++ /dev/null @@ -1,195 +0,0 @@ -#' @title Likelihood Free Bayesian Optimization Learner -#' -#' @name mlr_learners_regr_lfbo -#' -#' @description -#' Wraps a classification learner to be used as a regression learner for -#' Likelihood-free Bayesian Optimization (LFBO). -#' -#' @details -#' The regression data is internally converted to a weighted classification task. -#' The new objective is a weighted version of a prediction problem, where the goal is -#' to predict whether a target value is smaller than a gamma quantile of the target -#' distribution (assuming minimization). -# -#' Note that the optimization direction must be specified explicitly! -#' If minimization is required, set the value of the `lfbo.direction` parameter to `"minimize"`. -#' If maximization is required, set the value of the `lfbo.direction` parameter to `"maximize"`. -#' -#' To specify the weighting type, set the value of the `lfbo.wt` parameter (`"ei"` or `"pi"`). -#' To specify the `gamma` quantile of the target distribution set the value of the `lfbo.gamma` parameter. -#' -#' This learner should only be used for Bayesian Optimization in combination with an [AcqFunctionLFBO]. -#' -#' The `$param_set` consists of a [paradox::ParamSetCollection] of the [paradox::ParamSet] of the `$learner_classif` and -#' an `"lfbo"` [paradox::ParamSet] including the parameters `"lfbo.direction"`, `"lfbo.wt"`, and `"lfbo.gamma"`. -#' -#' @references -#' * `r format_bib("song_2022")` -#' -#' @export -#' @examples -#' \donttest{ -#' if (requireNamespace("mlr3learners") & -#' requireNamespace("ranger")) { -#' -#' library(bbotk) -#' library(paradox) -#' library(mlr3learners) -#' -#' fun = function(xs) { -#' list(y = xs$x ^ 2) -#' } -#' domain = ps(x = p_dbl(lower = -10, upper = 10)) -#' codomain = ps(y = p_dbl(tags = "minimize")) -#' objective = ObjectiveRFun$new(fun = fun, domain = domain, codomain = codomain) -#' -#' instance = OptimInstanceSingleCrit$new( -#' objective = objective, -#' terminator = trm("evals", n_evals = 5)) -#' -#' surrogate = srlrn(lrn("regr.lfbo", lrn("classif.ranger"), lfbo.direction = "minimize")) -#' -#' acq_function = acqf("lfbo") -#' -#' acq_optimizer = acqo( -#' optimizer = opt("random_search", batch_size = 100), -#' terminator = trm("evals", n_evals = 100)) -#' -#' optimizer = opt("mbo", -#' loop_function = bayesopt_ego, -#' surrogate = surrogate, -#' acq_function = acq_function, -#' acq_optimizer = acq_optimizer) -#' -#' optimizer$optimize(instance) -#' } -#' } -LearnerRegrLFBO = R6Class("LearnerRegrLFBO", - inherit = mlr3::LearnerRegr, - public = list( - #' @field learner_classif ([mlr3::LearnerClassif])\cr - #' Classification learner to be used for LFBO. - learner_classif = NULL, - - #' @description - #' Creates a new instance of this [R6][R6::R6Class] class. - #' - #' @param learner_classif ([mlr3::LearnerClassif])\cr - #' Classifcation learner to be used for LFBO. - #' Requires `predict_type = "prob"` which will be set automatically during construction. - #' Also requires the learner to be able to handle case weights, see `learner$properties`. - initialize = function(learner_classif) { - assert_class(learner_classif, "LearnerClassif") - assert_true("prob" %in% learner_classif$predict_types) - learner_classif$predict_type = "prob" - assert_true("weights" %in% learner_classif$properties) - self$learner_classif = learner_classif - - lfbo_param_set = ps(direction = p_fct(levels = c("minimize", "maximize"), tags = c("required", "train")), - wt = p_fct(levels = c("ei", "pi"), default = "ei", tags = "train"), - gamma = p_dbl(lower = 0, upper = 1, default = 1/3, tags = "train")) - lfbo_param_set$set_id = "lfbo" - - super$initialize( - id = "regr.lfbo", - param_set = ParamSetCollection$new(list(lfbo_param_set, self$learner_classif$param_set)), - feature_types = learner_classif$feature_types, - predict_types = "response", - properties = setdiff(learner_classif$properties, c("twoclass", "multiclass")), - data_formats = learner_classif$data_formats, - packages = learner_classif$packages, - label = learner_classif$label, - man = "mlr3mbo::mlr_learners_regr_lfbo" - ) - }, - - #' @description - #' Train the learner on a set of observations of the provided `task`. - #' Mutates the learner by reference, i.e. stores the model alongside other information in field `$state`. - #' - #' @param task ([TaskRegr]). - #' - #' @param row_ids (`integer()`)\cr - #' Vector of training indices as subset of `task$row_ids`. - #' For a simple split into training and test set, see [mlr3::partition()]. - #' - #' @return - #' Returns the object itself, but modified **by reference**. - #' You need to explicitly `$clone()` the object beforehand if you want to keeps - #' the object in its previous state. - train = function(task, row_ids = NULL) { - # create df with new data - if (is.null(self$param_set$values$lfbo.direction)) { - stopf("Learner '%s' requires the specification of the 'lfbo.direction' hyperparameter prior to training.", self$id) - } - row_ids = assert_row_ids(row_ids, null.ok = TRUE) - task_new = private$.to_weighted_task(task, row_ids) - self$learner_classif$train(task_new, row_ids) - self$state$train_task = task_rm_backend(task$clone(deep = TRUE)) - invisible(NULL) - }, - - #' @description - #' Uses the information stored during `$train()` in `$learner$classif$state` to create a new [mlr3::PredictionRegr] - #' for a set of observations of the provided `task`. - #' - #' @param task ([mlr3::TaskRegr]). - #' - #' @param row_ids (`integer()`)\cr - #' Vector of test indices as subset of `task$row_ids`. - #' For a simple split into training and test set, see [mlr3::partition()]. - #' - #' @return [PredictionRegr]. - predict = function(task, row_ids = NULL) { - row_ids = assert_row_ids(row_ids, null.ok = TRUE) - features = task$data(rows = row_ids, cols = task$feature_names) - colnames(features) = paste0("X.", colnames(features)) - pred = self$learner_classif$predict_newdata(features)$data - prediction_data = as_prediction_data(list(response = pred$prob[, 2L]), task = task, row_ids = pred$row_ids) - as_prediction(prediction_data) - } - ), - - private = list( - .to_weighted_task = function(task, row_ids) { - data = do.call("cbind", private$.regr_to_weights( - task$data(rows = row_ids, cols = task$feature_names), - task$data(rows = row_ids, cols = task$target_names)[[1]] - )) - task_new = TaskClassif$new("weighted_classif", data, target = "z") - task_new$set_col_roles("w", "weight") - }, - - .regr_to_weights = function(X, y) { - # direction, wt and gamma are inferred from the param_set - # adapted from https://github.com/lfbo-ml/lfbo - direction = self$param_set$values$lfbo.direction - wt = if (is.null(self$param_set$values$lfbo.wt)) "ei" else self$param_set$values$lfbo.wt - gamma = if (is.null(self$param_set$values$lfbo.gamma)) 1/3 else self$param_set$values$lfbo.gamma - if (direction == "maximize") { - y = -1L * y - } - tau = quantile(y, probs = gamma) - z = y < tau - if (wt == "ei") { - w1 = tau - y[z] - w1 = w1 / mean(w1) - Xn = rbind(X, X[z,]) - zn = c(rep(0, length(z)), z[z]) - wn = c(rep(1, length(z)) / length(z), w1 * sum(z)) - wn = wn / mean(wn) - } else if (wt == "pi") { - Xn = X - wn = rep(1, nrow(X)) - zn = z - } - return(list(X = Xn, w = wn, z = as.factor(zn))) - }, - - .train = function(task) {}, - - .predict = function(task) {} - ) -) - diff --git a/R/LearnerRegrRangerCustom.R b/R/LearnerRegrRangerCustom.R new file mode 100644 index 00000000..5da03bb2 --- /dev/null +++ b/R/LearnerRegrRangerCustom.R @@ -0,0 +1,145 @@ +#' @title Custom Ranger Regression Learner +#' +#' @name mlr_learners_regr.ranger +#' +#' @description +#' Random regression forest. +#' Calls [ranger::ranger()] from package \CRANpkg{ranger}. +#' ... +#' +#' @export +LearnerRegrRangerCustom = R6Class("LearnerRegrRangerCustom", + inherit = LearnerRegr, + public = list( + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + initialize = function() { + ps = ps( + alpha = p_dbl(default = 0.5, tags = "train"), + always.split.variables = p_uty(tags = "train"), + holdout = p_lgl(default = FALSE, tags = "train"), + importance = p_fct(c("none", "impurity", "impurity_corrected", "permutation"), tags = "train"), + keep.inbag = p_lgl(default = FALSE, tags = "train"), + max.depth = p_int(default = NULL, lower = 0L, special_vals = list(NULL), tags = "train"), + min.node.size = p_int(1L, default = 5L, special_vals = list(NULL), tags = "train"), + min.prop = p_dbl(default = 0.1, tags = "train"), + minprop = p_dbl(default = 0.1, tags = "train"), + mtry = p_int(lower = 1L, special_vals = list(NULL), tags = "train"), + mtry.ratio = p_dbl(lower = 0, upper = 1, tags = "train"), + num.random.splits = p_int(1L, default = 1L, tags = "train"), + num.threads = p_int(1L, default = 1L, tags = c("train", "predict", "threads")), + num.trees = p_int(1L, default = 500L, tags = c("train", "predict", "hotstart")), + oob.error = p_lgl(default = TRUE, tags = "train"), + quantreg = p_lgl(default = FALSE, tags = "train"), + regularization.factor = p_uty(default = 1, tags = "train"), + regularization.usedepth = p_lgl(default = FALSE, tags = "train"), + replace = p_lgl(default = TRUE, tags = "train"), + respect.unordered.factors = p_fct(c("ignore", "order", "partition"), default = "ignore", tags = "train"), + sample.fraction = p_dbl(0L, 1L, tags = "train"), + save.memory = p_lgl(default = FALSE, tags = "train"), + scale.permutation.importance = p_lgl(default = FALSE, tags = "train"), + se.method = p_fct(c("jack", "infjack", "simple"), default = "infjack", tags = "predict"), # FIXME: only works if predict_type == "se" + seed = p_int(default = NULL, special_vals = list(NULL), tags = c("train", "predict")), + split.select.weights = p_uty(default = NULL, tags = "train"), + splitrule = p_fct(c("variance", "extratrees", "maxstat"), default = "variance", tags = "train"), + verbose = p_lgl(default = TRUE, tags = c("train", "predict")), + write.forest = p_lgl(default = TRUE, tags = "train") + ) + + ps$values = list(num.threads = 1L) + + # deps + ps$add_dep("num.random.splits", "splitrule", CondEqual$new("extratrees")) + ps$add_dep("alpha", "splitrule", CondEqual$new("maxstat")) + ps$add_dep("minprop", "splitrule", CondEqual$new("maxstat")) + ps$add_dep("scale.permutation.importance", "importance", CondEqual$new("permutation")) + + + super$initialize( + id = "regr.ranger_custom", + param_set = ps, + predict_types = c("response", "se"), + feature_types = c("logical", "integer", "numeric", "character", "factor", "ordered"), + properties = c("weights", "importance", "oob_error", "hotstart_backward"), + packages = c("mlr3learners", "ranger"), + man = "mlr3learners::mlr_learners_regr.ranger_custom" + ) + }, + + #' @description + #' The importance scores are extracted from the model slot `variable.importance`. + #' Parameter `importance.mode` must be set to `"impurity"`, `"impurity_corrected"`, or + #' `"permutation"` + #' + #' @return Named `numeric()`. + importance = function() { + if (is.null(self$model)) { + stopf("No model stored") + } + if (self$model$importance.mode == "none") { + stopf("No importance stored") + } + + sort(self$model$variable.importance, decreasing = TRUE) + }, + + #' @description + #' The out-of-bag error, extracted from model slot `prediction.error`. + #' + #' @return `numeric(1)`. + oob_error = function() { + if (is.null(self$model)) { + stopf("No model stored") + } + self$model$prediction.error + } + ), + private = list( + .train = function(task) { + pv = self$param_set$get_values(tags = "train") + pv = mlr3learners:::convert_ratio(pv, "mtry", "mtry.ratio", length(task$feature_names)) + + if (self$predict_type == "se") { + pv$keep.inbag = TRUE # nolint + } + + mlr3misc::invoke(ranger::ranger, + dependent.variable.name = task$target_names, + data = task$data(), + case.weights = task$weights$weight, + .args = pv + ) + }, + .predict = function(task) { + pv = self$param_set$get_values(tags = "predict") + newdata = mlr3learners:::ordered_features(task, self) + + if (isTRUE(pv$se.method == "simple")) { + prediction = mlr3misc::invoke(predict, self$model, data = newdata, type = "response", .args = pv[setdiff(names(pv), "se.method")], predict.all = TRUE) + response = rowMeans(prediction$predictions) + variance = rowMeans(0.01 + prediction$predictions^2) - (response^2) # law of total variance assuming sigma_b(x) is 0 due to min.node.size = 1 and always splitting; then set 0.01 as lower bound for sigma_b(x) + list(response = response, se = sqrt(variance)) + } else { + prediction = mlr3misc::invoke(predict, self$model, data = newdata, type = self$predict_type, .args = pv) + list(response = prediction$predictions, se = prediction$se) + } + }, + .hotstart = function(task) { + model = self$model + model$num.trees = self$param_set$values$num.trees + model + } + ) +) + +#' @export +default_values.LearnerRegrRangerCustom = function(x, search_space, task, ...) { # nolint + special_defaults = list( + mtry = floor(sqrt(length(task$feature_names))), + mtry.ratio = floor(sqrt(length(task$feature_names))) / length(task$feature_names), + sample.fraction = 1 + ) + defaults = insert_named(default_values(x$param_set), special_defaults) + defaults[search_space$ids()] +} diff --git a/R/OptimizerChain.R b/R/OptimizerChain.R new file mode 100644 index 00000000..a25a08da --- /dev/null +++ b/R/OptimizerChain.R @@ -0,0 +1,114 @@ +#' @title Run Optimizers Sequentially +#' +#' @name mlr_optimizers_chain +#' +#' @description +#' `OptimizerChain` allows to run different [Optimizer]s sequentially. +#' +#' For each [Optimizer] an (optional) additional [Terminator] can be specified +#' during construction. +#' While the initial [Terminator] guards the optimization process as a whole, +#' the additional [Terminator]s guard each individual [Optimizer]. +#' +#' The optimization then works as follows: +#' The first [Optimizer] is run on the [OptimInstance] using a [TerminatorCombo] +#' of the original [Terminator] of the [OptimInstance] and the (optional) +#' additional [Terminator] as passed during construction. +#' Once this [TerminatorCombo] indicates termination (usually via the +#' additional [Terminator]), the second [Optimizer] (and so on) is run unless +#' the original [Terminator] of the [OptimInstance] indicates termination. +#' +#' [OptimizerChain] can also be used for random restarts of the same +#' [Optimizer] (if applicable) by setting the [Terminator] of the [OptimInstance] to +#' [TerminatorNone] and setting identical additional [Terminator]s during +#' construction. +#' +#' +#' @section Parameters: +#' Parameters are inherited from the individual [Optimizer]s and collected as a +#' [paradox::ParamSetCollection] (with `set_id`s potentially postfixed via `_1`, `_2`, +#' ..., if the same [Optimizer]s are used multiple times). +#' +#' +#' @export +OptimizerChain = R6Class("OptimizerChain", + inherit = bbotk::Optimizer, + public = list( + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + #' + #' @param optimizers (list of [Optimizer]s). + #' @param terminators (list of [Terminator]s | NULL). + initialize = function(optimizers, terminators = rep(list(NULL), length(optimizers))) { + assert_list(optimizers, types = "Optimizer", any.missing = FALSE) + assert_list(terminators, types = c("Terminator", "NULL"), len = length(optimizers)) + + param_sets = vector(mode = "list", length = length(optimizers)) + ids_taken = character(0L) + # for each optimizer check whether the id of the param_set + # (decuded from the optimizer class) is already taken; + # if necessary postfix the id + for (i_opt in seq_along(optimizers)) { + opt = optimizers[[i_opt]] + ps = opt$param_set$clone(deep = TRUE) + ps$set_id = class(opt)[[1L]] + try_postfix = 0L + while (ps$set_id %in% ids_taken) { + try_postfix = try_postfix + 1L + ps$set_id = paste0(class(opt)[[1L]], "_", try_postfix) + } + ids_taken[[i_opt]] = ps$set_id + param_sets[[i_opt]] = ps + } + private$.ids = map_chr(param_sets, "set_id") + super$initialize( + param_set = ParamSetCollection$new(param_sets), + param_classes = Reduce(intersect, mlr3misc::map(optimizers, "param_classes")), + properties = Reduce(intersect, mlr3misc::map(optimizers, "properties")), + packages = unique(unlist(mlr3misc::map(optimizers, "packages"))) + ) + private$.optimizers = optimizers + private$.terminators = terminators + } + ), + private = list( + .optimizers = NULL, + .terminators = NULL, + .ids = NULL, + .optimize = function(inst) { + terminator = inst$terminator + on.exit({ + inst$terminator = terminator + }) + inner_inst = inst$clone(deep = TRUE) + + for (i_opt in seq_along(private$.optimizers)) { + inner_term = private$.terminators[[i_opt]] + if (!is.null(inner_term)) { + inner_inst$terminator = TerminatorCombo$new(list(inner_term, terminator)) + } else { + inner_inst$terminator = terminator + } + opt = private$.optimizers[[i_opt]] + opt$param_set$values = self$param_set$.__enclos_env__$private$.sets[[i_opt]]$values + opt$optimize(inner_inst) + inner_inst$archive$data$batch_nr = max(inst$archive$data$batch_nr, 0L) + + inner_inst$archive$data$batch_nr + inner_inst$archive$data$optimizer = private$.ids[i_opt] + inst$archive$data = rbind(inst$archive$data, inner_inst$archive$data, fill = TRUE) + inner_inst$archive$data = data.table() + if (terminator$is_terminated(inst$archive)) { + break + } + } + }, + deep_clone = function(name, value) { + switch(name, + .optimizers = mlr3misc::map(value, .f = function(x) x$clone(deep = TRUE)), + .terminators = mlr3misc::map(value, .f = function(x) if (!is.null(x)) x$clone(deep = TRUE)), + value + ) + } + ) +) diff --git a/R/OptimizerLocalSearch.R b/R/OptimizerLocalSearch.R new file mode 100644 index 00000000..779506d9 --- /dev/null +++ b/R/OptimizerLocalSearch.R @@ -0,0 +1,140 @@ +#' @title Optimization via Local Search +#' +#' @name mlr_optimizers_local_search +#' +#' @description +#' `OptimizerLocalSearch` class that implements a simple Local Search. +#' Local Search starts by determining the `mu` initial best points present in the [Archive] of the +#' [OptimInstance]. If fewer points than `mu` are present, additional points sampled uniformly at +#' random are evaluated. +#' +#' In each iteration, for each of the `mu` initial best points, `n_points` neighbors are generated +#' by local mutation. Local mutation generates a neighbor by sampling a single parameter that is to +#' be mutated and then proceeds as follows: Double parameters ([paradox::ParamDbl]) are mutated via +#' Gaussian mutation (with a prior standardization to `[0, 1]` and retransformation after mutation). +#' Integer parameters ([paradox::ParamInt]) undergo the same mutation but are rounded to the closest +#' integer after mutation. Categorical parameters ([paradox::ParamFct] and [paradox::ParamLgl]) are +#' mutated via uniform mutation. Note that parameters that are conditioned on (i.e., they are +#' parents of a [paradox::Condition], see the dependencies of the search space) are not mutated. +#' +#' @section Parameters: +#' \describe{ +#' \item{`mu`}{`integer(1)`\cr +#' Size of the initial best points which are used as a starting points for the Local Search. +#' Default is `10`. +#' } +#' \item{`n_points`}{`integer(1)`\cr +#' Number of neighboring points to generate for each of the `mu` best starting points in each +#' iteration. +#' Default is `100`. +#' } +#' \item{`sigma`}{`numeric(1)`\cr +#' Standard deviation used for mutation of numeric parameters. +#' Default is `0.1`. +#' } +#' } +#' +#' @export +OptimizerLocalSearch = R6Class("OptimizerLocalSearch", + inherit = bbotk::Optimizer, + public = list( + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + initialize = function() { + param_set = ps( + mu = p_int(lower = 1L, default = 10L, tags = "required"), + n_points = p_int(lower = 1L, default = 100L, tags = "required"), + sigma = p_dbl(lower = 0L, default = 0.1, tags = "required") + ) + param_set$values = list(mu = 10L, n_points = 100L, sigma = 0.1) + + super$initialize( + id = "local_search", + param_set = param_set, + param_classes = c("ParamLgl", "ParamInt", "ParamDbl", "ParamFct"), + properties = c("dependencies", "single-crit"), # NOTE: think about multi-crit version + label = "Local Search", + man = "mlr3mbo::mlr_optimizers_local_search" + ) + } + ), + private = list( + .optimize = function(inst) { + mu = self$param_set$values$mu + mu_seq = seq_len(mu) + n_points = self$param_set$values$n_points + n_points_seq = seq_len(n_points) + sigma = self$param_set$values$sigma + + # if no reference points in archive, generate mu by sampling uniformly at random + if (inst$archive$n_evals < mu) { + data = generate_design_random(inst$search_space, n = mu - inst$archive$n_evals)$data + inst$eval_batch(data) + } + points = inst$archive$best(n_select = mu)[, c(inst$archive$cols_x, inst$archive$cols_y), with = FALSE] + + # we do not mutate parents of conditions + ids_to_mutate = setdiff(inst$search_space$ids(), unique(inst$search_space$deps$on)) + + ids_numeric = intersect(names(which(inst$search_space$is_number)), ids_to_mutate) + + ids_categorical = intersect(names(which(inst$search_space$is_categ)), ids_to_mutate) + ids_categorical = ids_categorical[map_lgl(inst$search_space$params[ids_categorical], function(x) x$nlevels > 1L)] + + point_id = ".point_id" + while (point_id %in% c(inst$archive$cols_x, inst$archive$cols_y)) { + point_id = paste0(".", point_id) + } + + repeat { # iterate until we have an exception from eval_batch + # generate neighbors + neighbors = map_dtr(mu_seq, function(i) { + neighbors_i = map_dtr(n_points_seq, function(j) { + # NOTE: mutating is currently quite slow because we sample the id to be mutated and the actual mutation for each neighbor and new point + mutate_point(points[i, inst$archive$cols_x, with = FALSE], search_space = inst$search_space, ids_numeric = ids_numeric, ids_categorical = ids_categorical, sigma = sigma) + }) + set(neighbors_i, j = point_id, value = i) + }) + + # evaluate neighbors + inst$eval_batch(neighbors) + + # update points if better neighbor found + for (i in mu_seq) { + tmp = inst$archive$data[batch_nr == inst$archive$n_batch & get(point_id) == i] + difference = (tmp[[inst$archive$cols_y]] * inst$objective_multiplicator) - (points[i, ][[inst$archive$cols_y]] * inst$objective_multiplicator) + if (any(difference < 0)) { + best = which.min(difference) + points[i, ] = tmp[best, c(inst$archive$cols_x, inst$archive$cols_y), with = FALSE] + } + } + } + } + ) +) + +mutate_point = function(point, search_space, ids_numeric, ids_categorical, sigma) { + neighbor = copy(point) + valid_numeric_to_mutate = intersect(names(which(!map_lgl(neighbor, is.na))), ids_numeric) + valid_cateorical_to_mutate = intersect(names(which(!map_lgl(neighbor, is.na))), ids_categorical) + id = sample(c(valid_numeric_to_mutate, valid_cateorical_to_mutate), size = 1L) + neighbor[1L, ][[id]] = mutate(neighbor[1L, ][[id]], param = search_space$params[[id]], sigma = sigma) + neighbor +} + +mutate = function(value, param, sigma) { + stopifnot(param$class %in% c("ParamDbl", "ParamFct", "ParamInt", "ParamLgl")) + if (param$class %in% c("ParamDbl", "ParamInt")) { + value_ = (value - param$lower) / (param$upper - param$lower) + value_ = max(0, min(stats::rnorm(1L, mean = value_, sd = sigma), 1)) + value = (value_ * (param$upper - param$lower)) + param$lower + if (param$class == "ParamInt") { + value = round(value, 0L) + } + value = min(max(value, param$lower), param$upper) + } else if (param$class %in% c("ParamFct", "ParamLgl")) { + value = sample(setdiff(param$levels, value), size = 1L) + } + value +} diff --git a/R/SurrogateLearner.R b/R/SurrogateLearner.R index 86b9ac53..3ed4e3e9 100644 --- a/R/SurrogateLearner.R +++ b/R/SurrogateLearner.R @@ -31,14 +31,14 @@ #' @export #' @examples #' if (requireNamespace("mlr3learners") & -#' requireNamespace("DiceKriging") & -#' requireNamespace("rgenoud")) { +#' requireNamespace("DiceKriging") & +#' requireNamespace("rgenoud")) { #' library(bbotk) #' library(paradox) #' library(mlr3learners) #' #' fun = function(xs) { -#' list(y = xs$x ^ 2) +#' list(y = xs$x^2) #' } #' domain = ps(x = p_dbl(lower = -10, upper = 10)) #' codomain = ps(y = p_dbl(tags = "minimize")) @@ -46,7 +46,8 @@ #' #' instance = OptimInstanceSingleCrit$new( #' objective = objective, -#' terminator = trm("evals", n_evals = 5)) +#' terminator = trm("evals", n_evals = 5) +#' ) #' #' xdt = generate_design_random(instance$search_space, n = 4)$data #' @@ -62,7 +63,6 @@ #' } SurrogateLearner = R6Class("SurrogateLearner", inherit = Surrogate, - public = list( #' @description @@ -85,7 +85,7 @@ SurrogateLearner = R6Class("SurrogateLearner", ps = ps( assert_insample_perf = p_lgl(), - perf_measure = p_uty(custom_check = function(x) check_r6(x, classes = "MeasureRegr")), # FIXME: actually want check_measure + perf_measure = p_uty(custom_check = function(x) check_r6(x, classes = "MeasureRegr")), # FIXME: actually want check_measure perf_threshold = p_dbl(lower = -Inf, upper = Inf), catch_errors = p_lgl() ) @@ -115,7 +115,6 @@ SurrogateLearner = R6Class("SurrogateLearner", } } ), - active = list( #' @template field_print_id @@ -209,7 +208,6 @@ SurrogateLearner = R6Class("SurrogateLearner", } } ), - private = list( # Train learner with new data. # Also calculates the insample performance based on the `perf_measure` hyperparameter if `assert_insample_perf = TRUE`. @@ -225,7 +223,6 @@ SurrogateLearner = R6Class("SurrogateLearner", self$assert_insample_perf } }, - deep_clone = function(name, value) { switch(name, learner = value$clone(deep = TRUE), @@ -236,4 +233,3 @@ SurrogateLearner = R6Class("SurrogateLearner", } ) ) - diff --git a/R/bayesopt_ego_log.R b/R/bayesopt_ego_log.R index e2873730..33c29777 100644 --- a/R/bayesopt_ego_log.R +++ b/R/bayesopt_ego_log.R @@ -21,7 +21,7 @@ #' Size of the initial design. #' If `NULL` and the [bbotk::Archive] contains no evaluations, \code{4 * d} is used with \code{d} being the #' dimensionality of the search space. -#' Points are drawn uniformly at random. +#' Points are generated via a Sobol sequence. #' @param surrogate ([Surrogate])\cr #' [Surrogate] to be used as a surrogate. #' Typically a [SurrogateLearner]. @@ -79,7 +79,7 @@ #' acq_function = acqf("ei") #' #' acq_optimizer = acqo( -#' optimizer = opt("random_search"), +#' optimizer = opt("random_search", batch_size = 100), #' terminator = trm("evals", n_evals = 100)) #' #' optimizer = opt("mbo", @@ -93,48 +93,46 @@ #'} bayesopt_ego_log = function( instance, - init_design_size = NULL, surrogate, acq_function, acq_optimizer, + init_design_size = NULL, random_interleave_iter = 0L, epsilon = 1e-3 ) { # assertions and defaults assert_r6(instance, "OptimInstanceSingleCrit") - assert_int(init_design_size, lower = 1L, null.ok = TRUE) assert_r6(surrogate, classes = "Surrogate") # cannot be SurrogateLearner due to EIPS assert_r6(acq_function, classes = "AcqFunction") # FIXME: should explicityly assert acqfs and make sure that codomain tag is handled assert_r6(acq_optimizer, classes = "AcqOptimizer") + assert_int(init_design_size, lower = 1L, null.ok = TRUE) assert_int(random_interleave_iter, lower = 0L) assert_number(epsilon, lower = 0, upper = 1) - archive = instance$archive - domain = instance$search_space - d = domain$length - if (is.null(init_design_size) && instance$archive$n_evals == 0L) init_design_size = 4L * d + # initial design + search_space = instance$search_space + if (is.null(init_design_size) && instance$archive$n_evals == 0L) { + init_design_size = 4L * search_space$length + } + if (!is.null(init_design_size) && instance$archive$n_evals == 0L) { + design = generate_design_sobol(search_space, n = init_design_size)$data + instance$eval_batch(design) + } - surrogate$archive = archive + # completing initialization + surrogate$archive = instance$archive acq_function$surrogate = surrogate acq_optimizer$acq_function = acq_function - surrogate$y_cols = "y_trafo" + surrogate$cols_y = "y_trafo" acq_function$surrogate_max_to_min = 1 - if (test_r6(acq_function, classes = "AcqFunctionCB") | test_r6(acq_function, classes = "AcqFunctionMean")) { + if (test_r6(acq_function, classes = "AcqFunctionCB") || test_r6(acq_function, classes = "AcqFunctionMean")) { acq_function$codomain$params[[acq_function$id]]$tags = "minimize" } - # initial design - if (isTRUE(init_design_size > 0L)) { - design = generate_design_random(domain, n = init_design_size)$data - instance$eval_batch(design) - } else { - init_design_size = instance$archive$n_evals - } - - # loop + # actual loop repeat { y = instance$archive$data[[instance$archive$cols_y]] * instance$objective_multiplicator[instance$archive$cols_y] min_y = min(y) - epsilon * diff(range(y)) @@ -153,7 +151,7 @@ bayesopt_ego_log = function( }, mbo_error = function(mbo_error_condition) { lg$info(paste0(class(mbo_error_condition), collapse = " / ")) lg$info("Proposing a randomly sampled point") - SamplerUnif$new(domain)$sample(1L)$data + generate_design_random(search_space, n = 1L)$data }) instance$eval_batch(xdt) diff --git a/R/bibentries.R b/R/bibentries.R index 1b823934..6c8d6009 100644 --- a/R/bibentries.R +++ b/R/bibentries.R @@ -14,7 +14,6 @@ bibentries = c( pages = "431--431", journal = "Journal of Global Optimization" ), - jones_1998 = bibentry("article", title = "Efficient Global Optimization of Expensive Black-Box Functions", author = "Jones, Donald R. and Schonlau, Matthias and Welch, William J.", @@ -24,7 +23,6 @@ bibentries = c( pages = "455--492", journal = "Journal of Global optimization" ), - ding_2010 = bibentry("article", title = "An Investigation of Missing Data Methods for Classification Trees Applied to Binary Response Data", author = "Ding, Yufeng and Simonoff, Jeffrey S", @@ -34,7 +32,6 @@ bibentries = c( pages = "131--170", journal = "Journal of Machine Learning Research" ), - beume_2007 = bibentry("article", title = "SMS-EMOA: Multiobjective selection based on dominated hypervolume", author = "Nicola Beume and Boris Naujoks and Michael Emmerich", @@ -44,8 +41,7 @@ bibentries = c( pages = "1653--1669", journal = "European Journal of Operational Research" ), - - snoek_2012 = bibentry("inproceedings", + snoek_2012 = bibentry("inproceedings", title = "Practical {B}ayesian Optimization of Machine Learning Algorithms", author = "Snoek, Jasper and Larochelle, Hugo and Adams, Ryan P", year = "2012", @@ -54,7 +50,6 @@ bibentries = c( pages = "2951--2959", volume = "25" ), - kushner_1964 = bibentry("article", title = "A New Method of Locating the Maximum Point of an Arbitrary Multipeak Curve in the Presence of Noise", author = "Kushner, H. J.", @@ -64,7 +59,6 @@ bibentries = c( number = "1", pages = "97--106", ), - ponweiser_2008 = bibentry("inproceedings", title = "Multiobjective Optimization on a Limited Budget of Evaluations Using Model-Assisted S-Metric Selection", author = "Ponweiser, Wolfgang and Wagner, Tobias and Biermann, Dirk and Vincze, Markus", @@ -72,7 +66,6 @@ bibentries = c( booktitle = "Proceedings of the 10th International Conference on Parallel Problem Solving from Nature", pages = "784--794" ), - wang_2020 = bibentry("article", title = "Parallel {B}ayesian Global Optimization of Expensive Functions", author = "Wang, Jialei and Clark, Scott C. and Liu, Eric and Frazier, Peter I.", @@ -82,7 +75,6 @@ bibentries = c( number = "6", pages = "1850--1865" ), - knowles_2006 = bibentry("article", title = "ParEGO: A Hybrid Algorithm With On-Line Landscape Approximation for Expensive Multiobjective Optimization Problems", volume = "10", @@ -92,39 +84,18 @@ bibentries = c( year = "2006", pages = "50--66" ), - - horn_2015 = bibentry("inproceedings", + horn_2015 = bibentry("inproceedings", title = "Model-Based Multi-objective Optimization: Taxonomy, Multi-Point Proposal, Toolbox and Benchmark", author = "Horn, Daniel and Wagner, Tobias and Biermann, Dirk and Weihs, Claus and Bischl, Bernd", year = "2015", booktitle = "International Conference on Evolutionary Multi-Criterion Optimization", pages = "64--78" ), - ginsbourger_2008 = bibentry("misc", title = "A Multi-Points Criterion for Deterministic Parallel Global Optimization Based on Gaussian Processes", author = "Ginsbourger, David and Le Riche, Rodolphe and Carraro, Laurent", year = "2008" ), - - qin_2017 = bibentry("article", - title = "Improving the Expected Improvement Algorithm", - author = "Qin, Chao and Klabjan, Diego and Russo, Daniel", - journal = "Advances in Neural Information Processing Systems", - volume = "30", - year = "2017" - ), - - song_2022 = bibentry("inproceedings", - title = "A General Recipe for Likelihood-Free {B}ayesian Optimization", - author = "Song, Jiaming and Yu, Lantao and Neiswanger, Willie and Ermon, Stefano", - year = "2022", - booktitle = "Proceedings of the 39th International Conference on Machine Learning", - editor = "K. Chaudhuri and S. Jegelka and L. Song and C. Szepesvari and G. Niu and S. Sabato", - pages = "20384--20404", - volume = "162" - ), - emmerich_2016 = bibentry("incollection", title = "A Multicriteria Generalization of Bayesian Global Optimization", author = "Emmerich, Michael and Yang, Kaifeng and Deutz, Andr{\\'e} and Wang, Hao and Fonseca, Carlos M.", @@ -135,7 +106,6 @@ bibentries = c( address = "Cham", pages = "229--242" ), - rahat_2022 = bibentry("inproceedings", title = "Efficient Approximation of Expected Hypervolume Improvement using Gauss-Hermit Quadrature", author = "Rahat, Alma and Chugh, Tinkle and Fieldsend, Jonathan and Allmendinger, Richard and Miettinen, Kaisa", diff --git a/R/zzz.R b/R/zzz.R index 9ca913c7..18ab60f4 100644 --- a/R/zzz.R +++ b/R/zzz.R @@ -44,7 +44,7 @@ register_mlr3tuning = function() { } # nocov end # static code checks should not complain about commonly used data.table columns -utils::globalVariables(c("y_scal", "y_trafo")) +utils::globalVariables(c("y_scal", "y_trafo", "predict", "batch_nr")) if (!("mlr3mbo" %in% Sys.getenv("DEVTOOLS_LOAD"))) { leanify_package() diff --git a/man/AcqFunction.Rd b/man/AcqFunction.Rd index 5084b504..638709e2 100644 --- a/man/AcqFunction.Rd +++ b/man/AcqFunction.Rd @@ -17,12 +17,11 @@ Other Acquisition Function: \code{\link{mlr_acqfunctions_ehvigh}}, \code{\link{mlr_acqfunctions_ei}}, \code{\link{mlr_acqfunctions_eips}}, -\code{\link{mlr_acqfunctions_lfbo}}, +\code{\link{mlr_acqfunctions_log_ei}}, \code{\link{mlr_acqfunctions_mean}}, \code{\link{mlr_acqfunctions_pi}}, \code{\link{mlr_acqfunctions_sd}}, -\code{\link{mlr_acqfunctions_smsego}}, -\code{\link{mlr_acqfunctions_ttei}} +\code{\link{mlr_acqfunctions_smsego}} } \concept{Acquisition Function} \section{Super class}{ diff --git a/man/SurrogateLearner.Rd b/man/SurrogateLearner.Rd index 37702fc2..16641305 100644 --- a/man/SurrogateLearner.Rd +++ b/man/SurrogateLearner.Rd @@ -35,14 +35,14 @@ Default is \code{TRUE}. \examples{ if (requireNamespace("mlr3learners") & - requireNamespace("DiceKriging") & - requireNamespace("rgenoud")) { + requireNamespace("DiceKriging") & + requireNamespace("rgenoud")) { library(bbotk) library(paradox) library(mlr3learners) fun = function(xs) { - list(y = xs$x ^ 2) + list(y = xs$x^2) } domain = ps(x = p_dbl(lower = -10, upper = 10)) codomain = ps(y = p_dbl(tags = "minimize")) @@ -50,7 +50,8 @@ if (requireNamespace("mlr3learners") & instance = OptimInstanceSingleCrit$new( objective = objective, - terminator = trm("evals", n_evals = 5)) + terminator = trm("evals", n_evals = 5) + ) xdt = generate_design_random(instance$search_space, n = 4)$data diff --git a/man/mlr_acqfunctions.Rd b/man/mlr_acqfunctions.Rd index 762a75f5..428e1f76 100644 --- a/man/mlr_acqfunctions.Rd +++ b/man/mlr_acqfunctions.Rd @@ -38,12 +38,11 @@ Other Acquisition Function: \code{\link{mlr_acqfunctions_ehvigh}}, \code{\link{mlr_acqfunctions_ei}}, \code{\link{mlr_acqfunctions_eips}}, -\code{\link{mlr_acqfunctions_lfbo}}, +\code{\link{mlr_acqfunctions_log_ei}}, \code{\link{mlr_acqfunctions_mean}}, \code{\link{mlr_acqfunctions_pi}}, \code{\link{mlr_acqfunctions_sd}}, -\code{\link{mlr_acqfunctions_smsego}}, -\code{\link{mlr_acqfunctions_ttei}} +\code{\link{mlr_acqfunctions_smsego}} } \concept{Acquisition Function} \concept{Dictionary} diff --git a/man/mlr_acqfunctions_aei.Rd b/man/mlr_acqfunctions_aei.Rd index 0ade2b53..2443a8ae 100644 --- a/man/mlr_acqfunctions_aei.Rd +++ b/man/mlr_acqfunctions_aei.Rd @@ -85,12 +85,11 @@ Other Acquisition Function: \code{\link{mlr_acqfunctions_ehvigh}}, \code{\link{mlr_acqfunctions_ei}}, \code{\link{mlr_acqfunctions_eips}}, -\code{\link{mlr_acqfunctions_lfbo}}, +\code{\link{mlr_acqfunctions_log_ei}}, \code{\link{mlr_acqfunctions_mean}}, \code{\link{mlr_acqfunctions_pi}}, \code{\link{mlr_acqfunctions_sd}}, -\code{\link{mlr_acqfunctions_smsego}}, -\code{\link{mlr_acqfunctions_ttei}} +\code{\link{mlr_acqfunctions_smsego}} } \concept{Acquisition Function} \section{Super classes}{ diff --git a/man/mlr_acqfunctions_cb.Rd b/man/mlr_acqfunctions_cb.Rd index 54b80702..488d7e6f 100644 --- a/man/mlr_acqfunctions_cb.Rd +++ b/man/mlr_acqfunctions_cb.Rd @@ -74,12 +74,11 @@ Other Acquisition Function: \code{\link{mlr_acqfunctions_ehvigh}}, \code{\link{mlr_acqfunctions_ei}}, \code{\link{mlr_acqfunctions_eips}}, -\code{\link{mlr_acqfunctions_lfbo}}, +\code{\link{mlr_acqfunctions_log_ei}}, \code{\link{mlr_acqfunctions_mean}}, \code{\link{mlr_acqfunctions_pi}}, \code{\link{mlr_acqfunctions_sd}}, -\code{\link{mlr_acqfunctions_smsego}}, -\code{\link{mlr_acqfunctions_ttei}} +\code{\link{mlr_acqfunctions_smsego}} } \concept{Acquisition Function} \section{Super classes}{ diff --git a/man/mlr_acqfunctions_ehvi.Rd b/man/mlr_acqfunctions_ehvi.Rd index 74d3dfac..06e41371 100644 --- a/man/mlr_acqfunctions_ehvi.Rd +++ b/man/mlr_acqfunctions_ehvi.Rd @@ -60,12 +60,11 @@ Other Acquisition Function: \code{\link{mlr_acqfunctions_ehvigh}}, \code{\link{mlr_acqfunctions_ei}}, \code{\link{mlr_acqfunctions_eips}}, -\code{\link{mlr_acqfunctions_lfbo}}, +\code{\link{mlr_acqfunctions_log_ei}}, \code{\link{mlr_acqfunctions_mean}}, \code{\link{mlr_acqfunctions_pi}}, \code{\link{mlr_acqfunctions_sd}}, -\code{\link{mlr_acqfunctions_smsego}}, -\code{\link{mlr_acqfunctions_ttei}} +\code{\link{mlr_acqfunctions_smsego}} } \concept{Acquisition Function} \section{Super classes}{ diff --git a/man/mlr_acqfunctions_ehvigh.Rd b/man/mlr_acqfunctions_ehvigh.Rd index 567c7a3d..e931f43f 100644 --- a/man/mlr_acqfunctions_ehvigh.Rd +++ b/man/mlr_acqfunctions_ehvigh.Rd @@ -74,12 +74,11 @@ Other Acquisition Function: \code{\link{mlr_acqfunctions_ehvi}}, \code{\link{mlr_acqfunctions_ei}}, \code{\link{mlr_acqfunctions_eips}}, -\code{\link{mlr_acqfunctions_lfbo}}, +\code{\link{mlr_acqfunctions_log_ei}}, \code{\link{mlr_acqfunctions_mean}}, \code{\link{mlr_acqfunctions_pi}}, \code{\link{mlr_acqfunctions_sd}}, -\code{\link{mlr_acqfunctions_smsego}}, -\code{\link{mlr_acqfunctions_ttei}} +\code{\link{mlr_acqfunctions_smsego}} } \concept{Acquisition Function} \section{Super classes}{ diff --git a/man/mlr_acqfunctions_ei.Rd b/man/mlr_acqfunctions_ei.Rd index d3f8f009..e67bdeec 100644 --- a/man/mlr_acqfunctions_ei.Rd +++ b/man/mlr_acqfunctions_ei.Rd @@ -66,12 +66,11 @@ Other Acquisition Function: \code{\link{mlr_acqfunctions_ehvi}}, \code{\link{mlr_acqfunctions_ehvigh}}, \code{\link{mlr_acqfunctions_eips}}, -\code{\link{mlr_acqfunctions_lfbo}}, +\code{\link{mlr_acqfunctions_log_ei}}, \code{\link{mlr_acqfunctions_mean}}, \code{\link{mlr_acqfunctions_pi}}, \code{\link{mlr_acqfunctions_sd}}, -\code{\link{mlr_acqfunctions_smsego}}, -\code{\link{mlr_acqfunctions_ttei}} +\code{\link{mlr_acqfunctions_smsego}} } \concept{Acquisition Function} \section{Super classes}{ diff --git a/man/mlr_acqfunctions_eips.Rd b/man/mlr_acqfunctions_eips.Rd index f81e24ba..08ae3229 100644 --- a/man/mlr_acqfunctions_eips.Rd +++ b/man/mlr_acqfunctions_eips.Rd @@ -74,12 +74,11 @@ Other Acquisition Function: \code{\link{mlr_acqfunctions_ehvi}}, \code{\link{mlr_acqfunctions_ehvigh}}, \code{\link{mlr_acqfunctions_ei}}, -\code{\link{mlr_acqfunctions_lfbo}}, +\code{\link{mlr_acqfunctions_log_ei}}, \code{\link{mlr_acqfunctions_mean}}, \code{\link{mlr_acqfunctions_pi}}, \code{\link{mlr_acqfunctions_sd}}, -\code{\link{mlr_acqfunctions_smsego}}, -\code{\link{mlr_acqfunctions_ttei}} +\code{\link{mlr_acqfunctions_smsego}} } \concept{Acquisition Function} \section{Super classes}{ diff --git a/man/mlr_acqfunctions_lfbo.Rd b/man/mlr_acqfunctions_lfbo.Rd deleted file mode 100644 index 48ddc5c8..00000000 --- a/man/mlr_acqfunctions_lfbo.Rd +++ /dev/null @@ -1,140 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/AcqFunctionLFBO.R -\name{mlr_acqfunctions_lfbo} -\alias{mlr_acqfunctions_lfbo} -\alias{AcqFunctionLFBO} -\title{Acquisition Function LFBO} -\description{ -Likelihood-Free Bayesian Optimization. -Parameters specifying the weighting type and the gamma quantile of the target distribution have to be set in -the \link[paradox:ParamSet]{paradox::ParamSet} of the \link{LearnerRegrLFBO} used within the \link{SurrogateLearner}. - -By default, it is assumed that the optimization problem is a minimization problem! -If maximization is required, change the value of the direction parameter of the \link{LearnerRegrLFBO} used within the -\link{SurrogateLearner}. -} -\section{Dictionary}{ - -This \link{AcqFunction} can be instantiated via the \link[mlr3misc:Dictionary]{dictionary} -\link{mlr_acqfunctions} or with the associated sugar function \code{\link[=acqf]{acqf()}}: - -\if{html}{\out{
}}\preformatted{mlr_acqfunctions$get("lfbo") -acqf("lfbo") -}\if{html}{\out{
}} -} - -\examples{ -\donttest{ -if (requireNamespace("mlr3learners") & - requireNamespace("ranger")) { - library(bbotk) - library(paradox) - library(mlr3learners) - library(data.table) - - fun = function(xs) { - list(y = xs$x ^ 2) - } - domain = ps(x = p_dbl(lower = -10, upper = 10)) - codomain = ps(y = p_dbl(tags = "minimize")) - objective = ObjectiveRFun$new(fun = fun, domain = domain, codomain = codomain) - - instance = OptimInstanceSingleCrit$new( - objective = objective, - terminator = trm("evals", n_evals = 5)) - - instance$eval_batch(data.table(x = c(-6, -5, 3, 9))) - - learner = lrn("regr.lfbo", learner_classif = lrn("classif.ranger"), lfbo.direction = "minimize") - - surrogate = srlrn(learner, archive = instance$archive) - - acq_function = acqf("lfbo", surrogate = surrogate) - - acq_function$surrogate$update() - acq_function$update() - acq_function$eval_dt(data.table(x = c(-1, 0, 1))) -} -} -} -\references{ -\itemize{ -\item Song, Jiaming, Yu, Lantao, Neiswanger, Willie, Ermon, Stefano (2022). -\dQuote{A General Recipe for Likelihood-Free Bayesian Optimization.} -In Chaudhuri K, Jegelka S, Song L, Szepesvari C, Niu G, Sabato S (eds.), \emph{Proceedings of the 39th International Conference on Machine Learning}, volume 162, 20384--20404. -} -} -\seealso{ -Other Acquisition Function: -\code{\link{AcqFunction}}, -\code{\link{mlr_acqfunctions}}, -\code{\link{mlr_acqfunctions_aei}}, -\code{\link{mlr_acqfunctions_cb}}, -\code{\link{mlr_acqfunctions_ehvi}}, -\code{\link{mlr_acqfunctions_ehvigh}}, -\code{\link{mlr_acqfunctions_ei}}, -\code{\link{mlr_acqfunctions_eips}}, -\code{\link{mlr_acqfunctions_mean}}, -\code{\link{mlr_acqfunctions_pi}}, -\code{\link{mlr_acqfunctions_sd}}, -\code{\link{mlr_acqfunctions_smsego}}, -\code{\link{mlr_acqfunctions_ttei}} -} -\concept{Acquisition Function} -\section{Super classes}{ -\code{\link[bbotk:Objective]{bbotk::Objective}} -> \code{\link[mlr3mbo:AcqFunction]{mlr3mbo::AcqFunction}} -> \code{AcqFunctionLFBO} -} -\section{Methods}{ -\subsection{Public methods}{ -\itemize{ -\item \href{#method-AcqFunctionLFBO-new}{\code{AcqFunctionLFBO$new()}} -\item \href{#method-AcqFunctionLFBO-clone}{\code{AcqFunctionLFBO$clone()}} -} -} -\if{html}{\out{ -
Inherited methods - -
-}} -\if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-AcqFunctionLFBO-new}{}}} -\subsection{Method \code{new()}}{ -Creates a new instance of this \link[R6:R6Class]{R6} class. -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{AcqFunctionLFBO$new(surrogate = NULL)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{surrogate}}{(\code{NULL} | \link{SurrogateLearner}).} -} -\if{html}{\out{
}} -} -} -\if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-AcqFunctionLFBO-clone}{}}} -\subsection{Method \code{clone()}}{ -The objects of this class are cloneable with this method. -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{AcqFunctionLFBO$clone(deep = FALSE)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{deep}}{Whether to make a deep clone.} -} -\if{html}{\out{
}} -} -} -} diff --git a/man/mlr_acqfunctions_log_ei.Rd b/man/mlr_acqfunctions_log_ei.Rd new file mode 100644 index 00000000..f76c215a --- /dev/null +++ b/man/mlr_acqfunctions_log_ei.Rd @@ -0,0 +1,111 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/AcqFunctionLogEI.R +\name{mlr_acqfunctions_log_ei} +\alias{mlr_acqfunctions_log_ei} +\alias{AcqFunctionLogEI} +\title{Acquisition Function Log Expected Improvement} +\description{ +Expected Improvement assuming that the target variable is to be minimized and has been modeled on log scale. +} +\section{Dictionary}{ + +This \link{AcqFunction} can be instantiated via the \link[mlr3misc:Dictionary]{dictionary} +\link{mlr_acqfunctions} or with the associated sugar function \code{\link[=acqf]{acqf()}}: + +\if{html}{\out{
}}\preformatted{mlr_acqfunctions$get("log_ei") +acqf("log_ei") +}\if{html}{\out{
}} +} + +\seealso{ +Other Acquisition Function: +\code{\link{AcqFunction}}, +\code{\link{mlr_acqfunctions}}, +\code{\link{mlr_acqfunctions_aei}}, +\code{\link{mlr_acqfunctions_cb}}, +\code{\link{mlr_acqfunctions_ehvi}}, +\code{\link{mlr_acqfunctions_ehvigh}}, +\code{\link{mlr_acqfunctions_ei}}, +\code{\link{mlr_acqfunctions_eips}}, +\code{\link{mlr_acqfunctions_mean}}, +\code{\link{mlr_acqfunctions_pi}}, +\code{\link{mlr_acqfunctions_sd}}, +\code{\link{mlr_acqfunctions_smsego}} +} +\concept{Acquisition Function} +\section{Super classes}{ +\code{\link[bbotk:Objective]{bbotk::Objective}} -> \code{\link[mlr3mbo:AcqFunction]{mlr3mbo::AcqFunction}} -> \code{AcqFunctionLogEI} +} +\section{Public fields}{ +\if{html}{\out{
}} +\describe{ +\item{\code{y_best}}{(\code{numeric(1)})\cr +Best objective function value observed so far. +On log scale.} +} +\if{html}{\out{
}} +} +\section{Methods}{ +\subsection{Public methods}{ +\itemize{ +\item \href{#method-AcqFunctionLogEI-new}{\code{AcqFunctionLogEI$new()}} +\item \href{#method-AcqFunctionLogEI-update}{\code{AcqFunctionLogEI$update()}} +\item \href{#method-AcqFunctionLogEI-clone}{\code{AcqFunctionLogEI$clone()}} +} +} +\if{html}{\out{ +
Inherited methods + +
+}} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqFunctionLogEI-new}{}}} +\subsection{Method \code{new()}}{ +Creates a new instance of this \link[R6:R6Class]{R6} class. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{AcqFunctionLogEI$new(surrogate = NULL)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{surrogate}}{(\code{NULL} | \link{SurrogateLearner}).} +} +\if{html}{\out{
}} +} +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqFunctionLogEI-update}{}}} +\subsection{Method \code{update()}}{ +Updates acquisition function and sets \code{y_best}. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{AcqFunctionLogEI$update()}\if{html}{\out{
}} +} + +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqFunctionLogEI-clone}{}}} +\subsection{Method \code{clone()}}{ +The objects of this class are cloneable with this method. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{AcqFunctionLogEI$clone(deep = FALSE)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{deep}}{Whether to make a deep clone.} +} +\if{html}{\out{
}} +} +} +} diff --git a/man/mlr_acqfunctions_mean.Rd b/man/mlr_acqfunctions_mean.Rd index 3757e5be..81931053 100644 --- a/man/mlr_acqfunctions_mean.Rd +++ b/man/mlr_acqfunctions_mean.Rd @@ -60,11 +60,10 @@ Other Acquisition Function: \code{\link{mlr_acqfunctions_ehvigh}}, \code{\link{mlr_acqfunctions_ei}}, \code{\link{mlr_acqfunctions_eips}}, -\code{\link{mlr_acqfunctions_lfbo}}, +\code{\link{mlr_acqfunctions_log_ei}}, \code{\link{mlr_acqfunctions_pi}}, \code{\link{mlr_acqfunctions_sd}}, -\code{\link{mlr_acqfunctions_smsego}}, -\code{\link{mlr_acqfunctions_ttei}} +\code{\link{mlr_acqfunctions_smsego}} } \concept{Acquisition Function} \section{Super classes}{ diff --git a/man/mlr_acqfunctions_pi.Rd b/man/mlr_acqfunctions_pi.Rd index d0b4fa4f..172fd8f5 100644 --- a/man/mlr_acqfunctions_pi.Rd +++ b/man/mlr_acqfunctions_pi.Rd @@ -67,11 +67,10 @@ Other Acquisition Function: \code{\link{mlr_acqfunctions_ehvigh}}, \code{\link{mlr_acqfunctions_ei}}, \code{\link{mlr_acqfunctions_eips}}, -\code{\link{mlr_acqfunctions_lfbo}}, +\code{\link{mlr_acqfunctions_log_ei}}, \code{\link{mlr_acqfunctions_mean}}, \code{\link{mlr_acqfunctions_sd}}, -\code{\link{mlr_acqfunctions_smsego}}, -\code{\link{mlr_acqfunctions_ttei}} +\code{\link{mlr_acqfunctions_smsego}} } \concept{Acquisition Function} \section{Super classes}{ diff --git a/man/mlr_acqfunctions_sd.Rd b/man/mlr_acqfunctions_sd.Rd index eacb329d..255de8d0 100644 --- a/man/mlr_acqfunctions_sd.Rd +++ b/man/mlr_acqfunctions_sd.Rd @@ -60,11 +60,10 @@ Other Acquisition Function: \code{\link{mlr_acqfunctions_ehvigh}}, \code{\link{mlr_acqfunctions_ei}}, \code{\link{mlr_acqfunctions_eips}}, -\code{\link{mlr_acqfunctions_lfbo}}, +\code{\link{mlr_acqfunctions_log_ei}}, \code{\link{mlr_acqfunctions_mean}}, \code{\link{mlr_acqfunctions_pi}}, -\code{\link{mlr_acqfunctions_smsego}}, -\code{\link{mlr_acqfunctions_ttei}} +\code{\link{mlr_acqfunctions_smsego}} } \concept{Acquisition Function} \section{Super classes}{ diff --git a/man/mlr_acqfunctions_smsego.Rd b/man/mlr_acqfunctions_smsego.Rd index 85538898..2d7b1f37 100644 --- a/man/mlr_acqfunctions_smsego.Rd +++ b/man/mlr_acqfunctions_smsego.Rd @@ -77,11 +77,10 @@ Other Acquisition Function: \code{\link{mlr_acqfunctions_ehvigh}}, \code{\link{mlr_acqfunctions_ei}}, \code{\link{mlr_acqfunctions_eips}}, -\code{\link{mlr_acqfunctions_lfbo}}, +\code{\link{mlr_acqfunctions_log_ei}}, \code{\link{mlr_acqfunctions_mean}}, \code{\link{mlr_acqfunctions_pi}}, -\code{\link{mlr_acqfunctions_sd}}, -\code{\link{mlr_acqfunctions_ttei}} +\code{\link{mlr_acqfunctions_sd}} } \concept{Acquisition Function} \section{Super classes}{ diff --git a/man/mlr_acqfunctions_ttei.Rd b/man/mlr_acqfunctions_ttei.Rd deleted file mode 100644 index f135d256..00000000 --- a/man/mlr_acqfunctions_ttei.Rd +++ /dev/null @@ -1,189 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/AcqFunctionTTEI.R -\name{mlr_acqfunctions_ttei} -\alias{mlr_acqfunctions_ttei} -\alias{AcqFunctionTTEI} -\title{Acquisition Function Top-Two Expected Improvement} -\description{ -Top-Two Expected Improvement. - -With probability \code{beta} simply the Expected Improvement. -With probability \code{1 - beta} the Expected Improvement over the current Expected Improvement. -This requires to first obtain the current argmax of the Expected Improvement to then be able to -formulate the Expected Improvement with respect to the posterior prediction of that argmax. -} -\section{Dictionary}{ - -This \link{AcqFunction} can be instantiated via the \link[mlr3misc:Dictionary]{dictionary} -\link{mlr_acqfunctions} or with the associated sugar function \code{\link[=acqf]{acqf()}}: - -\if{html}{\out{
}}\preformatted{mlr_acqfunctions$get("ttei") -acqf("ttei") -}\if{html}{\out{
}} -} - -\examples{ -if (requireNamespace("mlr3learners") & - requireNamespace("DiceKriging") & - requireNamespace("rgenoud")) { - library(bbotk) - library(paradox) - library(mlr3learners) - library(data.table) - - fun = function(xs) { - list(y = xs$x ^ 2) - } - domain = ps(x = p_dbl(lower = -10, upper = 10)) - codomain = ps(y = p_dbl(tags = "minimize")) - objective = ObjectiveRFun$new(fun = fun, domain = domain, codomain = codomain) - - instance = OptimInstanceSingleCrit$new( - objective = objective, - terminator = trm("evals", n_evals = 5)) - - instance$eval_batch(data.table(x = c(-6, -5, 3, 9))) - - learner = lrn("regr.km", - covtype = "matern3_2", - optim.method = "gen", - nugget.stability = 10^-8, - control = list(trace = FALSE)) - - surrogate = srlrn(learner, archive = instance$archive) - - acq_function = acqf("ttei", - surrogate = surrogate, - toplvl_acq_optimizer = acqo(opt("random_search"), terminator = trm("evals", n_evals = 100))) - - acq_function$surrogate$update() - acq_function$update() - acq_function$eval_dt(data.table(x = c(-1, 0, 1))) -} -} -\references{ -\itemize{ -\item Qin, Chao, Klabjan, Diego, Russo, Daniel (2017). -\dQuote{Improving the Expected Improvement Algorithm.} -\emph{Advances in Neural Information Processing Systems}, \bold{30}. -} -} -\seealso{ -Other Acquisition Function: -\code{\link{AcqFunction}}, -\code{\link{mlr_acqfunctions}}, -\code{\link{mlr_acqfunctions_aei}}, -\code{\link{mlr_acqfunctions_cb}}, -\code{\link{mlr_acqfunctions_ehvi}}, -\code{\link{mlr_acqfunctions_ehvigh}}, -\code{\link{mlr_acqfunctions_ei}}, -\code{\link{mlr_acqfunctions_eips}}, -\code{\link{mlr_acqfunctions_lfbo}}, -\code{\link{mlr_acqfunctions_mean}}, -\code{\link{mlr_acqfunctions_pi}}, -\code{\link{mlr_acqfunctions_sd}}, -\code{\link{mlr_acqfunctions_smsego}} -} -\concept{Acquisition Function} -\section{Super classes}{ -\code{\link[bbotk:Objective]{bbotk::Objective}} -> \code{\link[mlr3mbo:AcqFunction]{mlr3mbo::AcqFunction}} -> \code{AcqFunctionTTEI} -} -\section{Public fields}{ -\if{html}{\out{
}} -\describe{ -\item{\code{y_best}}{(\code{numeric(1)})\cr -Best objective function value observed so far.} - -\item{\code{ei_argmax_mean_se}}{(\code{\link[data.table:data.table]{data.table::data.table()}})\cr -data.table::data.table() containing one row with the surrogate mean and standard error -prediction for the current argmax of the Expected Improvement.} - -\item{\code{is_ei}}{(\code{logical(1)})\cr -Whether in the next iteration simply the Expected Improvement is to be optimized or the -Expected Improvement over the Expected Improvement.} -} -\if{html}{\out{
}} -} -\section{Methods}{ -\subsection{Public methods}{ -\itemize{ -\item \href{#method-AcqFunctionTTEI-new}{\code{AcqFunctionTTEI$new()}} -\item \href{#method-AcqFunctionTTEI-update}{\code{AcqFunctionTTEI$update()}} -\item \href{#method-AcqFunctionTTEI-clone}{\code{AcqFunctionTTEI$clone()}} -} -} -\if{html}{\out{ -
Inherited methods - -
-}} -\if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-AcqFunctionTTEI-new}{}}} -\subsection{Method \code{new()}}{ -Creates a new instance of this \link[R6:R6Class]{R6} class. -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{AcqFunctionTTEI$new(surrogate = NULL, beta = 0.5, toplvl_acq_optimizer = NULL)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{surrogate}}{(\code{NULL} | \link{SurrogateLearner}).} - -\item{\code{beta}}{(\code{numeric(1)})\cr -With probability \code{beta} the toplevel Expected Improvement acquisition function is to be -optimized and with \code{1 - beta} the Expected Improvement over the Expected Improvement. -Default is \code{0.5}.} - -\item{\code{toplvl_acq_optimizer}}{(\code{NULL} | \link{AcqOptimizer})\cr -Acquisition function optimizer for the toplevel Expected Improvement acquisition function -used to find the current argmax. -If \code{NULL} will be initialized to -\code{acqo(opt("random_search", batch_size = 1000), terminator = trm("evals", n_evals = 10000))}.} -} -\if{html}{\out{
}} -} -} -\if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-AcqFunctionTTEI-update}{}}} -\subsection{Method \code{update()}}{ -Updates acquisition function and performs the following steps: -\itemize{ -\item sets \code{y_best} -\item updates the internal toplevel Expected Improvement acquisition function -\item samples whether simply the Expected Improvement is to be optimized or the Expected -Improvement over the Expected Improvement (depending on \code{beta}) and sets \code{is_ei} accordingly -\item if \code{is_ei = FALSE}, proceeds to optimize the toplevel Expected Improvement acquisition function to -find the current argmax and sets \code{ei_argmax_mean_se} -} -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{AcqFunctionTTEI$update()}\if{html}{\out{
}} -} - -} -\if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-AcqFunctionTTEI-clone}{}}} -\subsection{Method \code{clone()}}{ -The objects of this class are cloneable with this method. -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{AcqFunctionTTEI$clone(deep = FALSE)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{deep}}{Whether to make a deep clone.} -} -\if{html}{\out{
}} -} -} -} diff --git a/man/mlr_learners_regr.ranger.Rd b/man/mlr_learners_regr.ranger.Rd new file mode 100644 index 00000000..34d569cf --- /dev/null +++ b/man/mlr_learners_regr.ranger.Rd @@ -0,0 +1,93 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/LearnerRegrRangerCustom.R +\name{mlr_learners_regr.ranger} +\alias{mlr_learners_regr.ranger} +\alias{LearnerRegrRangerCustom} +\title{Custom Ranger Regression Learner} +\description{ +Random regression forest. +Calls \code{\link[ranger:ranger]{ranger::ranger()}} from package \CRANpkg{ranger}. +... +} +\section{Super classes}{ +\code{\link[mlr3:Learner]{mlr3::Learner}} -> \code{\link[mlr3:LearnerRegr]{mlr3::LearnerRegr}} -> \code{LearnerRegrRangerCustom} +} +\section{Methods}{ +\subsection{Public methods}{ +\itemize{ +\item \href{#method-LearnerRegrRangerCustom-new}{\code{LearnerRegrRangerCustom$new()}} +\item \href{#method-LearnerRegrRangerCustom-importance}{\code{LearnerRegrRangerCustom$importance()}} +\item \href{#method-LearnerRegrRangerCustom-oob_error}{\code{LearnerRegrRangerCustom$oob_error()}} +\item \href{#method-LearnerRegrRangerCustom-clone}{\code{LearnerRegrRangerCustom$clone()}} +} +} +\if{html}{\out{ +
Inherited methods + +
+}} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-LearnerRegrRangerCustom-new}{}}} +\subsection{Method \code{new()}}{ +Creates a new instance of this \link[R6:R6Class]{R6} class. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{LearnerRegrRangerCustom$new()}\if{html}{\out{
}} +} + +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-LearnerRegrRangerCustom-importance}{}}} +\subsection{Method \code{importance()}}{ +The importance scores are extracted from the model slot \code{variable.importance}. +Parameter \code{importance.mode} must be set to \code{"impurity"}, \code{"impurity_corrected"}, or +\code{"permutation"} +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{LearnerRegrRangerCustom$importance()}\if{html}{\out{
}} +} + +\subsection{Returns}{ +Named \code{numeric()}. +} +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-LearnerRegrRangerCustom-oob_error}{}}} +\subsection{Method \code{oob_error()}}{ +The out-of-bag error, extracted from model slot \code{prediction.error}. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{LearnerRegrRangerCustom$oob_error()}\if{html}{\out{
}} +} + +\subsection{Returns}{ +\code{numeric(1)}. +} +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-LearnerRegrRangerCustom-clone}{}}} +\subsection{Method \code{clone()}}{ +The objects of this class are cloneable with this method. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{LearnerRegrRangerCustom$clone(deep = FALSE)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{deep}}{Whether to make a deep clone.} +} +\if{html}{\out{
}} +} +} +} diff --git a/man/mlr_learners_regr_lfbo.Rd b/man/mlr_learners_regr_lfbo.Rd deleted file mode 100644 index e261cb66..00000000 --- a/man/mlr_learners_regr_lfbo.Rd +++ /dev/null @@ -1,194 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/LearnerRegrLFBO.R -\name{mlr_learners_regr_lfbo} -\alias{mlr_learners_regr_lfbo} -\alias{LearnerRegrLFBO} -\title{Likelihood Free Bayesian Optimization Learner} -\description{ -Wraps a classification learner to be used as a regression learner for -Likelihood-free Bayesian Optimization (LFBO). -} -\details{ -The regression data is internally converted to a weighted classification task. -The new objective is a weighted version of a prediction problem, where the goal is -to predict whether a target value is smaller than a gamma quantile of the target -distribution (assuming minimization). -Note that the optimization direction must be specified explicitly! -If minimization is required, set the value of the \code{lfbo.direction} parameter to \code{"minimize"}. -If maximization is required, set the value of the \code{lfbo.direction} parameter to \code{"maximize"}. - -To specify the weighting type, set the value of the \code{lfbo.wt} parameter (\code{"ei"} or \code{"pi"}). -To specify the \code{gamma} quantile of the target distribution set the value of the \code{lfbo.gamma} parameter. - -This learner should only be used for Bayesian Optimization in combination with an \link{AcqFunctionLFBO}. - -The \verb{$param_set} consists of a \link[paradox:ParamSetCollection]{paradox::ParamSetCollection} of the \link[paradox:ParamSet]{paradox::ParamSet} of the \verb{$learner_classif} and -an \code{"lfbo"} \link[paradox:ParamSet]{paradox::ParamSet} including the parameters \code{"lfbo.direction"}, \code{"lfbo.wt"}, and \code{"lfbo.gamma"}. -} -\examples{ -\donttest{ -if (requireNamespace("mlr3learners") & - requireNamespace("ranger")) { - - library(bbotk) - library(paradox) - library(mlr3learners) - - fun = function(xs) { - list(y = xs$x ^ 2) - } - domain = ps(x = p_dbl(lower = -10, upper = 10)) - codomain = ps(y = p_dbl(tags = "minimize")) - objective = ObjectiveRFun$new(fun = fun, domain = domain, codomain = codomain) - - instance = OptimInstanceSingleCrit$new( - objective = objective, - terminator = trm("evals", n_evals = 5)) - - surrogate = srlrn(lrn("regr.lfbo", lrn("classif.ranger"), lfbo.direction = "minimize")) - - acq_function = acqf("lfbo") - - acq_optimizer = acqo( - optimizer = opt("random_search", batch_size = 100), - terminator = trm("evals", n_evals = 100)) - - optimizer = opt("mbo", - loop_function = bayesopt_ego, - surrogate = surrogate, - acq_function = acq_function, - acq_optimizer = acq_optimizer) - - optimizer$optimize(instance) -} -} -} -\references{ -\itemize{ -\item Song, Jiaming, Yu, Lantao, Neiswanger, Willie, Ermon, Stefano (2022). -\dQuote{A General Recipe for Likelihood-Free Bayesian Optimization.} -In Chaudhuri K, Jegelka S, Song L, Szepesvari C, Niu G, Sabato S (eds.), \emph{Proceedings of the 39th International Conference on Machine Learning}, volume 162, 20384--20404. -} -} -\section{Super classes}{ -\code{\link[mlr3:Learner]{mlr3::Learner}} -> \code{\link[mlr3:LearnerRegr]{mlr3::LearnerRegr}} -> \code{LearnerRegrLFBO} -} -\section{Public fields}{ -\if{html}{\out{
}} -\describe{ -\item{\code{learner_classif}}{(\link[mlr3:LearnerClassif]{mlr3::LearnerClassif})\cr -Classification learner to be used for LFBO.} -} -\if{html}{\out{
}} -} -\section{Methods}{ -\subsection{Public methods}{ -\itemize{ -\item \href{#method-LearnerRegrLFBO-new}{\code{LearnerRegrLFBO$new()}} -\item \href{#method-LearnerRegrLFBO-train}{\code{LearnerRegrLFBO$train()}} -\item \href{#method-LearnerRegrLFBO-predict}{\code{LearnerRegrLFBO$predict()}} -\item \href{#method-LearnerRegrLFBO-clone}{\code{LearnerRegrLFBO$clone()}} -} -} -\if{html}{\out{ -
Inherited methods - -
-}} -\if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-LearnerRegrLFBO-new}{}}} -\subsection{Method \code{new()}}{ -Creates a new instance of this \link[R6:R6Class]{R6} class. -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{LearnerRegrLFBO$new(learner_classif)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{learner_classif}}{(\link[mlr3:LearnerClassif]{mlr3::LearnerClassif})\cr -Classifcation learner to be used for LFBO. -Requires \code{predict_type = "prob"} which will be set automatically during construction. -Also requires the learner to be able to handle case weights, see \code{learner$properties}.} -} -\if{html}{\out{
}} -} -} -\if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-LearnerRegrLFBO-train}{}}} -\subsection{Method \code{train()}}{ -Train the learner on a set of observations of the provided \code{task}. -Mutates the learner by reference, i.e. stores the model alongside other information in field \verb{$state}. -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{LearnerRegrLFBO$train(task, row_ids = NULL)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{task}}{(\link{TaskRegr}).} - -\item{\code{row_ids}}{(\code{integer()})\cr -Vector of training indices as subset of \code{task$row_ids}. -For a simple split into training and test set, see \code{\link[mlr3:partition]{mlr3::partition()}}.} -} -\if{html}{\out{
}} -} -\subsection{Returns}{ -Returns the object itself, but modified \strong{by reference}. -You need to explicitly \verb{$clone()} the object beforehand if you want to keeps -the object in its previous state. -} -} -\if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-LearnerRegrLFBO-predict}{}}} -\subsection{Method \code{predict()}}{ -Uses the information stored during \verb{$train()} in \verb{$learner$classif$state} to create a new \link[mlr3:PredictionRegr]{mlr3::PredictionRegr} -for a set of observations of the provided \code{task}. -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{LearnerRegrLFBO$predict(task, row_ids = NULL)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{task}}{(\link[mlr3:TaskRegr]{mlr3::TaskRegr}).} - -\item{\code{row_ids}}{(\code{integer()})\cr -Vector of test indices as subset of \code{task$row_ids}. -For a simple split into training and test set, see \code{\link[mlr3:partition]{mlr3::partition()}}.} -} -\if{html}{\out{
}} -} -\subsection{Returns}{ -\link{PredictionRegr}. -} -} -\if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-LearnerRegrLFBO-clone}{}}} -\subsection{Method \code{clone()}}{ -The objects of this class are cloneable with this method. -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{LearnerRegrLFBO$clone(deep = FALSE)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{deep}}{Whether to make a deep clone.} -} -\if{html}{\out{
}} -} -} -} diff --git a/man/mlr_loop_functions_ego_log.Rd b/man/mlr_loop_functions_ego_log.Rd index 4c927ddd..728db8db 100644 --- a/man/mlr_loop_functions_ego_log.Rd +++ b/man/mlr_loop_functions_ego_log.Rd @@ -7,10 +7,10 @@ \usage{ bayesopt_ego_log( instance, - init_design_size = NULL, surrogate, acq_function, acq_optimizer, + init_design_size = NULL, random_interleave_iter = 0L, epsilon = 0.001 ) @@ -19,12 +19,6 @@ bayesopt_ego_log( \item{instance}{(\link[bbotk:OptimInstanceSingleCrit]{bbotk::OptimInstanceSingleCrit})\cr The \link[bbotk:OptimInstanceSingleCrit]{bbotk::OptimInstanceSingleCrit} to be optimized.} -\item{init_design_size}{(\code{NULL} | \code{integer(1)})\cr -Size of the initial design. -If \code{NULL} and the \link[bbotk:Archive]{bbotk::Archive} contains no evaluations, \code{4 * d} is used with \code{d} being the -dimensionality of the search space. -Points are drawn uniformly at random.} - \item{surrogate}{(\link{Surrogate})\cr \link{Surrogate} to be used as a surrogate. Typically a \link{SurrogateLearner}.} @@ -35,6 +29,12 @@ Typically a \link{SurrogateLearner}.} \item{acq_optimizer}{(\link{AcqOptimizer})\cr \link{AcqOptimizer} to be used as acquisition function optimizer.} +\item{init_design_size}{(\code{NULL} | \code{integer(1)})\cr +Size of the initial design. +If \code{NULL} and the \link[bbotk:Archive]{bbotk::Archive} contains no evaluations, \code{4 * d} is used with \code{d} being the +dimensionality of the search space. +Points are generated via a Sobol sequence.} + \item{random_interleave_iter}{(\code{integer(1)})\cr Every \code{random_interleave_iter} iteration (starting after the initial design), a point is sampled uniformly at random and evaluated (instead of a model based proposal). @@ -97,7 +97,7 @@ if (requireNamespace("mlr3learners") & acq_function = acqf("ei") acq_optimizer = acqo( - optimizer = opt("random_search"), + optimizer = opt("random_search", batch_size = 100), terminator = trm("evals", n_evals = 100)) optimizer = opt("mbo", diff --git a/man/mlr_optimizers_chain.Rd b/man/mlr_optimizers_chain.Rd new file mode 100644 index 00000000..839d8a26 --- /dev/null +++ b/man/mlr_optimizers_chain.Rd @@ -0,0 +1,94 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/OptimizerChain.R +\name{mlr_optimizers_chain} +\alias{mlr_optimizers_chain} +\alias{OptimizerChain} +\title{Run Optimizers Sequentially} +\description{ +\code{OptimizerChain} allows to run different \link{Optimizer}s sequentially. + +For each \link{Optimizer} an (optional) additional \link{Terminator} can be specified +during construction. +While the initial \link{Terminator} guards the optimization process as a whole, +the additional \link{Terminator}s guard each individual \link{Optimizer}. + +The optimization then works as follows: +The first \link{Optimizer} is run on the \link{OptimInstance} using a \link{TerminatorCombo} +of the original \link{Terminator} of the \link{OptimInstance} and the (optional) +additional \link{Terminator} as passed during construction. +Once this \link{TerminatorCombo} indicates termination (usually via the +additional \link{Terminator}), the second \link{Optimizer} (and so on) is run unless +the original \link{Terminator} of the \link{OptimInstance} indicates termination. + +\link{OptimizerChain} can also be used for random restarts of the same +\link{Optimizer} (if applicable) by setting the \link{Terminator} of the \link{OptimInstance} to +\link{TerminatorNone} and setting identical additional \link{Terminator}s during +construction. +} +\section{Parameters}{ + +Parameters are inherited from the individual \link{Optimizer}s and collected as a +\link[paradox:ParamSetCollection]{paradox::ParamSetCollection} (with \code{set_id}s potentially postfixed via \verb{_1}, \verb{_2}, +..., if the same \link{Optimizer}s are used multiple times). +} + +\section{Super class}{ +\code{\link[bbotk:Optimizer]{bbotk::Optimizer}} -> \code{OptimizerChain} +} +\section{Methods}{ +\subsection{Public methods}{ +\itemize{ +\item \href{#method-OptimizerChain-new}{\code{OptimizerChain$new()}} +\item \href{#method-OptimizerChain-clone}{\code{OptimizerChain$clone()}} +} +} +\if{html}{\out{ +
Inherited methods + +
+}} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-OptimizerChain-new}{}}} +\subsection{Method \code{new()}}{ +Creates a new instance of this \link[R6:R6Class]{R6} class. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{OptimizerChain$new( + optimizers, + terminators = rep(list(NULL), length(optimizers)) +)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{optimizers}}{(list of \link{Optimizer}s).} + +\item{\code{terminators}}{(list of \link{Terminator}s | NULL).} +} +\if{html}{\out{
}} +} +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-OptimizerChain-clone}{}}} +\subsection{Method \code{clone()}}{ +The objects of this class are cloneable with this method. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{OptimizerChain$clone(deep = FALSE)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{deep}}{Whether to make a deep clone.} +} +\if{html}{\out{
}} +} +} +} diff --git a/man/mlr_optimizers_local_search.Rd b/man/mlr_optimizers_local_search.Rd new file mode 100644 index 00000000..731374c3 --- /dev/null +++ b/man/mlr_optimizers_local_search.Rd @@ -0,0 +1,88 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/OptimizerLocalSearch.R +\name{mlr_optimizers_local_search} +\alias{mlr_optimizers_local_search} +\alias{OptimizerLocalSearch} +\title{Optimization via Local Search} +\description{ +\code{OptimizerLocalSearch} class that implements a simple Local Search. +Local Search starts by determining the \code{mu} initial best points present in the \link{Archive} of the +\link{OptimInstance}. If fewer points than \code{mu} are present, additional points sampled uniformly at +random are evaluated. + +In each iteration, for each of the \code{mu} initial best points, \code{n_points} neighbors are generated +by local mutation. Local mutation generates a neighbor by sampling a single parameter that is to +be mutated and then proceeds as follows: Double parameters (\link[paradox:ParamDbl]{paradox::ParamDbl}) are mutated via +Gaussian mutation (with a prior standardization to \verb{[0, 1]} and retransformation after mutation). +Integer parameters (\link[paradox:ParamInt]{paradox::ParamInt}) undergo the same mutation but are rounded to the closest +integer after mutation. Categorical parameters (\link[paradox:ParamFct]{paradox::ParamFct} and \link[paradox:ParamLgl]{paradox::ParamLgl}) are +mutated via uniform mutation. Note that parameters that are conditioned on (i.e., they are +parents of a \link[paradox:Condition]{paradox::Condition}, see the dependencies of the search space) are not mutated. +} +\section{Parameters}{ + +\describe{ +\item{\code{mu}}{\code{integer(1)}\cr +Size of the initial best points which are used as a starting points for the Local Search. +Default is \code{10}. +} +\item{\code{n_points}}{\code{integer(1)}\cr +Number of neighboring points to generate for each of the \code{mu} best starting points in each +iteration. +Default is \code{100}. +} +\item{\code{sigma}}{\code{numeric(1)}\cr +Standard deviation used for mutation of numeric parameters. +Default is \code{0.1}. +} +} +} + +\section{Super class}{ +\code{\link[bbotk:Optimizer]{bbotk::Optimizer}} -> \code{OptimizerLocalSearch} +} +\section{Methods}{ +\subsection{Public methods}{ +\itemize{ +\item \href{#method-OptimizerLocalSearch-new}{\code{OptimizerLocalSearch$new()}} +\item \href{#method-OptimizerLocalSearch-clone}{\code{OptimizerLocalSearch$clone()}} +} +} +\if{html}{\out{ +
Inherited methods + +
+}} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-OptimizerLocalSearch-new}{}}} +\subsection{Method \code{new()}}{ +Creates a new instance of this \link[R6:R6Class]{R6} class. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{OptimizerLocalSearch$new()}\if{html}{\out{
}} +} + +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-OptimizerLocalSearch-clone}{}}} +\subsection{Method \code{clone()}}{ +The objects of this class are cloneable with this method. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{OptimizerLocalSearch$clone(deep = FALSE)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{deep}}{Whether to make a deep clone.} +} +\if{html}{\out{
}} +} +} +} diff --git a/tests/testthat/helper.R b/tests/testthat/helper.R index fb330231..dc4a225e 100644 --- a/tests/testthat/helper.R +++ b/tests/testthat/helper.R @@ -198,7 +198,7 @@ expect_acqfunction = function(acqf) { sortnames = function(x) { if (!is.null(names(x))) { - x <- x[order(names(x), decreasing = TRUE)] + x = x[order(names(x), decreasing = TRUE)] } x } From 171dccec324108fb264704e150dfc5c8f14695f0 Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Mon, 1 Apr 2024 19:36:06 +0200 Subject: [PATCH 062/126] .. --- DESCRIPTION | 2 +- R/LearnerRegrRangerCustom.R | 2 ++ R/OptimizerChain.R | 3 +++ R/OptimizerLocalSearch.R | 3 +++ 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 47372dea..4b27e129 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -89,9 +89,9 @@ Collate: 'AcqFunctionSmsEgo.R' 'AcqOptimizer.R' 'LearnerRegrRangerCustom.R' + 'aaa.R' 'OptimizerChain.R' 'OptimizerLocalSearch.R' - 'aaa.R' 'OptimizerMbo.R' 'mlr_result_assigners.R' 'ResultAssigner.R' diff --git a/R/LearnerRegrRangerCustom.R b/R/LearnerRegrRangerCustom.R index 5da03bb2..b0a51d6a 100644 --- a/R/LearnerRegrRangerCustom.R +++ b/R/LearnerRegrRangerCustom.R @@ -143,3 +143,5 @@ default_values.LearnerRegrRangerCustom = function(x, search_space, task, ...) { defaults = insert_named(default_values(x$param_set), special_defaults) defaults[search_space$ids()] } + +# FIXME: add to dictionary diff --git a/R/OptimizerChain.R b/R/OptimizerChain.R index a25a08da..393de1a0 100644 --- a/R/OptimizerChain.R +++ b/R/OptimizerChain.R @@ -112,3 +112,6 @@ OptimizerChain = R6Class("OptimizerChain", } ) ) + +#' @include aaa.R +optimizers[["chain"]] = OptimizerChain diff --git a/R/OptimizerLocalSearch.R b/R/OptimizerLocalSearch.R index 779506d9..a8072442 100644 --- a/R/OptimizerLocalSearch.R +++ b/R/OptimizerLocalSearch.R @@ -138,3 +138,6 @@ mutate = function(value, param, sigma) { } value } + +#' @include aaa.R +optimizers[["local_search"]] = OptimizerLocalSearch From 306138c3e22b3a7f20d5a586af8404e91da398ed Mon Sep 17 00:00:00 2001 From: be-marc Date: Wed, 29 May 2024 16:37:07 +0200 Subject: [PATCH 063/126] fix: remove reqired from parameters with default --- R/OptimizerLocalSearch.R | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/R/OptimizerLocalSearch.R b/R/OptimizerLocalSearch.R index a8072442..ef92ccff 100644 --- a/R/OptimizerLocalSearch.R +++ b/R/OptimizerLocalSearch.R @@ -43,9 +43,9 @@ OptimizerLocalSearch = R6Class("OptimizerLocalSearch", #' Creates a new instance of this [R6][R6::R6Class] class. initialize = function() { param_set = ps( - mu = p_int(lower = 1L, default = 10L, tags = "required"), - n_points = p_int(lower = 1L, default = 100L, tags = "required"), - sigma = p_dbl(lower = 0L, default = 0.1, tags = "required") + mu = p_int(lower = 1L, default = 10L), + n_points = p_int(lower = 1L, default = 100L), + sigma = p_dbl(lower = 0L, default = 0.1) ) param_set$values = list(mu = 10L, n_points = 100L, sigma = 0.1) From 9aa887f941d757d1b751d9d4de8bbd7fff37c295 Mon Sep 17 00:00:00 2001 From: be-marc Date: Thu, 27 Jun 2024 15:24:41 +0200 Subject: [PATCH 064/126] fix: bbotk update --- DESCRIPTION | 2 +- NAMESPACE | 4 ++-- R/LearnerRegrRangerCustom.R | 31 +++++++++++-------------- R/OptimizerLocalSearch.R | 6 ++--- R/aaa.R | 1 + R/bayesopt_ego_log.R | 25 ++++++++++---------- R/zzz.R | 11 ++++++++- man/mlr_acqfunctions_log_ei.Rd | 3 ++- man/mlr_learners_regr.ranger.Rd | 37 +++++++++++++++--------------- man/mlr_optimizers_chain.Rd | 1 - man/mlr_optimizers_local_search.Rd | 7 +++--- 11 files changed, 66 insertions(+), 62 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 4e843d98..77b80dc4 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -91,8 +91,8 @@ Collate: 'AcqFunctionSD.R' 'AcqFunctionSmsEgo.R' 'AcqOptimizer.R' - 'LearnerRegrRangerCustom.R' 'aaa.R' + 'LearnerRegrRangerCustom.R' 'OptimizerChain.R' 'OptimizerLocalSearch.R' 'OptimizerMbo.R' diff --git a/NAMESPACE b/NAMESPACE index a7ef0233..15171a89 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -3,7 +3,7 @@ S3method(as.data.table,DictionaryAcqFunction) S3method(as.data.table,DictionaryLoopFunction) S3method(as.data.table,DictionaryResultAssigner) -S3method(default_values,LearnerRegrRangerCustom) +S3method(default_values,LearnerRegrRangerMbo) S3method(print,loop_function) export(AcqFunction) export(AcqFunctionAEI) @@ -18,7 +18,7 @@ export(AcqFunctionPI) export(AcqFunctionSD) export(AcqFunctionSmsEgo) export(AcqOptimizer) -export(LearnerRegrRangerCustom) +export(LearnerRegrRangerMbo) export(OptimizerChain) export(OptimizerLocalSearch) export(OptimizerMbo) diff --git a/R/LearnerRegrRangerCustom.R b/R/LearnerRegrRangerCustom.R index b0a51d6a..5d5c0011 100644 --- a/R/LearnerRegrRangerCustom.R +++ b/R/LearnerRegrRangerCustom.R @@ -5,10 +5,9 @@ #' @description #' Random regression forest. #' Calls [ranger::ranger()] from package \CRANpkg{ranger}. -#' ... #' #' @export -LearnerRegrRangerCustom = R6Class("LearnerRegrRangerCustom", +LearnerRegrRangerMbo = R6Class("LearnerRegrRangerMbo", inherit = LearnerRegr, public = list( @@ -16,18 +15,17 @@ LearnerRegrRangerCustom = R6Class("LearnerRegrRangerCustom", #' Creates a new instance of this [R6][R6::R6Class] class. initialize = function() { ps = ps( - alpha = p_dbl(default = 0.5, tags = "train"), + alpha = p_dbl(default = 0.5, tags = "train", depends = quote(splitrule == "maxstat")), always.split.variables = p_uty(tags = "train"), holdout = p_lgl(default = FALSE, tags = "train"), importance = p_fct(c("none", "impurity", "impurity_corrected", "permutation"), tags = "train"), keep.inbag = p_lgl(default = FALSE, tags = "train"), max.depth = p_int(default = NULL, lower = 0L, special_vals = list(NULL), tags = "train"), min.node.size = p_int(1L, default = 5L, special_vals = list(NULL), tags = "train"), - min.prop = p_dbl(default = 0.1, tags = "train"), - minprop = p_dbl(default = 0.1, tags = "train"), + minprop = p_dbl(default = 0.1, tags = "train", depends = quote(splitrule == "maxstat")), mtry = p_int(lower = 1L, special_vals = list(NULL), tags = "train"), mtry.ratio = p_dbl(lower = 0, upper = 1, tags = "train"), - num.random.splits = p_int(1L, default = 1L, tags = "train"), + num.random.splits = p_int(1L, default = 1L, tags = "train", depends = quote(splitrule == "extratrees")), num.threads = p_int(1L, default = 1L, tags = c("train", "predict", "threads")), num.trees = p_int(1L, default = 500L, tags = c("train", "predict", "hotstart")), oob.error = p_lgl(default = TRUE, tags = "train"), @@ -38,7 +36,7 @@ LearnerRegrRangerCustom = R6Class("LearnerRegrRangerCustom", respect.unordered.factors = p_fct(c("ignore", "order", "partition"), default = "ignore", tags = "train"), sample.fraction = p_dbl(0L, 1L, tags = "train"), save.memory = p_lgl(default = FALSE, tags = "train"), - scale.permutation.importance = p_lgl(default = FALSE, tags = "train"), + scale.permutation.importance = p_lgl(default = FALSE, tags = "train", depends = quote(importance == "permutation")), se.method = p_fct(c("jack", "infjack", "simple"), default = "infjack", tags = "predict"), # FIXME: only works if predict_type == "se" seed = p_int(default = NULL, special_vals = list(NULL), tags = c("train", "predict")), split.select.weights = p_uty(default = NULL, tags = "train"), @@ -49,21 +47,15 @@ LearnerRegrRangerCustom = R6Class("LearnerRegrRangerCustom", ps$values = list(num.threads = 1L) - # deps - ps$add_dep("num.random.splits", "splitrule", CondEqual$new("extratrees")) - ps$add_dep("alpha", "splitrule", CondEqual$new("maxstat")) - ps$add_dep("minprop", "splitrule", CondEqual$new("maxstat")) - ps$add_dep("scale.permutation.importance", "importance", CondEqual$new("permutation")) - - super$initialize( - id = "regr.ranger_custom", + id = "regr.ranger_mbo", param_set = ps, predict_types = c("response", "se"), feature_types = c("logical", "integer", "numeric", "character", "factor", "ordered"), properties = c("weights", "importance", "oob_error", "hotstart_backward"), packages = c("mlr3learners", "ranger"), - man = "mlr3learners::mlr_learners_regr.ranger_custom" + label = "Custom MBO Random Forest", + man = "mlr3mbo::mlr_learners_regr.ranger_mbo" ) }, @@ -125,6 +117,7 @@ LearnerRegrRangerCustom = R6Class("LearnerRegrRangerCustom", list(response = prediction$predictions, se = prediction$se) } }, + .hotstart = function(task) { model = self$model model$num.trees = self$param_set$values$num.trees @@ -134,7 +127,7 @@ LearnerRegrRangerCustom = R6Class("LearnerRegrRangerCustom", ) #' @export -default_values.LearnerRegrRangerCustom = function(x, search_space, task, ...) { # nolint +default_values.LearnerRegrRangerMbo = function(x, search_space, task, ...) { # nolint special_defaults = list( mtry = floor(sqrt(length(task$feature_names))), mtry.ratio = floor(sqrt(length(task$feature_names))) / length(task$feature_names), @@ -144,4 +137,6 @@ default_values.LearnerRegrRangerCustom = function(x, search_space, task, ...) { defaults[search_space$ids()] } -# FIXME: add to dictionary +#' @include aaa.R +learners[["regr.ranger_mbo"]] = LearnerRegrRangerMbo + diff --git a/R/OptimizerLocalSearch.R b/R/OptimizerLocalSearch.R index ef92ccff..123ac77c 100644 --- a/R/OptimizerLocalSearch.R +++ b/R/OptimizerLocalSearch.R @@ -10,10 +10,10 @@ #' #' In each iteration, for each of the `mu` initial best points, `n_points` neighbors are generated #' by local mutation. Local mutation generates a neighbor by sampling a single parameter that is to -#' be mutated and then proceeds as follows: Double parameters ([paradox::ParamDbl]) are mutated via +#' be mutated and then proceeds as follows: Double parameters ([paradox::p_dbl]) are mutated via #' Gaussian mutation (with a prior standardization to `[0, 1]` and retransformation after mutation). -#' Integer parameters ([paradox::ParamInt]) undergo the same mutation but are rounded to the closest -#' integer after mutation. Categorical parameters ([paradox::ParamFct] and [paradox::ParamLgl]) are +#' Integer parameters ([paradox::p_int]) undergo the same mutation but are rounded to the closest +#' integer after mutation. Categorical parameters ([paradox::p_fct] and [paradox::p_lgl]) are #' mutated via uniform mutation. Note that parameters that are conditioned on (i.e., they are #' parents of a [paradox::Condition], see the dependencies of the search space) are not mutated. #' diff --git a/R/aaa.R b/R/aaa.R index ed02c255..8a83a3b6 100644 --- a/R/aaa.R +++ b/R/aaa.R @@ -1,2 +1,3 @@ optimizers = list() tuners = list() +learners = list() diff --git a/R/bayesopt_ego_log.R b/R/bayesopt_ego_log.R index 33c29777..6592e8f4 100644 --- a/R/bayesopt_ego_log.R +++ b/R/bayesopt_ego_log.R @@ -92,17 +92,17 @@ #'} #'} bayesopt_ego_log = function( - instance, - surrogate, - acq_function, - acq_optimizer, - init_design_size = NULL, - random_interleave_iter = 0L, - epsilon = 1e-3 + instance, + surrogate, + acq_function, + acq_optimizer, + init_design_size = NULL, + random_interleave_iter = 0L, + epsilon = 1e-3 ) { # assertions and defaults - assert_r6(instance, "OptimInstanceSingleCrit") + assert_r6(instance, "OptimInstanceBatchSingleCrit") assert_r6(surrogate, classes = "Surrogate") # cannot be SurrogateLearner due to EIPS assert_r6(acq_function, classes = "AcqFunction") # FIXME: should explicityly assert acqfs and make sure that codomain tag is handled assert_r6(acq_optimizer, classes = "AcqOptimizer") @@ -128,9 +128,11 @@ bayesopt_ego_log = function( surrogate$cols_y = "y_trafo" acq_function$surrogate_max_to_min = 1 - if (test_r6(acq_function, classes = "AcqFunctionCB") || test_r6(acq_function, classes = "AcqFunctionMean")) { - acq_function$codomain$params[[acq_function$id]]$tags = "minimize" - } + # FIXME: Cannot be set with new paradox + # + # if (test_r6(acq_function, classes = "AcqFunctionCB") || test_r6(acq_function, classes = "AcqFunctionMean")) { + # acq_function$codomain$params[[acq_function$id]]$tags = "minimize" + # } # actual loop repeat { @@ -168,4 +170,3 @@ attr(bayesopt_ego_log, "instance") = "single-crit" attr(bayesopt_ego_log, "man") = "mlr3mbo::mlr_loop_functions_ego_log" mlr_loop_functions$add("bayesopt_ego_log", bayesopt_ego_log) - diff --git a/R/zzz.R b/R/zzz.R index 18ab60f4..38290528 100644 --- a/R/zzz.R +++ b/R/zzz.R @@ -19,6 +19,13 @@ register_bbotk = function() { iwalk(optimizers, function(obj, nm) x$add(nm, obj)) } # nocov end +#' @include aaa.R +register_mlr3 = function() { + x = utils::getFromNamespace("mlr_learners", ns = "mlr3") + + iwalk(learners, function(obj, nm) x$add(nm, obj)) +} + register_mlr3tuning = function() { # nocov start x = utils::getFromNamespace("mlr_tuners", ns = "mlr3tuning") @@ -29,6 +36,7 @@ register_mlr3tuning = function() { # nocov start register_namespace_callback(pkgname, "bbotk", register_bbotk) register_namespace_callback(pkgname, "mlr3tuning", register_mlr3tuning) + register_namespace_callback(pkgname, "mlr3", register_mlr3) assign("lg", lgr::get_logger("bbotk"), envir = parent.env(environment())) @@ -39,12 +47,13 @@ register_mlr3tuning = function() { .onUnload = function(libpaths) { # nolint # nocov start + walk(names(learners), function(id) mlr3::mlr_learners$remove(id)) walk(names(optimizers), function(id) bbotk::mlr_optimizers$remove(id)) walk(names(tuners), function(id) mlr3tuning::mlr_tuners$remove(id)) } # nocov end # static code checks should not complain about commonly used data.table columns -utils::globalVariables(c("y_scal", "y_trafo", "predict", "batch_nr")) +utils::globalVariables("y_scal") if (!("mlr3mbo" %in% Sys.getenv("DEVTOOLS_LOAD"))) { leanify_package() diff --git a/man/mlr_acqfunctions_log_ei.Rd b/man/mlr_acqfunctions_log_ei.Rd index f76c215a..3aa53e80 100644 --- a/man/mlr_acqfunctions_log_ei.Rd +++ b/man/mlr_acqfunctions_log_ei.Rd @@ -54,10 +54,11 @@ On log scale.} } } \if{html}{\out{ -
Inherited methods +
Inherited methods
  • bbotk::Objective$eval()
  • bbotk::Objective$format()
    • +
  • bbotk::Objective$help()
  • bbotk::Objective$print()
  • mlr3mbo::AcqFunction$eval_dt()
  • mlr3mbo::AcqFunction$eval_many()
    • diff --git a/man/mlr_learners_regr.ranger.Rd b/man/mlr_learners_regr.ranger.Rd index 34d569cf..20bdeb38 100644 --- a/man/mlr_learners_regr.ranger.Rd +++ b/man/mlr_learners_regr.ranger.Rd @@ -2,23 +2,22 @@ % Please edit documentation in R/LearnerRegrRangerCustom.R \name{mlr_learners_regr.ranger} \alias{mlr_learners_regr.ranger} -\alias{LearnerRegrRangerCustom} +\alias{LearnerRegrRangerMbo} \title{Custom Ranger Regression Learner} \description{ Random regression forest. Calls \code{\link[ranger:ranger]{ranger::ranger()}} from package \CRANpkg{ranger}. -... } \section{Super classes}{ -\code{\link[mlr3:Learner]{mlr3::Learner}} -> \code{\link[mlr3:LearnerRegr]{mlr3::LearnerRegr}} -> \code{LearnerRegrRangerCustom} +\code{\link[mlr3:Learner]{mlr3::Learner}} -> \code{\link[mlr3:LearnerRegr]{mlr3::LearnerRegr}} -> \code{LearnerRegrRangerMbo} } \section{Methods}{ \subsection{Public methods}{ \itemize{ -\item \href{#method-LearnerRegrRangerCustom-new}{\code{LearnerRegrRangerCustom$new()}} -\item \href{#method-LearnerRegrRangerCustom-importance}{\code{LearnerRegrRangerCustom$importance()}} -\item \href{#method-LearnerRegrRangerCustom-oob_error}{\code{LearnerRegrRangerCustom$oob_error()}} -\item \href{#method-LearnerRegrRangerCustom-clone}{\code{LearnerRegrRangerCustom$clone()}} +\item \href{#method-LearnerRegrRangerMbo-new}{\code{LearnerRegrRangerMbo$new()}} +\item \href{#method-LearnerRegrRangerMbo-importance}{\code{LearnerRegrRangerMbo$importance()}} +\item \href{#method-LearnerRegrRangerMbo-oob_error}{\code{LearnerRegrRangerMbo$oob_error()}} +\item \href{#method-LearnerRegrRangerMbo-clone}{\code{LearnerRegrRangerMbo$clone()}} } } \if{html}{\out{ @@ -36,24 +35,24 @@ Calls \code{\link[ranger:ranger]{ranger::ranger()}} from package \CRANpkg{ranger
}} \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-LearnerRegrRangerCustom-new}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-LearnerRegrRangerMbo-new}{}}} \subsection{Method \code{new()}}{ Creates a new instance of this \link[R6:R6Class]{R6} class. \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{LearnerRegrRangerCustom$new()}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{LearnerRegrRangerMbo$new()}\if{html}{\out{
}} } } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-LearnerRegrRangerCustom-importance}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-LearnerRegrRangerMbo-importance}{}}} \subsection{Method \code{importance()}}{ The importance scores are extracted from the model slot \code{variable.importance}. Parameter \code{importance.mode} must be set to \code{"impurity"}, \code{"impurity_corrected"}, or \code{"permutation"} \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{LearnerRegrRangerCustom$importance()}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{LearnerRegrRangerMbo$importance()}\if{html}{\out{
}} } \subsection{Returns}{ @@ -61,12 +60,12 @@ Named \code{numeric()}. } } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-LearnerRegrRangerCustom-oob_error}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-LearnerRegrRangerMbo-oob_error}{}}} \subsection{Method \code{oob_error()}}{ The out-of-bag error, extracted from model slot \code{prediction.error}. \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{LearnerRegrRangerCustom$oob_error()}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{LearnerRegrRangerMbo$oob_error()}\if{html}{\out{
}} } \subsection{Returns}{ @@ -74,12 +73,12 @@ The out-of-bag error, extracted from model slot \code{prediction.error}. } } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-LearnerRegrRangerCustom-clone}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-LearnerRegrRangerMbo-clone}{}}} \subsection{Method \code{clone()}}{ The objects of this class are cloneable with this method. \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{LearnerRegrRangerCustom$clone(deep = FALSE)}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{LearnerRegrRangerMbo$clone(deep = FALSE)}\if{html}{\out{
}} } \subsection{Arguments}{ diff --git a/man/mlr_optimizers_chain.Rd b/man/mlr_optimizers_chain.Rd index 839d8a26..a3db29b7 100644 --- a/man/mlr_optimizers_chain.Rd +++ b/man/mlr_optimizers_chain.Rd @@ -47,7 +47,6 @@ Parameters are inherited from the individual \link{Optimizer}s and collected as
diff --git a/man/mlr_optimizers_local_search.Rd b/man/mlr_optimizers_local_search.Rd index 731374c3..30b11225 100644 --- a/man/mlr_optimizers_local_search.Rd +++ b/man/mlr_optimizers_local_search.Rd @@ -12,10 +12,10 @@ random are evaluated. In each iteration, for each of the \code{mu} initial best points, \code{n_points} neighbors are generated by local mutation. Local mutation generates a neighbor by sampling a single parameter that is to -be mutated and then proceeds as follows: Double parameters (\link[paradox:ParamDbl]{paradox::ParamDbl}) are mutated via +be mutated and then proceeds as follows: Double parameters (\link[paradox:Domain]{paradox::p_dbl}) are mutated via Gaussian mutation (with a prior standardization to \verb{[0, 1]} and retransformation after mutation). -Integer parameters (\link[paradox:ParamInt]{paradox::ParamInt}) undergo the same mutation but are rounded to the closest -integer after mutation. Categorical parameters (\link[paradox:ParamFct]{paradox::ParamFct} and \link[paradox:ParamLgl]{paradox::ParamLgl}) are +Integer parameters (\link[paradox:Domain]{paradox::p_int}) undergo the same mutation but are rounded to the closest +integer after mutation. Categorical parameters (\link[paradox:Domain]{paradox::p_fct} and \link[paradox:Domain]{paradox::p_lgl}) are mutated via uniform mutation. Note that parameters that are conditioned on (i.e., they are parents of a \link[paradox:Condition]{paradox::Condition}, see the dependencies of the search space) are not mutated. } @@ -53,7 +53,6 @@ Default is \code{0.1}. From 7cb1fa57444e60ba3fafa86e36079b710ebe8f42 Mon Sep 17 00:00:00 2001 From: be-marc Date: Fri, 28 Jun 2024 20:41:10 +0200 Subject: [PATCH 065/126] fix: optimizerchain --- R/OptimizerChain.R | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/R/OptimizerChain.R b/R/OptimizerChain.R index 393de1a0..7e1a3794 100644 --- a/R/OptimizerChain.R +++ b/R/OptimizerChain.R @@ -44,24 +44,26 @@ OptimizerChain = R6Class("OptimizerChain", assert_list(optimizers, types = "Optimizer", any.missing = FALSE) assert_list(terminators, types = c("Terminator", "NULL"), len = length(optimizers)) - param_sets = vector(mode = "list", length = length(optimizers)) - ids_taken = character(0L) + # param_sets = vector(mode = "list", length = length(optimizers)) + # ids_taken = character(0L) # for each optimizer check whether the id of the param_set # (decuded from the optimizer class) is already taken; # if necessary postfix the id - for (i_opt in seq_along(optimizers)) { - opt = optimizers[[i_opt]] - ps = opt$param_set$clone(deep = TRUE) - ps$set_id = class(opt)[[1L]] - try_postfix = 0L - while (ps$set_id %in% ids_taken) { - try_postfix = try_postfix + 1L - ps$set_id = paste0(class(opt)[[1L]], "_", try_postfix) - } - ids_taken[[i_opt]] = ps$set_id - param_sets[[i_opt]] = ps - } - private$.ids = map_chr(param_sets, "set_id") + # for (i_opt in seq_along(optimizers)) { + # opt = optimizers[[i_opt]] + # ps = opt$param_set$clone(deep = TRUE) + # ps$set_id = class(opt)[[1L]] + # try_postfix = 0L + # while (ps$set_id %in% ids_taken) { + # try_postfix = try_postfix + 1L + # ps$set_id = paste0(class(opt)[[1L]], "_", try_postfix) + # } + # ids_taken[[i_opt]] = ps$set_id + # param_sets[[i_opt]] = ps + # } + + param_sets = map(optimizers, function(x) x$param_set$clone(deep = TRUE)) + private$.ids = map_chr(optimizers, "id") super$initialize( param_set = ParamSetCollection$new(param_sets), param_classes = Reduce(intersect, mlr3misc::map(optimizers, "param_classes")), From 6a99e1f2c801b8764641dcb6befb12e2fb7d3501 Mon Sep 17 00:00:00 2001 From: be-marc Date: Sat, 29 Jun 2024 08:48:49 +0200 Subject: [PATCH 066/126] update --- R/OptimizerChain.R | 2 +- R/OptimizerLocalSearch.R | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/R/OptimizerChain.R b/R/OptimizerChain.R index 7e1a3794..f9c1da9f 100644 --- a/R/OptimizerChain.R +++ b/R/OptimizerChain.R @@ -32,7 +32,7 @@ #' #' @export OptimizerChain = R6Class("OptimizerChain", - inherit = bbotk::Optimizer, + inherit = bbotk::OptimizerBatch, public = list( #' @description diff --git a/R/OptimizerLocalSearch.R b/R/OptimizerLocalSearch.R index 123ac77c..16621aaa 100644 --- a/R/OptimizerLocalSearch.R +++ b/R/OptimizerLocalSearch.R @@ -36,7 +36,7 @@ #' #' @export OptimizerLocalSearch = R6Class("OptimizerLocalSearch", - inherit = bbotk::Optimizer, + inherit = bbotk::OptimizerBatch, public = list( #' @description @@ -80,7 +80,7 @@ OptimizerLocalSearch = R6Class("OptimizerLocalSearch", ids_numeric = intersect(names(which(inst$search_space$is_number)), ids_to_mutate) ids_categorical = intersect(names(which(inst$search_space$is_categ)), ids_to_mutate) - ids_categorical = ids_categorical[map_lgl(inst$search_space$params[ids_categorical], function(x) x$nlevels > 1L)] + ids_categorical = ids_categorical[which(inst$search_space$nlevels > 1)] point_id = ".point_id" while (point_id %in% c(inst$archive$cols_x, inst$archive$cols_y)) { @@ -123,8 +123,11 @@ mutate_point = function(point, search_space, ids_numeric, ids_categorical, sigma neighbor } -mutate = function(value, param, sigma) { - stopifnot(param$class %in% c("ParamDbl", "ParamFct", "ParamInt", "ParamLgl")) +mutate = function(value, , sigma) { + #stopifnot(param$class %in% c("ParamDbl", "ParamFct", "ParamInt", "ParamLgl")) + + + if (param$class %in% c("ParamDbl", "ParamInt")) { value_ = (value - param$lower) / (param$upper - param$lower) value_ = max(0, min(stats::rnorm(1L, mean = value_, sd = sigma), 1)) From 5578435188e2f37c9d8b505a7727df1b3fbdfa72 Mon Sep 17 00:00:00 2001 From: be-marc Date: Sat, 29 Jun 2024 16:48:15 +0200 Subject: [PATCH 067/126] refactor: local search with new paradox --- DESCRIPTION | 2 +- NAMESPACE | 4 +-- ...alSearch.R => OptimizerBatchLocalSearch.R} | 36 ++++++++----------- R/OptimizerChain.R | 8 ++--- man/mlr_optimizers_chain.Rd | 27 +++++++------- man/mlr_optimizers_local_search.Rd | 27 +++++++------- 6 files changed, 50 insertions(+), 54 deletions(-) rename R/{OptimizerLocalSearch.R => OptimizerBatchLocalSearch.R} (82%) diff --git a/DESCRIPTION b/DESCRIPTION index 77b80dc4..054e9042 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -93,8 +93,8 @@ Collate: 'AcqOptimizer.R' 'aaa.R' 'LearnerRegrRangerCustom.R' + 'OptimizerBatchLocalSearch.R' 'OptimizerChain.R' - 'OptimizerLocalSearch.R' 'OptimizerMbo.R' 'mlr_result_assigners.R' 'ResultAssigner.R' diff --git a/NAMESPACE b/NAMESPACE index 15171a89..eade4952 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -19,8 +19,8 @@ export(AcqFunctionSD) export(AcqFunctionSmsEgo) export(AcqOptimizer) export(LearnerRegrRangerMbo) -export(OptimizerChain) -export(OptimizerLocalSearch) +export(OptimizerBatchChain) +export(OptimizerBatchLocalSearch) export(OptimizerMbo) export(ResultAssigner) export(ResultAssignerArchive) diff --git a/R/OptimizerLocalSearch.R b/R/OptimizerBatchLocalSearch.R similarity index 82% rename from R/OptimizerLocalSearch.R rename to R/OptimizerBatchLocalSearch.R index 16621aaa..6a748259 100644 --- a/R/OptimizerLocalSearch.R +++ b/R/OptimizerBatchLocalSearch.R @@ -3,7 +3,7 @@ #' @name mlr_optimizers_local_search #' #' @description -#' `OptimizerLocalSearch` class that implements a simple Local Search. +#' `OptimizerBatchLocalSearch` class that implements a simple Local Search. #' Local Search starts by determining the `mu` initial best points present in the [Archive] of the #' [OptimInstance]. If fewer points than `mu` are present, additional points sampled uniformly at #' random are evaluated. @@ -35,7 +35,7 @@ #' } #' #' @export -OptimizerLocalSearch = R6Class("OptimizerLocalSearch", +OptimizerBatchLocalSearch = R6Class("OptimizerBatchLocalSearch", inherit = bbotk::OptimizerBatch, public = list( @@ -76,11 +76,9 @@ OptimizerLocalSearch = R6Class("OptimizerLocalSearch", # we do not mutate parents of conditions ids_to_mutate = setdiff(inst$search_space$ids(), unique(inst$search_space$deps$on)) - - ids_numeric = intersect(names(which(inst$search_space$is_number)), ids_to_mutate) - - ids_categorical = intersect(names(which(inst$search_space$is_categ)), ids_to_mutate) - ids_categorical = ids_categorical[which(inst$search_space$nlevels > 1)] + ids_numeric = intersect(inst$search_space$ids(class = c("ParamDbl", "ParamInt")), ids_to_mutate) + ids_categorical = intersect(inst$search_space$ids(class = c("ParamLgl", "ParamFct")), ids_to_mutate) + ids_categorical = intersect(ids_categorical, search_space$ids()[search_space$nlevels > 1]) point_id = ".point_id" while (point_id %in% c(inst$archive$cols_x, inst$archive$cols_y)) { @@ -119,28 +117,24 @@ mutate_point = function(point, search_space, ids_numeric, ids_categorical, sigma valid_numeric_to_mutate = intersect(names(which(!map_lgl(neighbor, is.na))), ids_numeric) valid_cateorical_to_mutate = intersect(names(which(!map_lgl(neighbor, is.na))), ids_categorical) id = sample(c(valid_numeric_to_mutate, valid_cateorical_to_mutate), size = 1L) - neighbor[1L, ][[id]] = mutate(neighbor[1L, ][[id]], param = search_space$params[[id]], sigma = sigma) + neighbor[1L, ][[id]] = mutate(neighbor[1L, ][[id]], subspace = search_space$subspaces(ids = id)[[1]], sigma = sigma) neighbor } -mutate = function(value, , sigma) { - #stopifnot(param$class %in% c("ParamDbl", "ParamFct", "ParamInt", "ParamLgl")) - - - - if (param$class %in% c("ParamDbl", "ParamInt")) { - value_ = (value - param$lower) / (param$upper - param$lower) +mutate = function(value, subspace, sigma) { + if (subspace$class %in% c("ParamDbl", "ParamInt")) { + value_ = (value - subspace$lower) / (subspace$upper - subspace$lower) value_ = max(0, min(stats::rnorm(1L, mean = value_, sd = sigma), 1)) - value = (value_ * (param$upper - param$lower)) + param$lower - if (param$class == "ParamInt") { + value = (value_ * (subspace$upper - subspace$lower)) + subspace$lower + if (subspace$class == "ParamInt") { value = round(value, 0L) } - value = min(max(value, param$lower), param$upper) - } else if (param$class %in% c("ParamFct", "ParamLgl")) { - value = sample(setdiff(param$levels, value), size = 1L) + value = min(max(value, subspace$lower), subspace$upper) + } else if (subspace$class %in% c("ParamFct", "ParamLgl")) { + value = sample(setdiff(subspace$levels[[1]], value), size = 1L) } value } #' @include aaa.R -optimizers[["local_search"]] = OptimizerLocalSearch +optimizers[["local_search"]] = OptimizerBatchLocalSearch diff --git a/R/OptimizerChain.R b/R/OptimizerChain.R index f9c1da9f..32564292 100644 --- a/R/OptimizerChain.R +++ b/R/OptimizerChain.R @@ -3,7 +3,7 @@ #' @name mlr_optimizers_chain #' #' @description -#' `OptimizerChain` allows to run different [Optimizer]s sequentially. +#' `OptimizerBatchChain` allows to run different [Optimizer]s sequentially. #' #' For each [Optimizer] an (optional) additional [Terminator] can be specified #' during construction. @@ -18,7 +18,7 @@ #' additional [Terminator]), the second [Optimizer] (and so on) is run unless #' the original [Terminator] of the [OptimInstance] indicates termination. #' -#' [OptimizerChain] can also be used for random restarts of the same +#' [OptimizerBatchChain] can also be used for random restarts of the same #' [Optimizer] (if applicable) by setting the [Terminator] of the [OptimInstance] to #' [TerminatorNone] and setting identical additional [Terminator]s during #' construction. @@ -31,7 +31,7 @@ #' #' #' @export -OptimizerChain = R6Class("OptimizerChain", +OptimizerBatchChain = R6Class("OptimizerBatchChain", inherit = bbotk::OptimizerBatch, public = list( @@ -116,4 +116,4 @@ OptimizerChain = R6Class("OptimizerChain", ) #' @include aaa.R -optimizers[["chain"]] = OptimizerChain +optimizers[["chain"]] = OptimizerBatchChain diff --git a/man/mlr_optimizers_chain.Rd b/man/mlr_optimizers_chain.Rd index a3db29b7..3adccc8f 100644 --- a/man/mlr_optimizers_chain.Rd +++ b/man/mlr_optimizers_chain.Rd @@ -2,10 +2,10 @@ % Please edit documentation in R/OptimizerChain.R \name{mlr_optimizers_chain} \alias{mlr_optimizers_chain} -\alias{OptimizerChain} +\alias{OptimizerBatchChain} \title{Run Optimizers Sequentially} \description{ -\code{OptimizerChain} allows to run different \link{Optimizer}s sequentially. +\code{OptimizerBatchChain} allows to run different \link{Optimizer}s sequentially. For each \link{Optimizer} an (optional) additional \link{Terminator} can be specified during construction. @@ -20,7 +20,7 @@ Once this \link{TerminatorCombo} indicates termination (usually via the additional \link{Terminator}), the second \link{Optimizer} (and so on) is run unless the original \link{Terminator} of the \link{OptimInstance} indicates termination. -\link{OptimizerChain} can also be used for random restarts of the same +\link{OptimizerBatchChain} can also be used for random restarts of the same \link{Optimizer} (if applicable) by setting the \link{Terminator} of the \link{OptimInstance} to \link{TerminatorNone} and setting identical additional \link{Terminator}s during construction. @@ -32,14 +32,14 @@ Parameters are inherited from the individual \link{Optimizer}s and collected as ..., if the same \link{Optimizer}s are used multiple times). } -\section{Super class}{ -\code{\link[bbotk:Optimizer]{bbotk::Optimizer}} -> \code{OptimizerChain} +\section{Super classes}{ +\code{\link[bbotk:Optimizer]{bbotk::Optimizer}} -> \code{\link[bbotk:OptimizerBatch]{bbotk::OptimizerBatch}} -> \code{OptimizerBatchChain} } \section{Methods}{ \subsection{Public methods}{ \itemize{ -\item \href{#method-OptimizerChain-new}{\code{OptimizerChain$new()}} -\item \href{#method-OptimizerChain-clone}{\code{OptimizerChain$clone()}} +\item \href{#method-OptimizerBatchChain-new}{\code{OptimizerBatchChain$new()}} +\item \href{#method-OptimizerBatchChain-clone}{\code{OptimizerBatchChain$clone()}} } } \if{html}{\out{ @@ -48,16 +48,17 @@ Parameters are inherited from the individual \link{Optimizer}s and collected as
  • bbotk::Optimizer$format()
  • bbotk::Optimizer$help()
  • bbotk::Optimizer$print()
    • +
  • bbotk::OptimizerBatch$optimize()
    • }} \if{html}{\out{
    }} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-OptimizerChain-new}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-OptimizerBatchChain-new}{}}} \subsection{Method \code{new()}}{ Creates a new instance of this \link[R6:R6Class]{R6} class. \subsection{Usage}{ -\if{html}{\out{
    }}\preformatted{OptimizerChain$new( +\if{html}{\out{
    }}\preformatted{OptimizerBatchChain$new( optimizers, terminators = rep(list(NULL), length(optimizers)) )}\if{html}{\out{
    }} @@ -74,12 +75,12 @@ Creates a new instance of this \link[R6:R6Class]{R6} class. } } \if{html}{\out{
    }} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-OptimizerChain-clone}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-OptimizerBatchChain-clone}{}}} \subsection{Method \code{clone()}}{ The objects of this class are cloneable with this method. \subsection{Usage}{ -\if{html}{\out{
    }}\preformatted{OptimizerChain$clone(deep = FALSE)}\if{html}{\out{
    }} +\if{html}{\out{
    }}\preformatted{OptimizerBatchChain$clone(deep = FALSE)}\if{html}{\out{
    }} } \subsection{Arguments}{ diff --git a/man/mlr_optimizers_local_search.Rd b/man/mlr_optimizers_local_search.Rd index 30b11225..4a4f368f 100644 --- a/man/mlr_optimizers_local_search.Rd +++ b/man/mlr_optimizers_local_search.Rd @@ -1,11 +1,11 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/OptimizerLocalSearch.R +% Please edit documentation in R/OptimizerBatchLocalSearch.R \name{mlr_optimizers_local_search} \alias{mlr_optimizers_local_search} -\alias{OptimizerLocalSearch} +\alias{OptimizerBatchLocalSearch} \title{Optimization via Local Search} \description{ -\code{OptimizerLocalSearch} class that implements a simple Local Search. +\code{OptimizerBatchLocalSearch} class that implements a simple Local Search. Local Search starts by determining the \code{mu} initial best points present in the \link{Archive} of the \link{OptimInstance}. If fewer points than \code{mu} are present, additional points sampled uniformly at random are evaluated. @@ -38,14 +38,14 @@ Default is \code{0.1}. } } -\section{Super class}{ -\code{\link[bbotk:Optimizer]{bbotk::Optimizer}} -> \code{OptimizerLocalSearch} +\section{Super classes}{ +\code{\link[bbotk:Optimizer]{bbotk::Optimizer}} -> \code{\link[bbotk:OptimizerBatch]{bbotk::OptimizerBatch}} -> \code{OptimizerBatchLocalSearch} } \section{Methods}{ \subsection{Public methods}{ \itemize{ -\item \href{#method-OptimizerLocalSearch-new}{\code{OptimizerLocalSearch$new()}} -\item \href{#method-OptimizerLocalSearch-clone}{\code{OptimizerLocalSearch$clone()}} +\item \href{#method-OptimizerBatchLocalSearch-new}{\code{OptimizerBatchLocalSearch$new()}} +\item \href{#method-OptimizerBatchLocalSearch-clone}{\code{OptimizerBatchLocalSearch$clone()}} } } \if{html}{\out{ @@ -54,26 +54,27 @@ Default is \code{0.1}.
  • bbotk::Optimizer$format()
  • bbotk::Optimizer$help()
  • bbotk::Optimizer$print()
    • +
  • bbotk::OptimizerBatch$optimize()
    • }} \if{html}{\out{
    }} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-OptimizerLocalSearch-new}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-OptimizerBatchLocalSearch-new}{}}} \subsection{Method \code{new()}}{ Creates a new instance of this \link[R6:R6Class]{R6} class. \subsection{Usage}{ -\if{html}{\out{
    }}\preformatted{OptimizerLocalSearch$new()}\if{html}{\out{
    }} +\if{html}{\out{
    }}\preformatted{OptimizerBatchLocalSearch$new()}\if{html}{\out{
    }} } } \if{html}{\out{
    }} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-OptimizerLocalSearch-clone}{}}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-OptimizerBatchLocalSearch-clone}{}}} \subsection{Method \code{clone()}}{ The objects of this class are cloneable with this method. \subsection{Usage}{ -\if{html}{\out{
    }}\preformatted{OptimizerLocalSearch$clone(deep = FALSE)}\if{html}{\out{
    }} +\if{html}{\out{
    }}\preformatted{OptimizerBatchLocalSearch$clone(deep = FALSE)}\if{html}{\out{
    }} } \subsection{Arguments}{ From aad068e39ca3dbd6021942129049917d5fbbb2f4 Mon Sep 17 00:00:00 2001 From: be-marc Date: Tue, 2 Jul 2024 07:22:57 +0200 Subject: [PATCH 068/126] fix: cols_y --- R/AcqFunctionLogEI.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/AcqFunctionLogEI.R b/R/AcqFunctionLogEI.R index e416a206..ed022cb1 100644 --- a/R/AcqFunctionLogEI.R +++ b/R/AcqFunctionLogEI.R @@ -35,7 +35,7 @@ AcqFunctionLogEI = R6Class("AcqFunctionLogEI", if (self$surrogate_max_to_min != 1L) { stop("Log EI assumes minimization of the log transformed target value.") } - self$y_best = min(self$surrogate_max_to_min * self$archive$data[[self$surrogate$y_cols]]) + self$y_best = min(self$surrogate_max_to_min * self$archive$data[[self$surrogate$cols_y]]) } ), private = list( From 30a3546e4ca5e30af099b39577d159159a9c31d8 Mon Sep 17 00:00:00 2001 From: be-marc Date: Tue, 2 Jul 2024 11:25:09 +0200 Subject: [PATCH 069/126] fix: remove browser() --- R/AcqFunctionEI.R | 1 + 1 file changed, 1 insertion(+) diff --git a/R/AcqFunctionEI.R b/R/AcqFunctionEI.R index 1782b972..c4fea1d7 100644 --- a/R/AcqFunctionEI.R +++ b/R/AcqFunctionEI.R @@ -77,6 +77,7 @@ AcqFunctionEI = R6Class("AcqFunctionEI", if (is.null(self$y_best)) { stop("$y_best is not set. Missed to call $update()?") } + browser() p = self$surrogate$predict(xdt) mu = p$mean se = p$se From 5ba9ea7b42de8ae337d367b487d629ea5b2fa735 Mon Sep 17 00:00:00 2001 From: be-marc Date: Tue, 2 Jul 2024 11:25:44 +0200 Subject: [PATCH 070/126] chore: browser --- R/AcqFunctionEI.R | 2 +- attic/issue_1.r | 173 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 174 insertions(+), 1 deletion(-) create mode 100644 attic/issue_1.r diff --git a/R/AcqFunctionEI.R b/R/AcqFunctionEI.R index c4fea1d7..9bb20234 100644 --- a/R/AcqFunctionEI.R +++ b/R/AcqFunctionEI.R @@ -77,7 +77,7 @@ AcqFunctionEI = R6Class("AcqFunctionEI", if (is.null(self$y_best)) { stop("$y_best is not set. Missed to call $update()?") } - browser() + p = self$surrogate$predict(xdt) mu = p$mean se = p$se diff --git a/attic/issue_1.r b/attic/issue_1.r new file mode 100644 index 00000000..88387653 --- /dev/null +++ b/attic/issue_1.r @@ -0,0 +1,173 @@ +devtools::load_all(".") +library(batchtools) +library(mlr3misc) +library(data.table) +library(paradox) +library(bbotk) +library(reticulate) +library(yahpogym) +library(batchtools) +library(mlr3misc) +library(data.table) +library(paradox) +library(bbotk) +library(mlr3learners) +library(mlr3mbo) +library(mlr3pipelines) +library(reticulate) +library(yahpogym) + +use_condaenv("yahpo_gym", required=TRUE) +yahpo_gym = import("yahpo_gym") + +log_scale = TRUE +init = "random" +init_size_fraction = 0.25 +random_interleave_iter = 0 +rf_type = "standard" +acqf = "EI" +acqopt = "RS_1000" +lambda = NA_real_ +id = 1L +config_hash = "88169553-ad40-4225-98ef-0579f4207f43" + +random_interleave_iter = as.numeric(random_interleave_iter) +init_size_fraction = as.numeric(init_size_fraction) +lambda = as.numeric(lambda) + +use_condaenv("yahpo_gym", required = TRUE) +yahpo_gym = import("yahpo_gym") + +benchmark = BenchmarkSet$new("rbv2_svm") +benchmark$subset_codomain("acc") +objective = benchmark$get_objective("40981", multifidelity = FALSE) + +optim_instance = oi( + objective, + search_space = benchmark$get_search_space(drop_fidelity_params = TRUE), + terminator = trm("evals", n_evals = 200), + check_values = FALSE) + +init_design_size = ceiling(as.numeric(init_size_fraction) * 200) +init_design = if (init == "random") { + generate_design_random(optim_instance$search_space, n = init_design_size)$data +} else if (init == "lhs") { + generate_design_lhs(optim_instance$search_space, n = init_design_size)$data +} else if (init == "sobol") { + generate_design_sobol(optim_instance$search_space, n = init_design_size)$data +} + +optim_instance$eval_batch(init_design) + +learner = LearnerRegrRangerMbo$new() +learner$predict_type = "se" +learner$param_set$values$keep.inbag = TRUE + +if (rf_type == "standard") { + learner$param_set$values$se.method = "jack" + learner$param_set$values$splitrule = "variance" + learner$param_set$values$num.trees = 1000L +} else if (rf_type == "extratrees") { + learner$param_set$values$se.method = "jack" + learner$param_set$values$splitrule = "extratrees" + learner$param_set$values$num.random.splits = 1L + learner$param_set$values$num.trees = 1000L +} else if (rf_type == "smaclike_boot") { + learner$param_set$values$se.method = "simple" + learner$param_set$values$splitrule = "extratrees" + learner$param_set$values$num.random.splits = 1L + learner$param_set$values$num.trees = 10L + learner$param_set$values$replace = TRUE + learner$param_set$values$sample.fraction = 1 + learner$param_set$values$min.node.size = 1 + learner$param_set$values$mtry.ratio = 1 +} else if (rf_type == "smaclike_no_boot") { + learner$param_set$values$se.method = "simple" + learner$param_set$values$splitrule = "extratrees" + learner$param_set$values$num.random.splits = 1L + learner$param_set$values$num.trees = 10L + learner$param_set$values$replace = FALSE + learner$param_set$values$sample.fraction = 1 + learner$param_set$values$min.node.size = 1 + learner$param_set$values$mtry.ratio = 1 +} + +surrogate = SurrogateLearner$new(GraphLearner$new(po("imputesample", affect_columns = selector_type("logical")) %>>% + po("imputeoor", multiplier = 3, affect_columns = selector_type(c("integer", "numeric", "character", "factor", "ordered"))) %>>% + po("colapply", applicator = as.factor, affect_columns = selector_type("character")) %>>% + learner)) +surrogate$param_set$values$catch_errors = TRUE + +acq_optimizer = if (acqopt == "RS_1000") { + AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 1000L)) +} else if (acqopt == "RS") { + AcqOptimizer$new(opt("random_search", batch_size = 1000L), terminator = trm("evals", n_evals = 20000L)) +} else if (acqopt == "FS") { + n_repeats = 2L + maxit = 9L + batch_size = ceiling((20000L / n_repeats) / (1 + maxit)) # 1000L + AcqOptimizer$new(opt("focus_search", n_points = batch_size, maxit = maxit), terminator = trm("evals", n_evals = 20000L)) +} else if (acqopt == "LS") { + optimizer = OptimizerChain$new(list(opt("local_search", n_points = 100L), opt("random_search", batch_size = 1000L)), terminators = list(trm("evals", n_evals = 10000L), trm("evals", n_evals = 10000L))) + acq_optimizer = AcqOptimizer$new(optimizer, terminator = trm("evals", n_evals = 20000L)) + acq_optimizer$param_set$values$warmstart = TRUE + acq_optimizer$param_set$values$warmstart_size = "all" + acq_optimizer +} +acq_optimizer$param_set$values$catch_errors = FALSE + +acq_function = if (acqf == "EI" && log_scale) { + AcqFunctionLogEI$new() +} else if (acqf == "EI" && !log_scale) { + AcqFunctionEI$new() +} else if (acqf == "CB") { + AcqFunctionCB$new(lambda = as.numeric(lambda)) +} else if (acqf == "PI") { + AcqFunctionPI$new() +} else if (acqf == "Mean") { + AcqFunctionMean$new() +} + +if (!log_scale) { + bayesopt_ego( + optim_instance, + surrogate = surrogate, + acq_function = acq_function, + acq_optimizer = acq_optimizer, + random_interleave_iter = random_interleave_iter, + init_design_size = init_design_size) +} else { + bayesopt_ego_log( + optim_instance, + surrogate = surrogate, + acq_function = acq_function, + acq_optimizer = acq_optimizer, + random_interleave_iter = random_interleave_iter, + init_design_size = init_design_size) +} + +target = optim_instance$archive$cols_y +best = optim_instance$archive$best()[[target]] +data.table(best = best, target = target, problem = job$prob.name, id = id, repl = job$repl) + + +# issue 1 +# Error in .subset2(x, i, exact = exact) : +# attempt to select less than one element in get1index + +# log_scale = TRUE +# init = "random" +# init_size_fraction = 0.25 +# random_interleave_iter = 0 +# rf_type = "standard" +# acqf = "EI" +# acqopt = "RS_1000" +# lambda = NA_real_ +# id = 1L +# config_hash = "88169553-ad40-4225-98ef-0579f4207f43" + +# xs = list(log_scale = TRUE, init = "random", init_size_fraction = 0.25, +# random_interleave_iter = 0, rf_type = "standard", acqf = "EI", +# acqopt = "RS_1000", lambda = NA_real_, id = 1L, config_hash = "88169553-ad40-4225-98ef-0579f4207f43") + +# invoke(run, .args = xs) From 77cb3fe6c96fa8737598945842f307b65534857e Mon Sep 17 00:00:00 2001 From: be-marc Date: Tue, 2 Jul 2024 12:43:58 +0200 Subject: [PATCH 071/126] fix: local search --- R/OptimizerBatchLocalSearch.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/OptimizerBatchLocalSearch.R b/R/OptimizerBatchLocalSearch.R index 6a748259..ffdda83c 100644 --- a/R/OptimizerBatchLocalSearch.R +++ b/R/OptimizerBatchLocalSearch.R @@ -78,7 +78,7 @@ OptimizerBatchLocalSearch = R6Class("OptimizerBatchLocalSearch", ids_to_mutate = setdiff(inst$search_space$ids(), unique(inst$search_space$deps$on)) ids_numeric = intersect(inst$search_space$ids(class = c("ParamDbl", "ParamInt")), ids_to_mutate) ids_categorical = intersect(inst$search_space$ids(class = c("ParamLgl", "ParamFct")), ids_to_mutate) - ids_categorical = intersect(ids_categorical, search_space$ids()[search_space$nlevels > 1]) + ids_categorical = intersect(ids_categorical, inst$search_space$ids()[inst$search_space$nlevels > 1]) point_id = ".point_id" while (point_id %in% c(inst$archive$cols_x, inst$archive$cols_y)) { From 9bbc6987080553ba351c84dc894788339e08c927 Mon Sep 17 00:00:00 2001 From: be-marc Date: Mon, 8 Jul 2024 17:55:45 +0200 Subject: [PATCH 072/126] fix: minimize --- R/bayesopt_ego_log.R | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/R/bayesopt_ego_log.R b/R/bayesopt_ego_log.R index 6592e8f4..1e149c4d 100644 --- a/R/bayesopt_ego_log.R +++ b/R/bayesopt_ego_log.R @@ -128,11 +128,9 @@ bayesopt_ego_log = function( surrogate$cols_y = "y_trafo" acq_function$surrogate_max_to_min = 1 - # FIXME: Cannot be set with new paradox - # - # if (test_r6(acq_function, classes = "AcqFunctionCB") || test_r6(acq_function, classes = "AcqFunctionMean")) { - # acq_function$codomain$params[[acq_function$id]]$tags = "minimize" - # } + if (test_r6(acq_function, classes = "AcqFunctionCB") || test_r6(acq_function, classes = "AcqFunctionMean")) { + acq_function$codomain$tags[[acq_function$id]] = "minimize" + } # actual loop repeat { From ae8702168cd358dfa27087581f122c815bad62ff Mon Sep 17 00:00:00 2001 From: be-marc Date: Tue, 13 Aug 2024 19:03:00 +0200 Subject: [PATCH 073/126] refactor: remove chain and local search --- DESCRIPTION | 4 +- NAMESPACE | 2 - R/OptimizerBatchLocalSearch.R | 140 ----------------------------- R/OptimizerChain.R | 119 ------------------------ man/mlr_optimizers_chain.Rd | 94 ------------------- man/mlr_optimizers_local_search.Rd | 88 ------------------ 6 files changed, 1 insertion(+), 446 deletions(-) delete mode 100644 R/OptimizerBatchLocalSearch.R delete mode 100644 R/OptimizerChain.R delete mode 100644 man/mlr_optimizers_chain.Rd delete mode 100644 man/mlr_optimizers_local_search.Rd diff --git a/DESCRIPTION b/DESCRIPTION index e1464c0b..a069e527 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -72,7 +72,7 @@ Config/testthat/edition: 3 Config/testthat/parallel: false NeedsCompilation: yes Roxygen: list(markdown = TRUE, r6 = TRUE) -RoxygenNote: 7.3.1 +RoxygenNote: 7.3.2 Collate: 'mlr_acqfunctions.R' 'AcqFunction.R' @@ -90,8 +90,6 @@ Collate: 'AcqOptimizer.R' 'aaa.R' 'LearnerRegrRangerCustom.R' - 'OptimizerBatchLocalSearch.R' - 'OptimizerChain.R' 'OptimizerMbo.R' 'mlr_result_assigners.R' 'ResultAssigner.R' diff --git a/NAMESPACE b/NAMESPACE index eade4952..c18f4fc7 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -19,8 +19,6 @@ export(AcqFunctionSD) export(AcqFunctionSmsEgo) export(AcqOptimizer) export(LearnerRegrRangerMbo) -export(OptimizerBatchChain) -export(OptimizerBatchLocalSearch) export(OptimizerMbo) export(ResultAssigner) export(ResultAssignerArchive) diff --git a/R/OptimizerBatchLocalSearch.R b/R/OptimizerBatchLocalSearch.R deleted file mode 100644 index ffdda83c..00000000 --- a/R/OptimizerBatchLocalSearch.R +++ /dev/null @@ -1,140 +0,0 @@ -#' @title Optimization via Local Search -#' -#' @name mlr_optimizers_local_search -#' -#' @description -#' `OptimizerBatchLocalSearch` class that implements a simple Local Search. -#' Local Search starts by determining the `mu` initial best points present in the [Archive] of the -#' [OptimInstance]. If fewer points than `mu` are present, additional points sampled uniformly at -#' random are evaluated. -#' -#' In each iteration, for each of the `mu` initial best points, `n_points` neighbors are generated -#' by local mutation. Local mutation generates a neighbor by sampling a single parameter that is to -#' be mutated and then proceeds as follows: Double parameters ([paradox::p_dbl]) are mutated via -#' Gaussian mutation (with a prior standardization to `[0, 1]` and retransformation after mutation). -#' Integer parameters ([paradox::p_int]) undergo the same mutation but are rounded to the closest -#' integer after mutation. Categorical parameters ([paradox::p_fct] and [paradox::p_lgl]) are -#' mutated via uniform mutation. Note that parameters that are conditioned on (i.e., they are -#' parents of a [paradox::Condition], see the dependencies of the search space) are not mutated. -#' -#' @section Parameters: -#' \describe{ -#' \item{`mu`}{`integer(1)`\cr -#' Size of the initial best points which are used as a starting points for the Local Search. -#' Default is `10`. -#' } -#' \item{`n_points`}{`integer(1)`\cr -#' Number of neighboring points to generate for each of the `mu` best starting points in each -#' iteration. -#' Default is `100`. -#' } -#' \item{`sigma`}{`numeric(1)`\cr -#' Standard deviation used for mutation of numeric parameters. -#' Default is `0.1`. -#' } -#' } -#' -#' @export -OptimizerBatchLocalSearch = R6Class("OptimizerBatchLocalSearch", - inherit = bbotk::OptimizerBatch, - public = list( - - #' @description - #' Creates a new instance of this [R6][R6::R6Class] class. - initialize = function() { - param_set = ps( - mu = p_int(lower = 1L, default = 10L), - n_points = p_int(lower = 1L, default = 100L), - sigma = p_dbl(lower = 0L, default = 0.1) - ) - param_set$values = list(mu = 10L, n_points = 100L, sigma = 0.1) - - super$initialize( - id = "local_search", - param_set = param_set, - param_classes = c("ParamLgl", "ParamInt", "ParamDbl", "ParamFct"), - properties = c("dependencies", "single-crit"), # NOTE: think about multi-crit version - label = "Local Search", - man = "mlr3mbo::mlr_optimizers_local_search" - ) - } - ), - private = list( - .optimize = function(inst) { - mu = self$param_set$values$mu - mu_seq = seq_len(mu) - n_points = self$param_set$values$n_points - n_points_seq = seq_len(n_points) - sigma = self$param_set$values$sigma - - # if no reference points in archive, generate mu by sampling uniformly at random - if (inst$archive$n_evals < mu) { - data = generate_design_random(inst$search_space, n = mu - inst$archive$n_evals)$data - inst$eval_batch(data) - } - points = inst$archive$best(n_select = mu)[, c(inst$archive$cols_x, inst$archive$cols_y), with = FALSE] - - # we do not mutate parents of conditions - ids_to_mutate = setdiff(inst$search_space$ids(), unique(inst$search_space$deps$on)) - ids_numeric = intersect(inst$search_space$ids(class = c("ParamDbl", "ParamInt")), ids_to_mutate) - ids_categorical = intersect(inst$search_space$ids(class = c("ParamLgl", "ParamFct")), ids_to_mutate) - ids_categorical = intersect(ids_categorical, inst$search_space$ids()[inst$search_space$nlevels > 1]) - - point_id = ".point_id" - while (point_id %in% c(inst$archive$cols_x, inst$archive$cols_y)) { - point_id = paste0(".", point_id) - } - - repeat { # iterate until we have an exception from eval_batch - # generate neighbors - neighbors = map_dtr(mu_seq, function(i) { - neighbors_i = map_dtr(n_points_seq, function(j) { - # NOTE: mutating is currently quite slow because we sample the id to be mutated and the actual mutation for each neighbor and new point - mutate_point(points[i, inst$archive$cols_x, with = FALSE], search_space = inst$search_space, ids_numeric = ids_numeric, ids_categorical = ids_categorical, sigma = sigma) - }) - set(neighbors_i, j = point_id, value = i) - }) - - # evaluate neighbors - inst$eval_batch(neighbors) - - # update points if better neighbor found - for (i in mu_seq) { - tmp = inst$archive$data[batch_nr == inst$archive$n_batch & get(point_id) == i] - difference = (tmp[[inst$archive$cols_y]] * inst$objective_multiplicator) - (points[i, ][[inst$archive$cols_y]] * inst$objective_multiplicator) - if (any(difference < 0)) { - best = which.min(difference) - points[i, ] = tmp[best, c(inst$archive$cols_x, inst$archive$cols_y), with = FALSE] - } - } - } - } - ) -) - -mutate_point = function(point, search_space, ids_numeric, ids_categorical, sigma) { - neighbor = copy(point) - valid_numeric_to_mutate = intersect(names(which(!map_lgl(neighbor, is.na))), ids_numeric) - valid_cateorical_to_mutate = intersect(names(which(!map_lgl(neighbor, is.na))), ids_categorical) - id = sample(c(valid_numeric_to_mutate, valid_cateorical_to_mutate), size = 1L) - neighbor[1L, ][[id]] = mutate(neighbor[1L, ][[id]], subspace = search_space$subspaces(ids = id)[[1]], sigma = sigma) - neighbor -} - -mutate = function(value, subspace, sigma) { - if (subspace$class %in% c("ParamDbl", "ParamInt")) { - value_ = (value - subspace$lower) / (subspace$upper - subspace$lower) - value_ = max(0, min(stats::rnorm(1L, mean = value_, sd = sigma), 1)) - value = (value_ * (subspace$upper - subspace$lower)) + subspace$lower - if (subspace$class == "ParamInt") { - value = round(value, 0L) - } - value = min(max(value, subspace$lower), subspace$upper) - } else if (subspace$class %in% c("ParamFct", "ParamLgl")) { - value = sample(setdiff(subspace$levels[[1]], value), size = 1L) - } - value -} - -#' @include aaa.R -optimizers[["local_search"]] = OptimizerBatchLocalSearch diff --git a/R/OptimizerChain.R b/R/OptimizerChain.R deleted file mode 100644 index 32564292..00000000 --- a/R/OptimizerChain.R +++ /dev/null @@ -1,119 +0,0 @@ -#' @title Run Optimizers Sequentially -#' -#' @name mlr_optimizers_chain -#' -#' @description -#' `OptimizerBatchChain` allows to run different [Optimizer]s sequentially. -#' -#' For each [Optimizer] an (optional) additional [Terminator] can be specified -#' during construction. -#' While the initial [Terminator] guards the optimization process as a whole, -#' the additional [Terminator]s guard each individual [Optimizer]. -#' -#' The optimization then works as follows: -#' The first [Optimizer] is run on the [OptimInstance] using a [TerminatorCombo] -#' of the original [Terminator] of the [OptimInstance] and the (optional) -#' additional [Terminator] as passed during construction. -#' Once this [TerminatorCombo] indicates termination (usually via the -#' additional [Terminator]), the second [Optimizer] (and so on) is run unless -#' the original [Terminator] of the [OptimInstance] indicates termination. -#' -#' [OptimizerBatchChain] can also be used for random restarts of the same -#' [Optimizer] (if applicable) by setting the [Terminator] of the [OptimInstance] to -#' [TerminatorNone] and setting identical additional [Terminator]s during -#' construction. -#' -#' -#' @section Parameters: -#' Parameters are inherited from the individual [Optimizer]s and collected as a -#' [paradox::ParamSetCollection] (with `set_id`s potentially postfixed via `_1`, `_2`, -#' ..., if the same [Optimizer]s are used multiple times). -#' -#' -#' @export -OptimizerBatchChain = R6Class("OptimizerBatchChain", - inherit = bbotk::OptimizerBatch, - public = list( - - #' @description - #' Creates a new instance of this [R6][R6::R6Class] class. - #' - #' @param optimizers (list of [Optimizer]s). - #' @param terminators (list of [Terminator]s | NULL). - initialize = function(optimizers, terminators = rep(list(NULL), length(optimizers))) { - assert_list(optimizers, types = "Optimizer", any.missing = FALSE) - assert_list(terminators, types = c("Terminator", "NULL"), len = length(optimizers)) - - # param_sets = vector(mode = "list", length = length(optimizers)) - # ids_taken = character(0L) - # for each optimizer check whether the id of the param_set - # (decuded from the optimizer class) is already taken; - # if necessary postfix the id - # for (i_opt in seq_along(optimizers)) { - # opt = optimizers[[i_opt]] - # ps = opt$param_set$clone(deep = TRUE) - # ps$set_id = class(opt)[[1L]] - # try_postfix = 0L - # while (ps$set_id %in% ids_taken) { - # try_postfix = try_postfix + 1L - # ps$set_id = paste0(class(opt)[[1L]], "_", try_postfix) - # } - # ids_taken[[i_opt]] = ps$set_id - # param_sets[[i_opt]] = ps - # } - - param_sets = map(optimizers, function(x) x$param_set$clone(deep = TRUE)) - private$.ids = map_chr(optimizers, "id") - super$initialize( - param_set = ParamSetCollection$new(param_sets), - param_classes = Reduce(intersect, mlr3misc::map(optimizers, "param_classes")), - properties = Reduce(intersect, mlr3misc::map(optimizers, "properties")), - packages = unique(unlist(mlr3misc::map(optimizers, "packages"))) - ) - private$.optimizers = optimizers - private$.terminators = terminators - } - ), - private = list( - .optimizers = NULL, - .terminators = NULL, - .ids = NULL, - .optimize = function(inst) { - terminator = inst$terminator - on.exit({ - inst$terminator = terminator - }) - inner_inst = inst$clone(deep = TRUE) - - for (i_opt in seq_along(private$.optimizers)) { - inner_term = private$.terminators[[i_opt]] - if (!is.null(inner_term)) { - inner_inst$terminator = TerminatorCombo$new(list(inner_term, terminator)) - } else { - inner_inst$terminator = terminator - } - opt = private$.optimizers[[i_opt]] - opt$param_set$values = self$param_set$.__enclos_env__$private$.sets[[i_opt]]$values - opt$optimize(inner_inst) - inner_inst$archive$data$batch_nr = max(inst$archive$data$batch_nr, 0L) + - inner_inst$archive$data$batch_nr - inner_inst$archive$data$optimizer = private$.ids[i_opt] - inst$archive$data = rbind(inst$archive$data, inner_inst$archive$data, fill = TRUE) - inner_inst$archive$data = data.table() - if (terminator$is_terminated(inst$archive)) { - break - } - } - }, - deep_clone = function(name, value) { - switch(name, - .optimizers = mlr3misc::map(value, .f = function(x) x$clone(deep = TRUE)), - .terminators = mlr3misc::map(value, .f = function(x) if (!is.null(x)) x$clone(deep = TRUE)), - value - ) - } - ) -) - -#' @include aaa.R -optimizers[["chain"]] = OptimizerBatchChain diff --git a/man/mlr_optimizers_chain.Rd b/man/mlr_optimizers_chain.Rd deleted file mode 100644 index 3adccc8f..00000000 --- a/man/mlr_optimizers_chain.Rd +++ /dev/null @@ -1,94 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/OptimizerChain.R -\name{mlr_optimizers_chain} -\alias{mlr_optimizers_chain} -\alias{OptimizerBatchChain} -\title{Run Optimizers Sequentially} -\description{ -\code{OptimizerBatchChain} allows to run different \link{Optimizer}s sequentially. - -For each \link{Optimizer} an (optional) additional \link{Terminator} can be specified -during construction. -While the initial \link{Terminator} guards the optimization process as a whole, -the additional \link{Terminator}s guard each individual \link{Optimizer}. - -The optimization then works as follows: -The first \link{Optimizer} is run on the \link{OptimInstance} using a \link{TerminatorCombo} -of the original \link{Terminator} of the \link{OptimInstance} and the (optional) -additional \link{Terminator} as passed during construction. -Once this \link{TerminatorCombo} indicates termination (usually via the -additional \link{Terminator}), the second \link{Optimizer} (and so on) is run unless -the original \link{Terminator} of the \link{OptimInstance} indicates termination. - -\link{OptimizerBatchChain} can also be used for random restarts of the same -\link{Optimizer} (if applicable) by setting the \link{Terminator} of the \link{OptimInstance} to -\link{TerminatorNone} and setting identical additional \link{Terminator}s during -construction. -} -\section{Parameters}{ - -Parameters are inherited from the individual \link{Optimizer}s and collected as a -\link[paradox:ParamSetCollection]{paradox::ParamSetCollection} (with \code{set_id}s potentially postfixed via \verb{_1}, \verb{_2}, -..., if the same \link{Optimizer}s are used multiple times). -} - -\section{Super classes}{ -\code{\link[bbotk:Optimizer]{bbotk::Optimizer}} -> \code{\link[bbotk:OptimizerBatch]{bbotk::OptimizerBatch}} -> \code{OptimizerBatchChain} -} -\section{Methods}{ -\subsection{Public methods}{ -\itemize{ -\item \href{#method-OptimizerBatchChain-new}{\code{OptimizerBatchChain$new()}} -\item \href{#method-OptimizerBatchChain-clone}{\code{OptimizerBatchChain$clone()}} -} -} -\if{html}{\out{ -
    Inherited methods - -
    -}} -\if{html}{\out{
    }} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-OptimizerBatchChain-new}{}}} -\subsection{Method \code{new()}}{ -Creates a new instance of this \link[R6:R6Class]{R6} class. -\subsection{Usage}{ -\if{html}{\out{
    }}\preformatted{OptimizerBatchChain$new( - optimizers, - terminators = rep(list(NULL), length(optimizers)) -)}\if{html}{\out{
    }} -} - -\subsection{Arguments}{ -\if{html}{\out{
    }} -\describe{ -\item{\code{optimizers}}{(list of \link{Optimizer}s).} - -\item{\code{terminators}}{(list of \link{Terminator}s | NULL).} -} -\if{html}{\out{
    }} -} -} -\if{html}{\out{
    }} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-OptimizerBatchChain-clone}{}}} -\subsection{Method \code{clone()}}{ -The objects of this class are cloneable with this method. -\subsection{Usage}{ -\if{html}{\out{
    }}\preformatted{OptimizerBatchChain$clone(deep = FALSE)}\if{html}{\out{
    }} -} - -\subsection{Arguments}{ -\if{html}{\out{
    }} -\describe{ -\item{\code{deep}}{Whether to make a deep clone.} -} -\if{html}{\out{
    }} -} -} -} diff --git a/man/mlr_optimizers_local_search.Rd b/man/mlr_optimizers_local_search.Rd deleted file mode 100644 index 4a4f368f..00000000 --- a/man/mlr_optimizers_local_search.Rd +++ /dev/null @@ -1,88 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/OptimizerBatchLocalSearch.R -\name{mlr_optimizers_local_search} -\alias{mlr_optimizers_local_search} -\alias{OptimizerBatchLocalSearch} -\title{Optimization via Local Search} -\description{ -\code{OptimizerBatchLocalSearch} class that implements a simple Local Search. -Local Search starts by determining the \code{mu} initial best points present in the \link{Archive} of the -\link{OptimInstance}. If fewer points than \code{mu} are present, additional points sampled uniformly at -random are evaluated. - -In each iteration, for each of the \code{mu} initial best points, \code{n_points} neighbors are generated -by local mutation. Local mutation generates a neighbor by sampling a single parameter that is to -be mutated and then proceeds as follows: Double parameters (\link[paradox:Domain]{paradox::p_dbl}) are mutated via -Gaussian mutation (with a prior standardization to \verb{[0, 1]} and retransformation after mutation). -Integer parameters (\link[paradox:Domain]{paradox::p_int}) undergo the same mutation but are rounded to the closest -integer after mutation. Categorical parameters (\link[paradox:Domain]{paradox::p_fct} and \link[paradox:Domain]{paradox::p_lgl}) are -mutated via uniform mutation. Note that parameters that are conditioned on (i.e., they are -parents of a \link[paradox:Condition]{paradox::Condition}, see the dependencies of the search space) are not mutated. -} -\section{Parameters}{ - -\describe{ -\item{\code{mu}}{\code{integer(1)}\cr -Size of the initial best points which are used as a starting points for the Local Search. -Default is \code{10}. -} -\item{\code{n_points}}{\code{integer(1)}\cr -Number of neighboring points to generate for each of the \code{mu} best starting points in each -iteration. -Default is \code{100}. -} -\item{\code{sigma}}{\code{numeric(1)}\cr -Standard deviation used for mutation of numeric parameters. -Default is \code{0.1}. -} -} -} - -\section{Super classes}{ -\code{\link[bbotk:Optimizer]{bbotk::Optimizer}} -> \code{\link[bbotk:OptimizerBatch]{bbotk::OptimizerBatch}} -> \code{OptimizerBatchLocalSearch} -} -\section{Methods}{ -\subsection{Public methods}{ -\itemize{ -\item \href{#method-OptimizerBatchLocalSearch-new}{\code{OptimizerBatchLocalSearch$new()}} -\item \href{#method-OptimizerBatchLocalSearch-clone}{\code{OptimizerBatchLocalSearch$clone()}} -} -} -\if{html}{\out{ -
    Inherited methods - -
    -}} -\if{html}{\out{
    }} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-OptimizerBatchLocalSearch-new}{}}} -\subsection{Method \code{new()}}{ -Creates a new instance of this \link[R6:R6Class]{R6} class. -\subsection{Usage}{ -\if{html}{\out{
    }}\preformatted{OptimizerBatchLocalSearch$new()}\if{html}{\out{
    }} -} - -} -\if{html}{\out{
    }} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-OptimizerBatchLocalSearch-clone}{}}} -\subsection{Method \code{clone()}}{ -The objects of this class are cloneable with this method. -\subsection{Usage}{ -\if{html}{\out{
    }}\preformatted{OptimizerBatchLocalSearch$clone(deep = FALSE)}\if{html}{\out{
    }} -} - -\subsection{Arguments}{ -\if{html}{\out{
    }} -\describe{ -\item{\code{deep}}{Whether to make a deep clone.} -} -\if{html}{\out{
    }} -} -} -} From d858ea26dfb3e62d7d6920f7e6976dc9501df831 Mon Sep 17 00:00:00 2001 From: be-marc Date: Tue, 13 Aug 2024 19:28:38 +0200 Subject: [PATCH 074/126] feat: add epsilon to log ei --- R/AcqFunctionLogEI.R | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/R/AcqFunctionLogEI.R b/R/AcqFunctionLogEI.R index ed022cb1..27dd8720 100644 --- a/R/AcqFunctionLogEI.R +++ b/R/AcqFunctionLogEI.R @@ -24,9 +24,14 @@ AcqFunctionLogEI = R6Class("AcqFunctionLogEI", #' Creates a new instance of this [R6][R6::R6Class] class. #' #' @param surrogate (`NULL` | [SurrogateLearner]). - initialize = function(surrogate = NULL) { + initialize = function(surrogate = NULL, epsilon = 0) { assert_r6(surrogate, "SurrogateLearner", null.ok = TRUE) - super$initialize("acq_log_ei", surrogate = surrogate, requires_predict_type_se = TRUE, direction = "maximize", label = "Log Expected Improvement", man = "mlr3mbo::mlr_acqfunctions_log_ei") + assert_number(epsilon, lower = 0, finite = TRUE) + + constants = ps(epsilon = p_dbl(lower = 0, default = 0)) + constants$values$epsilon = epsilon + + super$initialize("acq_log_ei", constants = constants, surrogate = surrogate, requires_predict_type_se = TRUE, direction = "maximize", label = "Log Expected Improvement", man = "mlr3mbo::mlr_acqfunctions_log_ei") }, #' @description @@ -39,17 +44,20 @@ AcqFunctionLogEI = R6Class("AcqFunctionLogEI", } ), private = list( - .fun = function(xdt) { + .fun = function(xdt, ...) { if (is.null(self$y_best)) { stop("$y_best is not set. Missed to call $update()?") } if (self$surrogate_max_to_min != 1L) { stop("Log EI assumes minimization of the log transformed target value.") } + constants = list(...) + epsilon = constants$epsilon p = self$surrogate$predict(xdt) mu = p$mean se = p$se - d_norm = (self$y_best - mu) / se + d = self$y_best - mu - epsilon + d_norm = d / se log_ei = (exp(self$y_best) * pnorm(d_norm)) - (exp((0.5 * se^2) + mu) * pnorm(d_norm - se)) log_ei = ifelse(se < 1e-20, 0, log_ei) data.table(acq_log_ei = log_ei) From c5a3670f019dffff31227d7d2463e84a74bfa5a1 Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Tue, 13 Aug 2024 20:30:12 +0200 Subject: [PATCH 075/126] .. --- R/AcqFunctionLogEI.R | 14 ++++++++++++-- man/mlr_acqfunctions_log_ei.Rd | 15 ++++++++++++++- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/R/AcqFunctionLogEI.R b/R/AcqFunctionLogEI.R index 27dd8720..60a66322 100644 --- a/R/AcqFunctionLogEI.R +++ b/R/AcqFunctionLogEI.R @@ -9,10 +9,18 @@ #' @description #' Expected Improvement assuming that the target variable is to be minimized and has been modeled on log scale. #' +#' @section Parameters: +#' * `"epsilon"` (`numeric(1)`)\cr +#' \eqn{\epsilon} value used to determine the amount of exploration. +#' Higher values result in the importance of improvements predicted by the posterior mean +#' decreasing relative to the importance of potential improvements in regions of high predictive uncertainty. +#' Defaults to `0` (standard Expected Improvement). +#' #' @family Acquisition Function #' @export AcqFunctionLogEI = R6Class("AcqFunctionLogEI", inherit = AcqFunction, + public = list( #' @field y_best (`numeric(1)`)\cr @@ -24,6 +32,7 @@ AcqFunctionLogEI = R6Class("AcqFunctionLogEI", #' Creates a new instance of this [R6][R6::R6Class] class. #' #' @param surrogate (`NULL` | [SurrogateLearner]). + #' @param epsilon (`numeric(1)`). initialize = function(surrogate = NULL, epsilon = 0) { assert_r6(surrogate, "SurrogateLearner", null.ok = TRUE) assert_number(epsilon, lower = 0, finite = TRUE) @@ -40,9 +49,10 @@ AcqFunctionLogEI = R6Class("AcqFunctionLogEI", if (self$surrogate_max_to_min != 1L) { stop("Log EI assumes minimization of the log transformed target value.") } - self$y_best = min(self$surrogate_max_to_min * self$archive$data[[self$surrogate$cols_y]]) + self$y_best = min(self$archive$data[[self$surrogate$cols_y]]) } ), + private = list( .fun = function(xdt, ...) { if (is.null(self$y_best)) { @@ -56,7 +66,7 @@ AcqFunctionLogEI = R6Class("AcqFunctionLogEI", p = self$surrogate$predict(xdt) mu = p$mean se = p$se - d = self$y_best - mu - epsilon + d = (self$y_best - mu) - epsilon d_norm = d / se log_ei = (exp(self$y_best) * pnorm(d_norm)) - (exp((0.5 * se^2) + mu) * pnorm(d_norm - se)) log_ei = ifelse(se < 1e-20, 0, log_ei) diff --git a/man/mlr_acqfunctions_log_ei.Rd b/man/mlr_acqfunctions_log_ei.Rd index 3aa53e80..771e469c 100644 --- a/man/mlr_acqfunctions_log_ei.Rd +++ b/man/mlr_acqfunctions_log_ei.Rd @@ -17,6 +17,17 @@ acqf("log_ei") }\if{html}{\out{
    }} } +\section{Parameters}{ + +\itemize{ +\item \code{"epsilon"} (\code{numeric(1)})\cr +\eqn{\epsilon} value used to determine the amount of exploration. +Higher values result in the importance of improvements predicted by the posterior mean +decreasing relative to the importance of potential improvements in regions of high predictive uncertainty. +Defaults to \code{0} (standard Expected Improvement). +} +} + \seealso{ Other Acquisition Function: \code{\link{AcqFunction}}, @@ -71,13 +82,15 @@ On log scale.} \subsection{Method \code{new()}}{ Creates a new instance of this \link[R6:R6Class]{R6} class. \subsection{Usage}{ -\if{html}{\out{
    }}\preformatted{AcqFunctionLogEI$new(surrogate = NULL)}\if{html}{\out{
    }} +\if{html}{\out{
    }}\preformatted{AcqFunctionLogEI$new(surrogate = NULL, epsilon = 0)}\if{html}{\out{
    }} } \subsection{Arguments}{ \if{html}{\out{
    }} \describe{ \item{\code{surrogate}}{(\code{NULL} | \link{SurrogateLearner}).} + +\item{\code{epsilon}}{(\code{numeric(1)}).} } \if{html}{\out{
    }} } From d57e6aef1f6296f89a85dd2bd4d527ccdf14d9bd Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Sat, 17 Aug 2024 14:55:51 +0200 Subject: [PATCH 076/126] feat: rework LearnerRegrRangerMbo --- ...rRangerCustom.R => LearnerRegrRangerMbo.R} | 54 +++++++++++++++---- 1 file changed, 45 insertions(+), 9 deletions(-) rename R/{LearnerRegrRangerCustom.R => LearnerRegrRangerMbo.R} (68%) diff --git a/R/LearnerRegrRangerCustom.R b/R/LearnerRegrRangerMbo.R similarity index 68% rename from R/LearnerRegrRangerCustom.R rename to R/LearnerRegrRangerMbo.R index 5d5c0011..3e4ac955 100644 --- a/R/LearnerRegrRangerCustom.R +++ b/R/LearnerRegrRangerMbo.R @@ -1,4 +1,4 @@ -#' @title Custom Ranger Regression Learner +#' @title Custom Ranger Regression Learner for Model Based Optimization #' #' @name mlr_learners_regr.ranger #' @@ -37,7 +37,7 @@ LearnerRegrRangerMbo = R6Class("LearnerRegrRangerMbo", sample.fraction = p_dbl(0L, 1L, tags = "train"), save.memory = p_lgl(default = FALSE, tags = "train"), scale.permutation.importance = p_lgl(default = FALSE, tags = "train", depends = quote(importance == "permutation")), - se.method = p_fct(c("jack", "infjack", "simple"), default = "infjack", tags = "predict"), # FIXME: only works if predict_type == "se" + se.method = p_fct(c("jack", "infjack", "simple"), default = "infjack", tags = c("train", "predict")), # FIXME: only works if predict_type == "se" seed = p_int(default = NULL, special_vals = list(NULL), tags = c("train", "predict")), split.select.weights = p_uty(default = NULL, tags = "train"), splitrule = p_fct(c("variance", "extratrees", "maxstat"), default = "variance", tags = "train"), @@ -91,29 +91,66 @@ LearnerRegrRangerMbo = R6Class("LearnerRegrRangerMbo", .train = function(task) { pv = self$param_set$get_values(tags = "train") pv = mlr3learners:::convert_ratio(pv, "mtry", "mtry.ratio", length(task$feature_names)) + if ("se.method" %in% names(pv)) { + if (self$predict_type != "se") { + stop('$predict_type must be "se" if "se.method" is set to "simple".') + } + pv[["se.method"]] = NULL + } if (self$predict_type == "se") { pv$keep.inbag = TRUE # nolint } - mlr3misc::invoke(ranger::ranger, + model = mlr3misc::invoke(ranger::ranger, dependent.variable.name = task$target_names, data = task$data(), case.weights = task$weights$weight, .args = pv ) + + if (isTRUE(self$param_set$get_values()[["se.method"]] == "simple")) { + data = mlr3learners:::ordered_features(task, self) + prediction_nodes = mlr3misc::invoke(predict, model, data = data, type = "terminalNodes", .args = pv[setdiff(names(pv), "se.method")], predict.all = TRUE) + y = task$data(cols = task$target_names)[[1L]] + observation_node_table = prediction_nodes$predictions + n_trees = NCOL(observation_node_table) + unique_nodes_per_tree = apply(observation_node_table, MARGIN = 2L, FUN = unique, simplify = FALSE) + mu_sigma2_per_node_per_tree = lapply(seq_len(n_trees), function(tree) { + nodes = unique_nodes_per_tree[[tree]] + setNames(lapply(nodes, function(node) { + y_tmp = y[observation_node_table[, tree] == node] + c(mu = mean(y_tmp), sigma2 = if (length(y_tmp) > 1L) var(y_tmp) else 0) + }), nm = nodes) + }) + list(model = model, mu_sigma2_per_node_per_tree = mu_sigma2_per_node_per_tree) + } else { + list(model = model) + } }, .predict = function(task) { pv = self$param_set$get_values(tags = "predict") newdata = mlr3learners:::ordered_features(task, self) if (isTRUE(pv$se.method == "simple")) { - prediction = mlr3misc::invoke(predict, self$model, data = newdata, type = "response", .args = pv[setdiff(names(pv), "se.method")], predict.all = TRUE) - response = rowMeans(prediction$predictions) - variance = rowMeans(0.01 + prediction$predictions^2) - (response^2) # law of total variance assuming sigma_b(x) is 0 due to min.node.size = 1 and always splitting; then set 0.01 as lower bound for sigma_b(x) - list(response = response, se = sqrt(variance)) + prediction_nodes = mlr3misc::invoke(predict, self$model$model, data = newdata, type = "terminalNodes", .args = pv[setdiff(names(pv), "se.method")], predict.all = TRUE) + n_observations = NROW(prediction_nodes$predictions) + n_trees = length(self$model$mu_sigma2_per_node_per_tree) + response = numeric(n_observations) + se = numeric(n_observations) + for (i in seq_len(n_observations)) { + mu_sigma2_per_tree = lapply(seq_len(n_trees), function(tree) { + self$model$mu_sigma2_per_node_per_tree[[tree]][[as.character(prediction_nodes$predictions[i, tree])]] + }) + mus = sapply(mu_sigma2_per_tree, "[[", 1) + sigmas2 = sapply(mu_sigma2_per_tree, "[[", 2) + response[i] = mean(mus) + # law of total variance assuming a mixture of normal distributions for each tree + se[i] = sqrt(mean((mus ^ 2) + sigmas2) - (response[i] ^ 2)) + } + list(response = response, se = se) } else { - prediction = mlr3misc::invoke(predict, self$model, data = newdata, type = self$predict_type, .args = pv) + prediction = mlr3misc::invoke(predict, self$model$model, data = newdata, type = self$predict_type, .args = pv) list(response = prediction$predictions, se = prediction$se) } }, @@ -139,4 +176,3 @@ default_values.LearnerRegrRangerMbo = function(x, search_space, task, ...) { # n #' @include aaa.R learners[["regr.ranger_mbo"]] = LearnerRegrRangerMbo - From ca0305954de61346b168c064d02d0d34bf350e51 Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Sat, 17 Aug 2024 16:42:56 +0200 Subject: [PATCH 077/126] .. --- DESCRIPTION | 2 +- R/LearnerRegrRangerMbo.R | 25 +++++++++++++++++++++---- man/mlr_learners_regr.ranger.Rd | 4 ++-- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index ce249ed6..bfe00dad 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -90,7 +90,7 @@ Collate: 'AcqFunctionSmsEgo.R' 'AcqOptimizer.R' 'aaa.R' - 'LearnerRegrRangerCustom.R' + 'LearnerRegrRangerMbo.R' 'OptimizerMbo.R' 'mlr_result_assigners.R' 'ResultAssigner.R' diff --git a/R/LearnerRegrRangerMbo.R b/R/LearnerRegrRangerMbo.R index 3e4ac955..2dc3c973 100644 --- a/R/LearnerRegrRangerMbo.R +++ b/R/LearnerRegrRangerMbo.R @@ -21,10 +21,12 @@ LearnerRegrRangerMbo = R6Class("LearnerRegrRangerMbo", importance = p_fct(c("none", "impurity", "impurity_corrected", "permutation"), tags = "train"), keep.inbag = p_lgl(default = FALSE, tags = "train"), max.depth = p_int(default = NULL, lower = 0L, special_vals = list(NULL), tags = "train"), + min.bucket = p_int(1L, default = 1L, tags = "train"), min.node.size = p_int(1L, default = 5L, special_vals = list(NULL), tags = "train"), minprop = p_dbl(default = 0.1, tags = "train", depends = quote(splitrule == "maxstat")), mtry = p_int(lower = 1L, special_vals = list(NULL), tags = "train"), mtry.ratio = p_dbl(lower = 0, upper = 1, tags = "train"), + node.stats = p_lgl(default = FALSE, tags = "train"), num.random.splits = p_int(1L, default = 1L, tags = "train", depends = quote(splitrule == "extratrees")), num.threads = p_int(1L, default = 1L, tags = c("train", "predict", "threads")), num.trees = p_int(1L, default = 500L, tags = c("train", "predict", "hotstart")), @@ -37,7 +39,7 @@ LearnerRegrRangerMbo = R6Class("LearnerRegrRangerMbo", sample.fraction = p_dbl(0L, 1L, tags = "train"), save.memory = p_lgl(default = FALSE, tags = "train"), scale.permutation.importance = p_lgl(default = FALSE, tags = "train", depends = quote(importance == "permutation")), - se.method = p_fct(c("jack", "infjack", "simple"), default = "infjack", tags = c("train", "predict")), # FIXME: only works if predict_type == "se" + se.method = p_fct(c("jack", "infjack", "simple", "law_of_total_variance"), default = "infjack", tags = c("train", "predict")), # FIXME: only works if predict_type == "se" seed = p_int(default = NULL, special_vals = list(NULL), tags = c("train", "predict")), split.select.weights = p_uty(default = NULL, tags = "train"), splitrule = p_fct(c("variance", "extratrees", "maxstat"), default = "variance", tags = "train"), @@ -93,7 +95,7 @@ LearnerRegrRangerMbo = R6Class("LearnerRegrRangerMbo", pv = mlr3learners:::convert_ratio(pv, "mtry", "mtry.ratio", length(task$feature_names)) if ("se.method" %in% names(pv)) { if (self$predict_type != "se") { - stop('$predict_type must be "se" if "se.method" is set to "simple".') + stop('$predict_type must be "se" if "se.method" is set to "simple" or "law_of_total_variance".') } pv[["se.method"]] = NULL } @@ -109,7 +111,7 @@ LearnerRegrRangerMbo = R6Class("LearnerRegrRangerMbo", .args = pv ) - if (isTRUE(self$param_set$get_values()[["se.method"]] == "simple")) { + if (isTRUE(self$param_set$get_values()[["se.method"]] %in% c("simple", "law_of_total_variance"))) { data = mlr3learners:::ordered_features(task, self) prediction_nodes = mlr3misc::invoke(predict, model, data = data, type = "terminalNodes", .args = pv[setdiff(names(pv), "se.method")], predict.all = TRUE) y = task$data(cols = task$target_names)[[1L]] @@ -132,7 +134,7 @@ LearnerRegrRangerMbo = R6Class("LearnerRegrRangerMbo", pv = self$param_set$get_values(tags = "predict") newdata = mlr3learners:::ordered_features(task, self) - if (isTRUE(pv$se.method == "simple")) { + if (isTRUE(pv$se.method == "law_of_total_variance")) { prediction_nodes = mlr3misc::invoke(predict, self$model$model, data = newdata, type = "terminalNodes", .args = pv[setdiff(names(pv), "se.method")], predict.all = TRUE) n_observations = NROW(prediction_nodes$predictions) n_trees = length(self$model$mu_sigma2_per_node_per_tree) @@ -149,6 +151,21 @@ LearnerRegrRangerMbo = R6Class("LearnerRegrRangerMbo", se[i] = sqrt(mean((mus ^ 2) + sigmas2) - (response[i] ^ 2)) } list(response = response, se = se) + } else if (isTRUE(pv$se.method == "simple")) { + prediction_nodes = mlr3misc::invoke(predict, self$model$model, data = newdata, type = "terminalNodes", .args = pv[setdiff(names(pv), "se.method")], predict.all = TRUE) + n_observations = NROW(prediction_nodes$predictions) + n_trees = length(self$model$mu_sigma2_per_node_per_tree) + response = numeric(n_observations) + se = numeric(n_observations) + for (i in seq_len(n_observations)) { + mu_sigma2_per_tree = lapply(seq_len(n_trees), function(tree) { + self$model$mu_sigma2_per_node_per_tree[[tree]][[as.character(prediction_nodes$predictions[i, tree])]] + }) + mus = sapply(mu_sigma2_per_tree, "[[", 1) + response[i] = mean(mus) + se[i] = sqrt(var(mus)) + } + list(response = response, se = se) } else { prediction = mlr3misc::invoke(predict, self$model$model, data = newdata, type = self$predict_type, .args = pv) list(response = prediction$predictions, se = prediction$se) diff --git a/man/mlr_learners_regr.ranger.Rd b/man/mlr_learners_regr.ranger.Rd index 20bdeb38..2bfad3f8 100644 --- a/man/mlr_learners_regr.ranger.Rd +++ b/man/mlr_learners_regr.ranger.Rd @@ -1,9 +1,9 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/LearnerRegrRangerCustom.R +% Please edit documentation in R/LearnerRegrRangerMbo.R \name{mlr_learners_regr.ranger} \alias{mlr_learners_regr.ranger} \alias{LearnerRegrRangerMbo} -\title{Custom Ranger Regression Learner} +\title{Custom Ranger Regression Learner for Model Based Optimization} \description{ Random regression forest. Calls \code{\link[ranger:ranger]{ranger::ranger()}} from package \CRANpkg{ranger}. From c572a14a5aecc7c089f517f5f5c9495ee204abd8 Mon Sep 17 00:00:00 2001 From: be-marc Date: Mon, 26 Aug 2024 12:24:04 +0200 Subject: [PATCH 078/126] fix: make log ei more robost --- R/AcqFunctionLogEI.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/AcqFunctionLogEI.R b/R/AcqFunctionLogEI.R index 60a66322..a901d584 100644 --- a/R/AcqFunctionLogEI.R +++ b/R/AcqFunctionLogEI.R @@ -69,7 +69,7 @@ AcqFunctionLogEI = R6Class("AcqFunctionLogEI", d = (self$y_best - mu) - epsilon d_norm = d / se log_ei = (exp(self$y_best) * pnorm(d_norm)) - (exp((0.5 * se^2) + mu) * pnorm(d_norm - se)) - log_ei = ifelse(se < 1e-20, 0, log_ei) + log_ei = ifelse(se < 1e-20 | is.na(log_ei), 0, log_ei) data.table(acq_log_ei = log_ei) } ) From 58f1e3a44eb11577326ad44c38d73269c065713a Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Mon, 2 Sep 2024 20:24:09 +0200 Subject: [PATCH 079/126] se >= 1e-8 for LearnerRegrRangerMbo and law of total variance --- R/LearnerRegrRangerMbo.R | 1 + 1 file changed, 1 insertion(+) diff --git a/R/LearnerRegrRangerMbo.R b/R/LearnerRegrRangerMbo.R index 2dc3c973..69fb04a9 100644 --- a/R/LearnerRegrRangerMbo.R +++ b/R/LearnerRegrRangerMbo.R @@ -150,6 +150,7 @@ LearnerRegrRangerMbo = R6Class("LearnerRegrRangerMbo", # law of total variance assuming a mixture of normal distributions for each tree se[i] = sqrt(mean((mus ^ 2) + sigmas2) - (response[i] ^ 2)) } + se[se < .Machine$double.eps] = 1e-8 list(response = response, se = se) } else if (isTRUE(pv$se.method == "simple")) { prediction_nodes = mlr3misc::invoke(predict, self$model$model, data = newdata, type = "terminalNodes", .args = pv[setdiff(names(pv), "se.method")], predict.all = TRUE) From f44a741dc8a0af2ee53e6cce52703893731c6a8a Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Mon, 2 Sep 2024 20:26:31 +0200 Subject: [PATCH 080/126] .. --- R/LearnerRegrRangerMbo.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/LearnerRegrRangerMbo.R b/R/LearnerRegrRangerMbo.R index 69fb04a9..a3943365 100644 --- a/R/LearnerRegrRangerMbo.R +++ b/R/LearnerRegrRangerMbo.R @@ -150,7 +150,7 @@ LearnerRegrRangerMbo = R6Class("LearnerRegrRangerMbo", # law of total variance assuming a mixture of normal distributions for each tree se[i] = sqrt(mean((mus ^ 2) + sigmas2) - (response[i] ^ 2)) } - se[se < .Machine$double.eps] = 1e-8 + se[se < .Machine$double.eps | is.na(se)] = 1e-8 list(response = response, se = se) } else if (isTRUE(pv$se.method == "simple")) { prediction_nodes = mlr3misc::invoke(predict, self$model$model, data = newdata, type = "terminalNodes", .args = pv[setdiff(names(pv), "se.method")], predict.all = TRUE) From ce2e16f7a5498d2aa434ce034af64a878d25d853 Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Tue, 19 Nov 2024 15:23:53 +0100 Subject: [PATCH 081/126] perf: speed up surrogate predictions --- R/SurrogateLearner.R | 12 +++++++++++- R/SurrogateLearnerCollection.R | 19 ++++++++++++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/R/SurrogateLearner.R b/R/SurrogateLearner.R index 18abc334..0891833e 100644 --- a/R/SurrogateLearner.R +++ b/R/SurrogateLearner.R @@ -113,7 +113,17 @@ SurrogateLearner = R6Class("SurrogateLearner", assert_xdt(xdt) xdt = fix_xdt_missing(xdt, cols_x = self$cols_x, archive = self$archive) - pred = self$learner$predict_newdata(newdata = xdt) + # speeding up some checks by constructing the predict task directly instead of relying on predict_newdata + task = self$learner$state$train_task$clone() + set(xdt, j = task$target_names, value = NA_real_) # tasks only have features and the target but we have to set the target to NA + newdata = as_data_backend(xdt) + task$backend = newdata + task$row_roles$use = task$backend$rownames + pred = self$learner$predict(task) + + # slow + #pred = self$learner$predict_newdata(newdata = xdt) + if (self$learner$predict_type == "se") { data.table(mean = pred$response, se = pred$se) } else { diff --git a/R/SurrogateLearnerCollection.R b/R/SurrogateLearnerCollection.R index 74aacff4..34684c62 100644 --- a/R/SurrogateLearnerCollection.R +++ b/R/SurrogateLearnerCollection.R @@ -130,14 +130,31 @@ SurrogateLearnerCollection = R6Class("SurrogateLearnerCollection", assert_xdt(xdt) xdt = fix_xdt_missing(xdt, cols_x = self$cols_x, archive = self$archive) + # speeding up some checks by constructing the predict task directly instead of relying on predict_newdata preds = lapply(self$learner, function(learner) { - pred = learner$predict_newdata(newdata = xdt) + task = learner$state$train_task$clone() + set(xdt, j = task$target_names, value = NA_real_) # tasks only have features and the target but we have to set the target to NA + newdata = as_data_backend(xdt) + task$backend = newdata + task$row_roles$use = task$backend$rownames + pred = learner$predict(task) if (learner$predict_type == "se") { data.table(mean = pred$response, se = pred$se) } else { data.table(mean = pred$response) } }) + + # slow + #preds = lapply(self$learner, function(learner) { + # pred = learner$predict_newdata(newdata = xdt) + # if (learner$predict_type == "se") { + # data.table(mean = pred$response, se = pred$se) + # } else { + # data.table(mean = pred$response) + # } + #}) + names(preds) = names(self$learner) preds } From 9992c6613e795206f3c5850137ebf8bee99f332e Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Tue, 19 Nov 2024 18:21:10 +0100 Subject: [PATCH 082/126] draft SurrogateGP --- DESCRIPTION | 1 + NAMESPACE | 1 + R/SurrogateGP.R | 298 +++++++++++++++++++++++++++++++++ R/SurrogateLearner.R | 4 +- R/SurrogateLearnerCollection.R | 4 +- man/SurrogateGP.Rd | 187 +++++++++++++++++++++ 6 files changed, 491 insertions(+), 4 deletions(-) create mode 100644 R/SurrogateGP.R create mode 100644 man/SurrogateGP.Rd diff --git a/DESCRIPTION b/DESCRIPTION index 984b0752..4de51e66 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -99,6 +99,7 @@ Collate: 'ResultAssignerArchive.R' 'ResultAssignerSurrogate.R' 'Surrogate.R' + 'SurrogateGP.R' 'SurrogateLearner.R' 'SurrogateLearnerCollection.R' 'TunerADBO.R' diff --git a/NAMESPACE b/NAMESPACE index e5b96257..57ca084a 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -26,6 +26,7 @@ export(ResultAssigner) export(ResultAssignerArchive) export(ResultAssignerSurrogate) export(Surrogate) +export(SurrogateGP) export(SurrogateLearner) export(SurrogateLearnerCollection) export(TunerADBO) diff --git a/R/SurrogateGP.R b/R/SurrogateGP.R new file mode 100644 index 00000000..87128c8c --- /dev/null +++ b/R/SurrogateGP.R @@ -0,0 +1,298 @@ +#' @title Surrogate Model Containing a Gaussian Process +#' +#' @description +#' Surrogate model containing a single Gaussian Process via [DiceKriging::km()] from package \CRANpkg{DiceKriging}. +#' Update and predict methods are inspired from [mlr3learners::LearnerRegrKM] from package \CRANpkg{mlr3learners}. +#' +#' Compared to using [mlr3learners::LearnerRegrKM] within a [SurrogateLeaner] the update and predict methods of this class are much more efficient +#' as they skip many assertions and checks naturally arising when using a [SurrogateLeaner] wrapping a [mlr3learners::LearnerRegrKM]. +#' +#' @section Parameters: +#' \describe{ +#' \item{`catch_errors`}{`logical(1)`\cr +#' Should errors during updating the surrogate be caught and propagated to the `loop_function` which can then handle +#' the failed acquisition function optimization (as a result of the failed surrogate) appropriately by, e.g., proposing a randomly sampled point for evaluation? +#' Default is `TRUE`. +#' } +#' \item{`impute_method`}{`character(1)`\cr +#' Method to impute missing values in the case of updating on an asynchronous [bbotk::ArchiveAsync] with pending evaluations. +#' Can be `"mean"` to use mean imputation or `"random"` to sample values uniformly at random between the empirical minimum and maximum. +#' Default is `"random"`. +#' } +#' } +#' For a description of all other parameters related to [DiceKriging::km()] directly, see the documentation of [DiceKriging::km()]. +#' * The predict type hyperparameter "type" defaults to "SK" (simple kriging). +#' * The additional hyperparameter `nugget.stability` is used to overwrite the +#' hyperparameter `nugget` with `nugget.stability * var(y)` before training to +#' improve the numerical stability. We recommend a value of `1e-8`. +#' * The additional hyperparameter `jitter` can be set to add +#' `N(0, [jitter])`-distributed noise to the data before prediction to avoid +#' perfect interpolation. We recommend a value of `1e-12`. +#' +#' @export +#' @examples +#' if (requireNamespace("DiceKriging") & +#' requireNamespace("rgenoud")) { +#' library(bbotk) +#' library(paradox) +#' +#' fun = function(xs) { +#' list(y = xs$x ^ 2) +#' } +#' domain = ps(x = p_dbl(lower = -10, upper = 10)) +#' codomain = ps(y = p_dbl(tags = "minimize")) +#' objective = ObjectiveRFun$new(fun = fun, domain = domain, codomain = codomain) +#' +#' instance = OptimInstanceBatchSingleCrit$new( +#' objective = objective, +#' terminator = trm("evals", n_evals = 5)) +#' +#' xdt = generate_design_random(instance$search_space, n = 4)$data +#' +#' instance$eval_batch(xdt) +#' +#' surrogate = SurrogateGP$new(archive = instance$archive) +#' surrogate$param_set$set_values( +#' covtype = "matern5_2", +#' optim.method = "gen", +#' control = list(trace = FALSE), +#' nugget.stability = 10^-8 +#' ) +#' +#' surrogate$update() +#' +#' surrogate$learner$model +#' } +SurrogateGP = R6Class("SurrogateGP", + inherit = Surrogate, + + public = list( + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + #' + #' @template param_archive_surrogate + #' @template param_col_y_surrogate + #' @template param_cols_x_surrogate + initialize = function(archive = NULL, cols_x = NULL, col_y = NULL) { + assert_r6(archive, classes = "Archive", null.ok = TRUE) + + assert_character(cols_x, min.len = 1L, null.ok = TRUE) + assert_string(col_y, null.ok = TRUE) + + # https://github.com/mlr-org/mlr3learners/blob/51c15755438078fc99b11a9ca0147c7f2dbb96d8/R/LearnerRegrKM.R#L34 + ps = ps( + bias.correct = p_lgl(default = FALSE, tags = "predict"), + checkNames = p_lgl(default = TRUE, tags = "predict"), + coef.cov = p_uty(default = NULL, tags = "train"), + coef.trend = p_uty(default = NULL, tags = "train"), + coef.var = p_uty(default = NULL, tags = "train"), + control = p_uty(default = NULL, tags = "train"), + # cov.compute = p_lgl(default = TRUE, tags = "predict"), + covtype = p_fct(c("gauss", "matern5_2", "matern3_2", "exp", "powexp"), default = "matern5_2", tags = "train"), + estim.method = p_fct(c("MLE", "LOO"), default = "MLE", tags = "train"), + gr = p_lgl(default = TRUE, tags = "train"), + iso = p_lgl(default = FALSE, tags = "train"), + jitter = p_dbl(0, default = 0, tags = "predict"), + kernel = p_uty(default = NULL, tags = "train"), + knots = p_uty(default = NULL, tags = "train", depends = quote(scaling == TRUE)), + light.return = p_lgl(default = FALSE, tags = "predict"), + lower = p_uty(default = NULL, tags = "train"), + multistart = p_int(default = 1, tags = "train", depends = quote(optim.method == "BFGS")), + noise.var = p_uty(default = NULL, tags = "train"), + nugget = p_dbl(tags = "train"), + nugget.estim = p_lgl(default = FALSE, tags = "train"), + nugget.stability = p_dbl(0, default = 0, tags = "train"), + optim.method = p_fct(c("BFGS", "gen"), default = "BFGS", tags = "train"), + parinit = p_uty(default = NULL, tags = "train"), + penalty = p_uty(default = NULL, tags = "train"), + scaling = p_lgl(default = FALSE, tags = "train"), + # se.compute = p_lgl(default = TRUE, tags = "predict"), + type = p_fct(c("SK", "UK"), default = "SK", tags = "predict"), + upper = p_uty(default = NULL, tags = "train"), + catch_errors = p_lgl(tags = "required"), + impute_method = p_fct(c("mean", "random"), tags = "required") + ) + ps$values = list(catch_errors = TRUE, impute_method = "random") + + super$initialize(learner = list(), archive = archive, cols_x = cols_x, cols_y = col_y, param_set = ps) + }, + + #' @description + #' Predict mean response and standard error. + #' + #' @param xdt ([data.table::data.table()])\cr + #' New data. One row per observation. + #' + #' @return [data.table::data.table()] with the columns `mean` and `se`. + predict = function(xdt) { + assert_xdt(xdt) + newdata = as_numeric_matrix(fix_xdt_missing(xdt, cols_x = self$cols_x, archive = self$archive)) + pv = self$param_set$get_values(tags = "predict") + + jitter = pv$jitter + if (!is.null(jitter) && jitter > 0) { + newdata = newdata + stats::rnorm(length(newdata), mean = 0, sd = jitter) + } + + p = invoke(DiceKriging::predict.km, + private$.model, + newdata = newdata, + type = if (is.null(pv$type)) "SK" else pv$type, + se.compute = TRUE, + .args = remove_named(pv, "jitter") + ) + + data.table(mean = p$mean, se = p$sd) + } + ), + + active = list( + + #' @template field_print_id + print_id = function(rhs) { + if (missing(rhs)) { + "DiceKriging::km" + } else { + stop("$print_id is read-only.") + } + }, + + #' @template field_assert_insample_perf_surrogate + assert_insample_perf = function(rhs) { + stopf("Not implemented.") + }, + + #' @template field_n_learner_surrogate + n_learner = function() { + 1L + }, + + + #' @template field_packages_surrogate + packages = function(rhs) { + if (missing(rhs)) { + "DiceKriging" + } else { + stop("$packages is read-only.") + } + }, + + #' @template field_feature_types_surrogate + feature_types = function(rhs) { + if (missing(rhs)) { + c("logical", "integer", "numeric") + } else { + stop("$feature_types is read-only.") + } + }, + + #' @template field_properties_surrogate + properties = function(rhs) { + if (missing(rhs)) { + character(0L) + } else { + stop("$properties is read-only.") + } + }, + + #' @template field_predict_type_surrogate + predict_type = function(rhs) { + if (missing(rhs)) { + "se" + } else { + stop("$predict_type is read-only.") + } + } + ), + + private = list( + + .model = NULL, + + # Train learner with new data. + .update = function() { + pv = self$param_set$get_values(tags = "train") + design = as_numeric_matrix(self$archive$data[, self$cols_x, with = FALSE]) + response = self$archive$data[[self$cols_y]] + + if (!is.null(pv$optim.method) && pv$optim.method == "gen" && !requireNamespace("rgenoud", quietly = TRUE)) { + stopf("The 'rgenoud' package is required for optimization method 'gen'.") + } + + ns = pv$nugget.stability + if (!is.null(ns)) { + pv$nugget = if (ns == 0) 0 else ns * stats::var(response) + } + + private$.model = invoke(DiceKriging::km, + response = response, + design = design, + control = pv$control, + .args = remove_named(pv, c("control", "nugget.stability")) + ) + self$learner$model = private$.model + invisible(NULL) + }, + + # Train learner with new data. + # Operates on an asynchronous archive and performs imputation as needed. + .update_async = function() { + xydt = self$archive$rush$fetch_tasks_with_state(states = c("queued", "running", "finished"))[, c(self$cols_x, self$cols_y, "state"), with = FALSE] + + if (self$param_set$values$impute_method == "mean") { + mean_y = mean(xydt[[self$cols_y]], na.rm = TRUE) + xydt[c("queued", "running"), (self$cols_y) := mean_y, on = "state"] + } else if (self$param_set$values$impute_method == "random") { + min_y = min(xydt[[self$cols_y]], na.rm = TRUE) + max_y = max(xydt[[self$cols_y]], na.rm = TRUE) + xydt[c("queued", "running"), (self$cols_y) := runif(.N, min = min_y, max = max_y), on = "state"] + } + set(xydt, j = "state", value = NULL) + + pv = self$param_set$get_values(tags = "train") + design = as_numeric_matrix(xydt[, self$cols_x, with = FALSE]) + response = xydt[[self$cols_y]] + + if (!is.null(pv$optim.method) && pv$optim.method == "gen" && !requireNamespace("rgenoud", quietly = TRUE)) { + stopf("The 'rgenoud' package is required for optimization method 'gen'.") + } + + ns = pv$nugget.stability + if (!is.null(ns)) { + pv$nugget = if (ns == 0) 0 else ns * stats::var(response) + } + + private$.model = invoke(DiceKriging::km, + response = response, + design = design, + control = pv$control, + .args = remove_named(pv, c("control", "nugget.stability")) + ) + self$learner$model = private$.model + invisible(NULL) + }, + + .reset = function() { + self$learner = list() + private$.model = NULL + }, + + deep_clone = function(name, value) { + switch(name, + .param_set = value$clone(deep = TRUE), + .archive = value$clone(deep = TRUE), + value + ) + } + ) +) + +# https://github.com/mlr-org/mlr3learners/blob/51c15755438078fc99b11a9ca0147c7f2dbb96d8/R/helpers.R#L16C1-L16C18 +as_numeric_matrix = function(x) { + x = as.matrix(x) + if (is.logical(x)) { + storage.mode(x) = "double" + } + x +} diff --git a/R/SurrogateLearner.R b/R/SurrogateLearner.R index 0891833e..86a7beb2 100644 --- a/R/SurrogateLearner.R +++ b/R/SurrogateLearner.R @@ -92,8 +92,8 @@ SurrogateLearner = R6Class("SurrogateLearner", assert_insample_perf = p_lgl(), perf_measure = p_uty(custom_check = function(x) check_r6(x, classes = "MeasureRegr")), # FIXME: actually want check_measure perf_threshold = p_dbl(lower = -Inf, upper = Inf), - catch_errors = p_lgl(), - impute_method = p_fct(c("mean", "random"), default = "random") + catch_errors = p_lgl(tags = "required"), + impute_method = p_fct(c("mean", "random"), tags = "required") ) ps$values = list(assert_insample_perf = FALSE, catch_errors = TRUE, impute_method = "random") ps$add_dep("perf_measure", on = "assert_insample_perf", cond = CondEqual$new(TRUE)) diff --git a/R/SurrogateLearnerCollection.R b/R/SurrogateLearnerCollection.R index 34684c62..67021749 100644 --- a/R/SurrogateLearnerCollection.R +++ b/R/SurrogateLearnerCollection.R @@ -107,8 +107,8 @@ SurrogateLearnerCollection = R6Class("SurrogateLearnerCollection", assert_insample_perf = p_lgl(), perf_measures = p_uty(custom_check = function(x) check_list(x, types = "MeasureRegr", any.missing = FALSE, len = length(learners))), # FIXME: actually want check_measures perf_thresholds = p_uty(custom_check = function(x) check_double(x, lower = -Inf, upper = Inf, any.missing = FALSE, len = length(learners))), - catch_errors = p_lgl(), - impute_method = p_fct(c("mean", "random"), default = "random") + catch_errors = p_lgl(tags = "required"), + impute_method = p_fct(c("mean", "random"), tags = "required") ) ps$values = list(assert_insample_perf = FALSE, catch_errors = TRUE, impute_method = "random") ps$add_dep("perf_measures", on = "assert_insample_perf", cond = CondEqual$new(TRUE)) diff --git a/man/SurrogateGP.Rd b/man/SurrogateGP.Rd new file mode 100644 index 00000000..aeb4f032 --- /dev/null +++ b/man/SurrogateGP.Rd @@ -0,0 +1,187 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/SurrogateGP.R +\name{SurrogateGP} +\alias{SurrogateGP} +\title{Surrogate Model Containing a Gaussian Process} +\description{ +Surrogate model containing a single Gaussian Process via \code{\link[DiceKriging:km]{DiceKriging::km()}} from package \CRANpkg{DiceKriging}. +Update and predict methods are inspired from \link[mlr3learners:mlr_learners_regr.km]{mlr3learners::LearnerRegrKM} from package \CRANpkg{mlr3learners}. + +Compared to using \link[mlr3learners:mlr_learners_regr.km]{mlr3learners::LearnerRegrKM} within a \link{SurrogateLeaner} the update and predict methods of this class are much more efficient +as they skip many assertions and checks naturally arising when using a \link{SurrogateLeaner} wrapping a \link[mlr3learners:mlr_learners_regr.km]{mlr3learners::LearnerRegrKM}. +} +\section{Parameters}{ + +\describe{ +\item{\code{catch_errors}}{\code{logical(1)}\cr +Should errors during updating the surrogate be caught and propagated to the \code{loop_function} which can then handle +the failed acquisition function optimization (as a result of the failed surrogate) appropriately by, e.g., proposing a randomly sampled point for evaluation? +Default is \code{TRUE}. +} +\item{\code{impute_method}}{\code{character(1)}\cr +Method to impute missing values in the case of updating on an asynchronous \link[bbotk:ArchiveAsync]{bbotk::ArchiveAsync} with pending evaluations. +Can be \code{"mean"} to use mean imputation or \code{"random"} to sample values uniformly at random between the empirical minimum and maximum. +Default is \code{"random"}. +} +} +For a description of all other parameters related to \code{\link[DiceKriging:km]{DiceKriging::km()}} directly, see the documentation of \code{\link[DiceKriging:km]{DiceKriging::km()}}. +\itemize{ +\item The predict type hyperparameter "type" defaults to "SK" (simple kriging). +\item The additional hyperparameter \code{nugget.stability} is used to overwrite the +hyperparameter \code{nugget} with \code{nugget.stability * var(y)} before training to +improve the numerical stability. We recommend a value of \code{1e-8}. +\item The additional hyperparameter \code{jitter} can be set to add +\verb{N(0, [jitter])}-distributed noise to the data before prediction to avoid +perfect interpolation. We recommend a value of \code{1e-12}. +} +} + +\examples{ +if (requireNamespace("DiceKriging") & + requireNamespace("rgenoud")) { + library(bbotk) + library(paradox) + + fun = function(xs) { + list(y = xs$x ^ 2) + } + domain = ps(x = p_dbl(lower = -10, upper = 10)) + codomain = ps(y = p_dbl(tags = "minimize")) + objective = ObjectiveRFun$new(fun = fun, domain = domain, codomain = codomain) + + instance = OptimInstanceBatchSingleCrit$new( + objective = objective, + terminator = trm("evals", n_evals = 5)) + + xdt = generate_design_random(instance$search_space, n = 4)$data + + instance$eval_batch(xdt) + + surrogate = SurrogateGP$new(archive = instance$archive) + surrogate$param_set$set_values( + covtype = "matern5_2", + optim.method = "gen", + control = list(trace = FALSE), + nugget.stability = 10^-8 + ) + + surrogate$update() + + surrogate$learner$model +} +} +\section{Super class}{ +\code{\link[mlr3mbo:Surrogate]{mlr3mbo::Surrogate}} -> \code{SurrogateGP} +} +\section{Active bindings}{ +\if{html}{\out{
    }} +\describe{ +\item{\code{print_id}}{(\code{character})\cr +Id used when printing.} + +\item{\code{assert_insample_perf}}{(\code{numeric()})\cr +Asserts whether the current insample performance meets the performance threshold.} + +\item{\code{n_learner}}{(\code{integer(1)})\cr +Returns the number of surrogate models.} + +\item{\code{packages}}{(\code{character()})\cr +Set of required packages. +A warning is signaled if at least one of the packages is not installed, but loaded (not attached) later on-demand via \code{\link[=requireNamespace]{requireNamespace()}}.} + +\item{\code{feature_types}}{(\code{character()})\cr +Stores the feature types the surrogate can handle, e.g. \code{"logical"}, \code{"numeric"}, or \code{"factor"}. +A complete list of candidate feature types, grouped by task type, is stored in \code{\link[mlr3:mlr_reflections]{mlr_reflections$task_feature_types}}.} + +\item{\code{properties}}{(\code{character()})\cr +Stores a set of properties/capabilities the surrogate has. +A complete list of candidate properties, grouped by task type, is stored in \code{\link[mlr3:mlr_reflections]{mlr_reflections$learner_properties}}.} + +\item{\code{predict_type}}{(\code{character(1)})\cr +Retrieves the currently active predict type, e.g. \code{"response"}.} +} +\if{html}{\out{
    }} +} +\section{Methods}{ +\subsection{Public methods}{ +\itemize{ +\item \href{#method-SurrogateGP-new}{\code{SurrogateGP$new()}} +\item \href{#method-SurrogateGP-predict}{\code{SurrogateGP$predict()}} +\item \href{#method-SurrogateGP-clone}{\code{SurrogateGP$clone()}} +} +} +\if{html}{\out{ +
    Inherited methods + +
    +}} +\if{html}{\out{
    }} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-SurrogateGP-new}{}}} +\subsection{Method \code{new()}}{ +Creates a new instance of this \link[R6:R6Class]{R6} class. +\subsection{Usage}{ +\if{html}{\out{
    }}\preformatted{SurrogateGP$new(archive = NULL, cols_x = NULL, col_y = NULL)}\if{html}{\out{
    }} +} + +\subsection{Arguments}{ +\if{html}{\out{
    }} +\describe{ +\item{\code{archive}}{(\link[bbotk:Archive]{bbotk::Archive} | \code{NULL})\cr +\link[bbotk:Archive]{bbotk::Archive} of the \link[bbotk:OptimInstance]{bbotk::OptimInstance}.} + +\item{\code{cols_x}}{(\code{character()} | \code{NULL})\cr +Column id's of variables that should be used as features. +By default, automatically inferred based on the archive.} + +\item{\code{col_y}}{(\code{character(1)} | \code{NULL})\cr +Column id of variable that should be used as a target. +By default, automatically inferred based on the archive.} +} +\if{html}{\out{
    }} +} +} +\if{html}{\out{
    }} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-SurrogateGP-predict}{}}} +\subsection{Method \code{predict()}}{ +Predict mean response and standard error. +\subsection{Usage}{ +\if{html}{\out{
    }}\preformatted{SurrogateGP$predict(xdt)}\if{html}{\out{
    }} +} + +\subsection{Arguments}{ +\if{html}{\out{
    }} +\describe{ +\item{\code{xdt}}{(\code{\link[data.table:data.table]{data.table::data.table()}})\cr +New data. One row per observation.} +} +\if{html}{\out{
    }} +} +\subsection{Returns}{ +\code{\link[data.table:data.table]{data.table::data.table()}} with the columns \code{mean} and \code{se}. +} +} +\if{html}{\out{
    }} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-SurrogateGP-clone}{}}} +\subsection{Method \code{clone()}}{ +The objects of this class are cloneable with this method. +\subsection{Usage}{ +\if{html}{\out{
    }}\preformatted{SurrogateGP$clone(deep = FALSE)}\if{html}{\out{
    }} +} + +\subsection{Arguments}{ +\if{html}{\out{
    }} +\describe{ +\item{\code{deep}}{Whether to make a deep clone.} +} +\if{html}{\out{
    }} +} +} +} From bef4ebbf59b1b0b17f595a59ae062940301e8a5f Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Tue, 19 Nov 2024 18:36:51 +0100 Subject: [PATCH 083/126] typo --- R/SurrogateGP.R | 4 ++-- man/SurrogateGP.Rd | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/R/SurrogateGP.R b/R/SurrogateGP.R index 87128c8c..64dad4f8 100644 --- a/R/SurrogateGP.R +++ b/R/SurrogateGP.R @@ -4,8 +4,8 @@ #' Surrogate model containing a single Gaussian Process via [DiceKriging::km()] from package \CRANpkg{DiceKriging}. #' Update and predict methods are inspired from [mlr3learners::LearnerRegrKM] from package \CRANpkg{mlr3learners}. #' -#' Compared to using [mlr3learners::LearnerRegrKM] within a [SurrogateLeaner] the update and predict methods of this class are much more efficient -#' as they skip many assertions and checks naturally arising when using a [SurrogateLeaner] wrapping a [mlr3learners::LearnerRegrKM]. +#' Compared to using [mlr3learners::LearnerRegrKM] within a [SurrogateLearner] the update and predict methods of this class are much more efficient +#' as they skip many assertions and checks naturally arising when using a [SurrogateLearner] wrapping a [mlr3learners::LearnerRegrKM]. #' #' @section Parameters: #' \describe{ diff --git a/man/SurrogateGP.Rd b/man/SurrogateGP.Rd index aeb4f032..89e3b6a4 100644 --- a/man/SurrogateGP.Rd +++ b/man/SurrogateGP.Rd @@ -7,8 +7,8 @@ Surrogate model containing a single Gaussian Process via \code{\link[DiceKriging:km]{DiceKriging::km()}} from package \CRANpkg{DiceKriging}. Update and predict methods are inspired from \link[mlr3learners:mlr_learners_regr.km]{mlr3learners::LearnerRegrKM} from package \CRANpkg{mlr3learners}. -Compared to using \link[mlr3learners:mlr_learners_regr.km]{mlr3learners::LearnerRegrKM} within a \link{SurrogateLeaner} the update and predict methods of this class are much more efficient -as they skip many assertions and checks naturally arising when using a \link{SurrogateLeaner} wrapping a \link[mlr3learners:mlr_learners_regr.km]{mlr3learners::LearnerRegrKM}. +Compared to using \link[mlr3learners:mlr_learners_regr.km]{mlr3learners::LearnerRegrKM} within a \link{SurrogateLearner} the update and predict methods of this class are much more efficient +as they skip many assertions and checks naturally arising when using a \link{SurrogateLearner} wrapping a \link[mlr3learners:mlr_learners_regr.km]{mlr3learners::LearnerRegrKM}. } \section{Parameters}{ From 2a2f574c06320c4024b54dd1de7b965f0bbe29b2 Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Mon, 21 Apr 2025 22:36:40 +0200 Subject: [PATCH 084/126] .. --- R/bibentries.R | 20 ++++++++++++++++---- R/helper.R | 12 ------------ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/R/bibentries.R b/R/bibentries.R index c2eda867..467d7879 100644 --- a/R/bibentries.R +++ b/R/bibentries.R @@ -14,6 +14,7 @@ bibentries = c( pages = "431--431", journal = "Journal of Global Optimization" ), + jones_1998 = bibentry("article", title = "Efficient Global Optimization of Expensive Black-Box Functions", author = "Jones, Donald R. and Schonlau, Matthias and Welch, William J.", @@ -23,6 +24,7 @@ bibentries = c( pages = "455--492", journal = "Journal of Global optimization" ), + ding_2010 = bibentry("article", title = "An Investigation of Missing Data Methods for Classification Trees Applied to Binary Response Data", author = "Ding, Yufeng and Simonoff, Jeffrey S", @@ -32,6 +34,7 @@ bibentries = c( pages = "131--170", journal = "Journal of Machine Learning Research" ), + beume_2007 = bibentry("article", title = "SMS-EMOA: Multiobjective selection based on dominated hypervolume", author = "Nicola Beume and Boris Naujoks and Michael Emmerich", @@ -41,8 +44,9 @@ bibentries = c( pages = "1653--1669", journal = "European Journal of Operational Research" ), - snoek_2012 = bibentry("inproceedings", - title = "Practical {B}ayesian Optimization of Machine Learning Algorithms", + + snoek_2012 = bibentry("inproceedings", + title = "Practical Bayesian Optimization of Machine Learning Algorithms", author = "Snoek, Jasper and Larochelle, Hugo and Adams, Ryan P", year = "2012", booktitle = "Advances in Neural Information Processing Systems", @@ -50,6 +54,7 @@ bibentries = c( pages = "2951--2959", volume = "25" ), + kushner_1964 = bibentry("article", title = "A New Method of Locating the Maximum Point of an Arbitrary Multipeak Curve in the Presence of Noise", author = "Kushner, H. J.", @@ -59,6 +64,7 @@ bibentries = c( number = "1", pages = "97--106", ), + ponweiser_2008 = bibentry("inproceedings", title = "Multiobjective Optimization on a Limited Budget of Evaluations Using Model-Assisted S-Metric Selection", author = "Ponweiser, Wolfgang and Wagner, Tobias and Biermann, Dirk and Vincze, Markus", @@ -66,8 +72,9 @@ bibentries = c( booktitle = "Proceedings of the 10th International Conference on Parallel Problem Solving from Nature", pages = "784--794" ), + wang_2020 = bibentry("article", - title = "Parallel {B}ayesian Global Optimization of Expensive Functions", + title = "Parallel Bayesian Global Optimization of Expensive Functions", author = "Wang, Jialei and Clark, Scott C. and Liu, Eric and Frazier, Peter I.", year = "2020", journal = "Operations Research", @@ -75,6 +82,7 @@ bibentries = c( number = "6", pages = "1850--1865" ), + knowles_2006 = bibentry("article", title = "ParEGO: A Hybrid Algorithm With On-Line Landscape Approximation for Expensive Multiobjective Optimization Problems", volume = "10", @@ -84,18 +92,21 @@ bibentries = c( year = "2006", pages = "50--66" ), - horn_2015 = bibentry("inproceedings", + + horn_2015 = bibentry("inproceedings", title = "Model-Based Multi-objective Optimization: Taxonomy, Multi-Point Proposal, Toolbox and Benchmark", author = "Horn, Daniel and Wagner, Tobias and Biermann, Dirk and Weihs, Claus and Bischl, Bernd", year = "2015", booktitle = "International Conference on Evolutionary Multi-Criterion Optimization", pages = "64--78" ), + ginsbourger_2008 = bibentry("misc", title = "A Multi-Points Criterion for Deterministic Parallel Global Optimization Based on Gaussian Processes", author = "Ginsbourger, David and Le Riche, Rodolphe and Carraro, Laurent", year = "2008" ), + emmerich_2016 = bibentry("incollection", title = "A Multicriteria Generalization of Bayesian Global Optimization", author = "Emmerich, Michael and Yang, Kaifeng and Deutz, Andr{\\'e} and Wang, Hao and Fonseca, Carlos M.", @@ -106,6 +117,7 @@ bibentries = c( address = "Cham", pages = "229--242" ), + rahat_2022 = bibentry("inproceedings", title = "Efficient Approximation of Expected Hypervolume Improvement using Gauss-Hermit Quadrature", author = "Rahat, Alma and Chugh, Tinkle and Fieldsend, Jonathan and Allmendinger, Richard and Miettinen, Kaisa", diff --git a/R/helper.R b/R/helper.R index 585c9ff5..25a39fb1 100644 --- a/R/helper.R +++ b/R/helper.R @@ -169,18 +169,6 @@ assert_xdt = function(xdt) { assert_data_table(xdt) } -task_rm_backend = function(task) { - # fix task hash - ee = get_private(task) - ee$.hash = force(task$hash) - ee$.col_hashes = force(task$col_hashes) - - # NULL backend - task$backend = NULL - - task -} - assert_learner_surrogate = function(x, .var.name = vname(x)) { # NOTE: this is buggy in checkmate; assert should always return x invisible not TRUE as is the case here assert(check_learner_surrogate(x), .var.name = .var.name) From 605a3055a7f564a98aabd23736b90f72e8f88030 Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Fri, 16 May 2025 19:51:12 +0200 Subject: [PATCH 085/126] .. --- R/SurrogateGP.R | 5 ----- man/SurrogateGP.Rd | 3 --- 2 files changed, 8 deletions(-) diff --git a/R/SurrogateGP.R b/R/SurrogateGP.R index 64dad4f8..5303f6eb 100644 --- a/R/SurrogateGP.R +++ b/R/SurrogateGP.R @@ -158,11 +158,6 @@ SurrogateGP = R6Class("SurrogateGP", } }, - #' @template field_assert_insample_perf_surrogate - assert_insample_perf = function(rhs) { - stopf("Not implemented.") - }, - #' @template field_n_learner_surrogate n_learner = function() { 1L diff --git a/man/SurrogateGP.Rd b/man/SurrogateGP.Rd index 89e3b6a4..cf488404 100644 --- a/man/SurrogateGP.Rd +++ b/man/SurrogateGP.Rd @@ -79,9 +79,6 @@ if (requireNamespace("DiceKriging") & \item{\code{print_id}}{(\code{character})\cr Id used when printing.} -\item{\code{assert_insample_perf}}{(\code{numeric()})\cr -Asserts whether the current insample performance meets the performance threshold.} - \item{\code{n_learner}}{(\code{integer(1)})\cr Returns the number of surrogate models.} From 58146347ad39d0f6290db6d71f45cf7571bc5182 Mon Sep 17 00:00:00 2001 From: Lennart Schneider Date: Tue, 27 May 2025 11:03:50 +0200 Subject: [PATCH 086/126] .. --- R/zzz.R | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/R/zzz.R b/R/zzz.R index b7d2b6fe..ef5efbc5 100644 --- a/R/zzz.R +++ b/R/zzz.R @@ -19,12 +19,11 @@ register_bbotk = function() { iwalk(optimizers, function(obj, nm) x$add(nm, obj)) } # nocov end -#' @include aaa.R register_mlr3 = function() { + # nocov start x = utils::getFromNamespace("mlr_learners", ns = "mlr3") - iwalk(learners, function(obj, nm) x$add(nm, obj)) -} +} # nocov end register_mlr3tuning = function() { # nocov start From 53925b83f90c444fc740edbe18ef83d201849310 Mon Sep 17 00:00:00 2001 From: sumny Date: Thu, 12 Jun 2025 17:03:24 +0200 Subject: [PATCH 087/126] .. --- R/InputTrafoUnitcube.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/InputTrafoUnitcube.R b/R/InputTrafoUnitcube.R index d9144770..6934a296 100644 --- a/R/InputTrafoUnitcube.R +++ b/R/InputTrafoUnitcube.R @@ -75,7 +75,7 @@ InputTrafoUnitcube= R6Class("InputTrafoUnitcube", } xdt = copy(xdt) parameters = names(which(self$search_space$is_number)) # numeric or integer - assert_set_equal(parameters, self$cols_x) + assert_subset(parameters, self$cols_x) for (parameter in parameters) { set(xdt, j = parameter, value = (xdt[[parameter]] - self$search_space$lower[[parameter]]) / (self$search_space$upper[[parameter]] - self$search_space$lower[[parameter]])) } From a9961c15c3d706586d02a11c836b98a1e429c936 Mon Sep 17 00:00:00 2001 From: be-marc Date: Mon, 7 Jul 2025 17:22:43 +0200 Subject: [PATCH 088/126] ... --- R/AcqOptimizerDirect.R | 106 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 R/AcqOptimizerDirect.R diff --git a/R/AcqOptimizerDirect.R b/R/AcqOptimizerDirect.R new file mode 100644 index 00000000..8a7091e6 --- /dev/null +++ b/R/AcqOptimizerDirect.R @@ -0,0 +1,106 @@ +#' @export +AcqOptimizerDirect = R6Class("AcqOptimizerDirect", + inherit = AcqOptimizer, + public = list( + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + #' + #' @param optimizer ([bbotk::OptimizerBatch]). + #' @param terminator ([bbotk::Terminator]). + #' @param acq_function (`NULL` | [AcqFunction]). + #' @param callbacks (`NULL` | list of [mlr3misc::Callback]) + initialize = function(terminator, acq_function = NULL) { + self$terminator = assert_r6(terminator, "Terminator") + self$acq_function = assert_r6(acq_function, "AcqFunction", null.ok = TRUE) + ps = ps( + n_candidates = p_int(lower = 1, default = 1L), + logging_level = p_fct(levels = c("fatal", "error", "warn", "info", "debug", "trace"), default = "warn"), + warmstart = p_lgl(default = FALSE), + warmstart_size = p_int(lower = 1L, special_vals = list("all")), + skip_already_evaluated = p_lgl(default = TRUE), + catch_errors = p_lgl(default = TRUE) + ) + ps$values = list(n_candidates = 1, logging_level = "warn", warmstart = FALSE, skip_already_evaluated = TRUE, catch_errors = TRUE) + ps$add_dep("warmstart_size", on = "warmstart", cond = CondEqual$new(TRUE)) + private$.param_set = ps + }, + + #' @description + #' Optimize the acquisition function. + #' + #' @return [data.table::data.table()] with 1 row per candidate. + optimize = function() { + lg = lgr::get_logger("mlr3/bbotk") + old_threshold = lg$threshold + lg$set_threshold(self$param_set$values$logging_level) + on.exit(lg$set_threshold(old_threshold)) + + if (inherits(self$terminator, "TerminatorEvals")) { + maxeval = self$terminator$param_set$values$n_evals + } else { + stopf("Only TerminatorEvals is supported for AcqOptimizerDirect.") + } + + wrapper = function(x) { + constants = self$acq_function$constants$values + as.numeric(mlr3misc::invoke(get_private(self$acq_function)$.fun, xdt = as.data.table(as.list(set_names(x, self$acq_function$domain$ids()))), .args = constants)) + } + + x0 = as.numeric(self$acq_function$archive$best()[, self$acq_function$domain$ids(), with = FALSE]) + + res = invoke(nloptr::nloptr, + eval_f = wrapper, + lb = self$acq_function$domain$lower, + ub = self$acq_function$domain$upper, + opts = list(algorithm = "NLOPT_GN_DIRECT_L", maxeval = maxeval), + eval_grad_f = NULL, + x0 = x0) + + as.data.table(as.list(set_names(res$solution, self$acq_function$domain$ids()))) + }, + + #' @description + #' Reset the acquisition function optimizer. + #' + #' Currently not used. + reset = function() { + + } + ), + + active = list( + #' @template field_print_id + print_id = function(rhs) { + if (missing(rhs)) { + paste0("(", class(self$optimizer)[1L], " | ", class(self$terminator)[1L], ")") + } else { + stop("$print_id is read-only.") + } + }, + + #' @field param_set ([paradox::ParamSet])\cr + #' Set of hyperparameters. + param_set = function(rhs) { + if (!missing(rhs) && !identical(rhs, private$.param_set)) { + stop("$param_set is read-only.") + } + private$.param_set + } + ), + + private = list( + .param_set = NULL, + + deep_clone = function(name, value) { + switch(name, + optimizer = value$clone(deep = TRUE), + terminator = value$clone(deep = TRUE), + acq_function = if (!is.null(value)) value$clone(deep = TRUE) else NULL, + .param_set = value$clone(deep = TRUE), + value + ) + } + ) +) + From e8e28dd9dee592ef5751869a89575b668d861862 Mon Sep 17 00:00:00 2001 From: be-marc Date: Tue, 8 Jul 2025 15:16:23 +0200 Subject: [PATCH 089/126] ... --- R/AcqOptimizerCmaes.R | 120 +++++++++++++++++++++++ R/AcqOptimizerDirect.R | 61 ++++++------ tests/testthat/test_AcqOptimizerCmaes.R | 29 ++++++ tests/testthat/test_AcqOptimizerDirect.R | 29 ++++++ 4 files changed, 209 insertions(+), 30 deletions(-) create mode 100644 R/AcqOptimizerCmaes.R create mode 100644 tests/testthat/test_AcqOptimizerCmaes.R create mode 100644 tests/testthat/test_AcqOptimizerDirect.R diff --git a/R/AcqOptimizerCmaes.R b/R/AcqOptimizerCmaes.R new file mode 100644 index 00000000..d8d0682b --- /dev/null +++ b/R/AcqOptimizerCmaes.R @@ -0,0 +1,120 @@ +#' @export +AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", + inherit = AcqOptimizer, + public = list( + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + #' + #' @param acq_function (`NULL` | [AcqFunction]). + initialize = function(acq_function = NULL) { + self$acq_function = assert_r6(acq_function, "AcqFunction", null.ok = TRUE) + ps = ps( + fnscale = p_dbl(default = 1), + maxit = p_int(lower = 1L), + stopfitness = p_dbl(default = -Inf), + keep.best = p_lgl(default = TRUE), + sigma = p_uty(default = 0.5), + mu = p_int(lower = 1L), + lambda = p_int(lower = 1L), + weights = p_uty(), + damps = p_dbl(), + cs = p_dbl(), + ccum = p_dbl(), + ccov.1 = p_dbl(lower = 0), + ccov.mu = p_dbl(lower = 0), + diag.sigma = p_lgl(default = FALSE), + diag.eigen = p_lgl(default = FALSE), + diag.pop = p_lgl(default = FALSE), + diag.value = p_lgl(default = FALSE), + stop.tolx = p_dbl() # undocumented stop criterion + # start_values = p_fct(default = "random", levels = c("random", "center", "custom")), + # start = p_uty(default = NULL, depends = start_values == "custom"), + # n_candidates = p_int(lower = 1, default = 1L), + # logging_level = p_fct(levels = c("fatal", "error", "warn", "info", "debug", "trace"), default = "warn"), + # warmstart = p_lgl(default = FALSE), + # warmstart_size = p_int(lower = 1L, special_vals = list("all")), + # skip_already_evaluated = p_lgl(default = TRUE), + # catch_errors = p_lgl(default = TRUE) + ) + # ps$values = list(n_candidates = 1, logging_level = "warn", warmstart = FALSE, skip_already_evaluated = TRUE, catch_errors = TRUE) + # ps$add_dep("warmstart_size", on = "warmstart", cond = CondEqual$new(TRUE)) + private$.param_set = ps + }, + + #' @description + #' Optimize the acquisition function. + #' + #' @return [data.table::data.table()] with 1 row per candidate. + optimize = function() { + pv = self$param_set$values + pv$vectorized = TRUE + par = set_names(as.numeric(self$acq_function$archive$best()[, self$acq_function$domain$ids(), with = FALSE][[1]]), self$acq_function$domain$ids()) + + wrapper = function(xmat, fun, constants, direction) { + xdt = as.data.table(t(xmat)) + res = mlr3misc::invoke(fun, xdt = xdt, .args = constants)[[1]] + res * direction + } + + fun = get_private(self$acq_function)$.fun + constants = self$acq_function$constants$values + direction = self$acq_function$codomain$direction + + res = invoke(cmaes::cma_es, + par = par, + fn = wrapper, + lower = self$acq_function$domain$lower, + upper = self$acq_function$domain$upper, + control = pv, + fun = fun, + constants = constants, + direction = direction) + + as.data.table(as.list(set_names(c(res$par, res$value), c(self$acq_function$domain$ids(), self$acq_function$codomain$ids())))) + }, + + #' @description + #' Reset the acquisition function optimizer. + #' + #' Currently not used. + reset = function() { + + } + ), + + active = list( + #' @template field_print_id + print_id = function(rhs) { + if (missing(rhs)) { + paste0("(", class(self$optimizer)[1L], " | ", class(self$terminator)[1L], ")") + } else { + stop("$print_id is read-only.") + } + }, + + #' @field param_set ([paradox::ParamSet])\cr + #' Set of hyperparameters. + param_set = function(rhs) { + if (!missing(rhs) && !identical(rhs, private$.param_set)) { + stop("$param_set is read-only.") + } + private$.param_set + } + ), + + private = list( + .param_set = NULL, + + deep_clone = function(name, value) { + switch(name, + optimizer = value$clone(deep = TRUE), + terminator = value$clone(deep = TRUE), + acq_function = if (!is.null(value)) value$clone(deep = TRUE) else NULL, + .param_set = value$clone(deep = TRUE), + value + ) + } + ) +) + diff --git a/R/AcqOptimizerDirect.R b/R/AcqOptimizerDirect.R index 8a7091e6..86c1b0da 100644 --- a/R/AcqOptimizerDirect.R +++ b/R/AcqOptimizerDirect.R @@ -6,23 +6,26 @@ AcqOptimizerDirect = R6Class("AcqOptimizerDirect", #' @description #' Creates a new instance of this [R6][R6::R6Class] class. #' - #' @param optimizer ([bbotk::OptimizerBatch]). - #' @param terminator ([bbotk::Terminator]). #' @param acq_function (`NULL` | [AcqFunction]). - #' @param callbacks (`NULL` | list of [mlr3misc::Callback]) - initialize = function(terminator, acq_function = NULL) { - self$terminator = assert_r6(terminator, "Terminator") + initialize = function(acq_function = NULL) { self$acq_function = assert_r6(acq_function, "AcqFunction", null.ok = TRUE) ps = ps( - n_candidates = p_int(lower = 1, default = 1L), - logging_level = p_fct(levels = c("fatal", "error", "warn", "info", "debug", "trace"), default = "warn"), - warmstart = p_lgl(default = FALSE), - warmstart_size = p_int(lower = 1L, special_vals = list("all")), - skip_already_evaluated = p_lgl(default = TRUE), - catch_errors = p_lgl(default = TRUE) + stopval = p_dbl(default = -Inf, lower = -Inf, upper = Inf), + xtol_rel = p_dbl(default = 1e-06, lower = 0, upper = Inf, special_vals = list(-1)), + xtol_abs = p_dbl(default = 0, lower = 0, upper = Inf, special_vals = list(-1)), + maxeval = p_int(lower = 1, default = 1000L, special_vals = list(-1)), + ftol_rel = p_dbl(default = 0, lower = 0, upper = Inf, special_vals = list(-1)), + ftol_abs = p_dbl(default = 0, lower = 0, upper = Inf, special_vals = list(-1)), + minf_max = p_dbl(default = -Inf) + # n_candidates = p_int(lower = 1, default = 1L), + # logging_level = p_fct(levels = c("fatal", "error", "warn", "info", "debug", "trace"), default = "warn"), + # warmstart = p_lgl(default = FALSE), + # warmstart_size = p_int(lower = 1L, special_vals = list("all")), + # skip_already_evaluated = p_lgl(default = TRUE), + # catch_errors = p_lgl(default = TRUE) ) - ps$values = list(n_candidates = 1, logging_level = "warn", warmstart = FALSE, skip_already_evaluated = TRUE, catch_errors = TRUE) - ps$add_dep("warmstart_size", on = "warmstart", cond = CondEqual$new(TRUE)) + # ps$values = list(n_candidates = 1, logging_level = "warn", warmstart = FALSE, skip_already_evaluated = TRUE, catch_errors = TRUE) + # ps$add_dep("warmstart_size", on = "warmstart", cond = CondEqual$new(TRUE)) private$.param_set = ps }, @@ -31,33 +34,31 @@ AcqOptimizerDirect = R6Class("AcqOptimizerDirect", #' #' @return [data.table::data.table()] with 1 row per candidate. optimize = function() { - lg = lgr::get_logger("mlr3/bbotk") - old_threshold = lg$threshold - lg$set_threshold(self$param_set$values$logging_level) - on.exit(lg$set_threshold(old_threshold)) - - if (inherits(self$terminator, "TerminatorEvals")) { - maxeval = self$terminator$param_set$values$n_evals - } else { - stopf("Only TerminatorEvals is supported for AcqOptimizerDirect.") - } + pv = self$param_set$values + x0 = as.numeric(self$acq_function$archive$best()[, self$acq_function$domain$ids(), with = FALSE]) - wrapper = function(x) { - constants = self$acq_function$constants$values - as.numeric(mlr3misc::invoke(get_private(self$acq_function)$.fun, xdt = as.data.table(as.list(set_names(x, self$acq_function$domain$ids()))), .args = constants)) + wrapper = function(x, fun, constants, direction) { + xdt = as.data.table(as.list(set_names(x, self$acq_function$domain$ids()))) + res = mlr3misc::invoke(fun, xdt = xdt, .args = constants)[[1]] + res * direction } - x0 = as.numeric(self$acq_function$archive$best()[, self$acq_function$domain$ids(), with = FALSE]) + fun = get_private(self$acq_function)$.fun + constants = self$acq_function$constants$values + direction = self$acq_function$codomain$direction res = invoke(nloptr::nloptr, eval_f = wrapper, lb = self$acq_function$domain$lower, ub = self$acq_function$domain$upper, - opts = list(algorithm = "NLOPT_GN_DIRECT_L", maxeval = maxeval), + opts = c(pv, list(algorithm = "NLOPT_GN_DIRECT_L")), eval_grad_f = NULL, - x0 = x0) + x0 = x0, + fun = fun, + constants = constants, + direction = direction) - as.data.table(as.list(set_names(res$solution, self$acq_function$domain$ids()))) + as.data.table(as.list(set_names(c(res$solution, res$objective), c(self$acq_function$domain$ids(), self$acq_function$codomain$ids())))) }, #' @description diff --git a/tests/testthat/test_AcqOptimizerCmaes.R b/tests/testthat/test_AcqOptimizerCmaes.R new file mode 100644 index 00000000..65862d4d --- /dev/null +++ b/tests/testthat/test_AcqOptimizerCmaes.R @@ -0,0 +1,29 @@ +test_that("AcqOptimizerCmaes works", { + instance = oi(OBJ_1D, terminator = trm("evals", n_evals = 5L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerCmaes$new(acq_function = acqfun) + acqopt$param_set$set_values(maxit = 10L) + acqfun$surrogate$update() + acqfun$update() + expect_data_table(acqopt$optimize(), nrows = 1L) +}) + +test_that("AcqOptimizerCmaes works", { + instance = oi(OBJ_1D, terminator = trm("evals", n_evals = 10L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerCmaes$new(acq_function = acqfun) + acqopt$param_set$set_values(maxit = 10L) + + optimizer = opt("mbo", acq_optimizer = acqopt, acq_function = acqfun, surrogate = surrogate) + optimizer$optimize(instance) + + instance$archive$data +}) diff --git a/tests/testthat/test_AcqOptimizerDirect.R b/tests/testthat/test_AcqOptimizerDirect.R new file mode 100644 index 00000000..e8aa6996 --- /dev/null +++ b/tests/testthat/test_AcqOptimizerDirect.R @@ -0,0 +1,29 @@ +test_that("AcqOptimizerDirect works", { + instance = oi(OBJ_1D, terminator = trm("evals", n_evals = 5L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerDirect$new(acq_function = acqfun) + acqopt$param_set$set_values(maxeval = 200L) + acqfun$surrogate$update() + acqfun$update() + expect_data_table(acqopt$optimize(), nrows = 1L) +}) + +test_that("AcqOptimizerDirect works", { + instance = oi(OBJ_1D, terminator = trm("evals", n_evals = 10L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerDirect$new(acq_function = acqfun) + acqopt$param_set$set_values(maxeval = 100L) + + optimizer = opt("mbo", acq_optimizer = acqopt, acq_function = acqfun, surrogate = surrogate) + optimizer$optimize(instance) + + instance$archive$data +}) From e7ae13b50e7cf50def3991948f6943d2e4fcd856 Mon Sep 17 00:00:00 2001 From: be-marc Date: Tue, 8 Jul 2025 15:17:14 +0200 Subject: [PATCH 090/126] ... --- DESCRIPTION | 2 ++ NAMESPACE | 2 ++ man/mlr_learners_regr.ranger.Rd | 1 + 3 files changed, 5 insertions(+) diff --git a/DESCRIPTION b/DESCRIPTION index a155134f..5341e1fc 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -91,6 +91,8 @@ Collate: 'AcqFunctionStochasticCB.R' 'AcqFunctionStochasticEI.R' 'AcqOptimizer.R' + 'AcqOptimizerCmaes.R' + 'AcqOptimizerDirect.R' 'mlr_input_trafos.R' 'InputTrafo.R' 'InputTrafoUnitcube.R' diff --git a/NAMESPACE b/NAMESPACE index 7438ea9b..550cf496 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -23,6 +23,8 @@ export(AcqFunctionSmsEgo) export(AcqFunctionStochasticCB) export(AcqFunctionStochasticEI) export(AcqOptimizer) +export(AcqOptimizerCmaes) +export(AcqOptimizerDirect) export(InputTrafo) export(InputTrafoUnitcube) export(LearnerRegrRangerMbo) diff --git a/man/mlr_learners_regr.ranger.Rd b/man/mlr_learners_regr.ranger.Rd index f0c824b1..f0eb53ee 100644 --- a/man/mlr_learners_regr.ranger.Rd +++ b/man/mlr_learners_regr.ranger.Rd @@ -30,6 +30,7 @@ Calls \code{\link[ranger:ranger]{ranger::ranger()}} from package \CRANpkg{ranger
  • mlr3::Learner$help()
  • mlr3::Learner$predict()
  • mlr3::Learner$predict_newdata()
    • +
  • mlr3::Learner$predict_newdata_fast()
  • mlr3::Learner$print()
  • mlr3::Learner$reset()
  • mlr3::Learner$selected_features()
    • From 64490d650ac75bf3f4f9b78fdf18ab06a40acc9d Mon Sep 17 00:00:00 2001 From: be-marc Date: Tue, 8 Jul 2025 15:43:36 +0200 Subject: [PATCH 091/126] ... --- R/AcqOptimizerCmaes.R | 2 +- tests/testthat/test_AcqOptimizerCmaes.R | 14 ++++++++++++++ tests/testthat/test_AcqOptimizerDirect.R | 14 ++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/R/AcqOptimizerCmaes.R b/R/AcqOptimizerCmaes.R index d8d0682b..2fd210d9 100644 --- a/R/AcqOptimizerCmaes.R +++ b/R/AcqOptimizerCmaes.R @@ -49,7 +49,7 @@ AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", optimize = function() { pv = self$param_set$values pv$vectorized = TRUE - par = set_names(as.numeric(self$acq_function$archive$best()[, self$acq_function$domain$ids(), with = FALSE][[1]]), self$acq_function$domain$ids()) + par = set_names(as.numeric(self$acq_function$archive$best()[, self$acq_function$domain$ids(), with = FALSE]), self$acq_function$domain$ids()) wrapper = function(xmat, fun, constants, direction) { xdt = as.data.table(t(xmat)) diff --git a/tests/testthat/test_AcqOptimizerCmaes.R b/tests/testthat/test_AcqOptimizerCmaes.R index 65862d4d..56c3b502 100644 --- a/tests/testthat/test_AcqOptimizerCmaes.R +++ b/tests/testthat/test_AcqOptimizerCmaes.R @@ -12,6 +12,20 @@ test_that("AcqOptimizerCmaes works", { expect_data_table(acqopt$optimize(), nrows = 1L) }) +test_that("AcqOptimizerCmaes works with 2D", { + instance = oi(OBJ_2D, terminator = trm("evals", n_evals = 5L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerCmaes$new(acq_function = acqfun) + acqopt$param_set$set_values(maxit = 10L) + acqfun$surrogate$update() + acqfun$update() + expect_data_table(acqopt$optimize(), nrows = 1L) +}) + test_that("AcqOptimizerCmaes works", { instance = oi(OBJ_1D, terminator = trm("evals", n_evals = 10L)) design = generate_design_grid(instance$search_space, resolution = 4L)$data diff --git a/tests/testthat/test_AcqOptimizerDirect.R b/tests/testthat/test_AcqOptimizerDirect.R index e8aa6996..16da3a2d 100644 --- a/tests/testthat/test_AcqOptimizerDirect.R +++ b/tests/testthat/test_AcqOptimizerDirect.R @@ -12,6 +12,20 @@ test_that("AcqOptimizerDirect works", { expect_data_table(acqopt$optimize(), nrows = 1L) }) +test_that("AcqOptimizerDirect works with 2D", { + instance = oi(OBJ_2D, terminator = trm("evals", n_evals = 5L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerDirect$new(acq_function = acqfun) + acqopt$param_set$set_values(maxeval = 200L) + acqfun$surrogate$update() + acqfun$update() + expect_data_table(acqopt$optimize(), nrows = 1L) +}) + test_that("AcqOptimizerDirect works", { instance = oi(OBJ_1D, terminator = trm("evals", n_evals = 10L)) design = generate_design_grid(instance$search_space, resolution = 4L)$data From 358c15081d60b0fc856dd099d7dfe569f0e1bdc0 Mon Sep 17 00:00:00 2001 From: be-marc Date: Tue, 8 Jul 2025 17:09:27 +0200 Subject: [PATCH 092/126] ... --- DESCRIPTION | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 5341e1fc..8017d009 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -65,7 +65,10 @@ Suggests: redux, rush, stringi, - testthat (>= 3.0.0) + testthat (>= 3.0.0), +Remotes: + mlr-org/mlr3@predict_newdata_fast, + mlr-org/mlr3learners@ranger_se ByteCompile: no Encoding: UTF-8 Config/testthat/edition: 3 From 1674a5b1e38322d3a5fa6d660e5bdab702af4129 Mon Sep 17 00:00:00 2001 From: be-marc Date: Tue, 8 Jul 2025 17:52:49 +0200 Subject: [PATCH 093/126] ... --- R/SurrogateLearner.R | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/R/SurrogateLearner.R b/R/SurrogateLearner.R index 3a23dab0..a452b6a6 100644 --- a/R/SurrogateLearner.R +++ b/R/SurrogateLearner.R @@ -107,28 +107,34 @@ SurrogateLearner = R6Class("SurrogateLearner", #' #' @return [data.table::data.table()] with the columns `mean` and `se`. predict = function(xdt) { - assert_xdt(xdt) - xdt = fix_xdt_missing(copy(xdt), cols_x = self$cols_x, archive = self$archive) + # assert_xdt(xdt) + + # xdt = fix_xdt_missing(copy(xdt), cols_x = self$cols_x, archive = self$archive) + if (!is.null(self$input_trafo)) { xdt = self$input_trafo$transform(xdt) } + + pred = self$learner$predict_newdata_fast(xdt) + pred = set_names(pred, c("mean", "se")) + # speeding up some checks by constructing the predict task directly instead of relying on predict_newdata - task = self$learner$state$train_task$clone() - set(xdt, j = task$target_names, value = NA_real_) # tasks only have features and the target but we have to set the target to NA - newdata = as_data_backend(xdt) - task$backend = newdata - task$row_roles$use = task$backend$rownames - pred = self$learner$predict(task) + # task = self$learner$state$train_task$clone() + # set(xdt, j = task$target_names, value = NA_real_) # tasks only have features and the target but we have to set the target to NA + # newdata = as_data_backend(xdt) + # task$backend = newdata + # task$row_roles$use = task$backend$rownames + # pred = self$learner$predict(task) # slow #pred = self$learner$predict_newdata(newdata = xdt) - pred = if (self$learner$predict_type == "se") { - data.table(mean = pred$response, se = pred$se) - } else { - data.table(mean = pred$response) - } + # pred = if (self$learner$predict_type == "se") { + # data.table(mean = pred$response, se = pred$se) + # } else { + # data.table(mean = pred$response) + # } if (!is.null(self$output_trafo) && self$output_trafo$invert_posterior) { pred = self$output_trafo$inverse_transform_posterior(pred) } From ecdc1d248eff3e849d9a152b23f34f8975f042fe Mon Sep 17 00:00:00 2001 From: be-marc Date: Tue, 8 Jul 2025 18:59:57 +0200 Subject: [PATCH 094/126] ... --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 97cd9a2b..4d76591f 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Type: Package Package: mlr3mbo Title: Flexible Bayesian Optimization -Version: 0.3.0.9000 +Version: 0.3.0.9002 Authors@R: c( person("Lennart", "Schneider", , "lennart.sch@web.de", role = c("cre", "aut"), comment = c(ORCID = "0000-0003-4152-5308")), From 375a68e4bd7b289d5a093e17d0b03615b6faa527 Mon Sep 17 00:00:00 2001 From: be-marc Date: Thu, 10 Jul 2025 09:54:13 +0200 Subject: [PATCH 095/126] ... --- R/SurrogateRF.R | 307 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 307 insertions(+) create mode 100644 R/SurrogateRF.R diff --git a/R/SurrogateRF.R b/R/SurrogateRF.R new file mode 100644 index 00000000..f74c23ca --- /dev/null +++ b/R/SurrogateRF.R @@ -0,0 +1,307 @@ +#' @title Surrogate Model Containing a Random Forest +#' +#' @description +#' Surrogate model containing a single Random Forest via [ranger::ranger()] from package \CRANpkg{ranger}. +#' Update and predict methods are inspired from [mlr3learners::LearnerRegrRanger] from package \CRANpkg{mlr3learners}. +#' +#' Compared to using [mlr3learners::LearnerRegrRanger] within a [SurrogateLearner] the update and predict methods of this class are much more efficient +#' as they skip many assertions and checks naturally arising when using a [SurrogateLearner] wrapping a [mlr3learners::LearnerRegrRanger]. +#' +#' @section Parameters: +#' \describe{ +#' \item{`catch_errors`}{`logical(1)`\cr +#' Should errors during updating the surrogate be caught and propagated to the `loop_function` which can then handle +#' the failed acquisition function optimization (as a result of the failed surrogate) appropriately by, e.g., proposing a randomly sampled point for evaluation? +#' Default is `TRUE`. +#' } +#' \item{`impute_method`}{`character(1)`\cr +#' Method to impute missing values in the case of updating on an asynchronous [bbotk::ArchiveAsync] with pending evaluations. +#' Can be `"mean"` to use mean imputation or `"random"` to sample values uniformly at random between the empirical minimum and maximum. +#' Default is `"random"`. +#' } +#' } +#' For a description of all other parameters related to [DiceKriging::km()] directly, see the documentation of [DiceKriging::km()]. +#' * The predict type hyperparameter "type" defaults to "SK" (simple kriging). +#' * The additional hyperparameter `nugget.stability` is used to overwrite the +#' hyperparameter `nugget` with `nugget.stability * var(y)` before training to +#' improve the numerical stability. We recommend a value of `1e-8`. +#' * The additional hyperparameter `jitter` can be set to add +#' `N(0, [jitter])`-distributed noise to the data before prediction to avoid +#' perfect interpolation. We recommend a value of `1e-12`. +#' +#' @export +#' @examples +#' if (requireNamespace("DiceKriging") & +#' requireNamespace("rgenoud")) { +#' library(bbotk) +#' library(paradox) +#' +#' fun = function(xs) { +#' list(y = xs$x ^ 2) +#' } +#' domain = ps(x = p_dbl(lower = -10, upper = 10)) +#' codomain = ps(y = p_dbl(tags = "minimize")) +#' objective = ObjectiveRFun$new(fun = fun, domain = domain, codomain = codomain) +#' +#' instance = OptimInstanceBatchSingleCrit$new( +#' objective = objective, +#' terminator = trm("evals", n_evals = 5)) +#' +#' xdt = generate_design_random(instance$search_space, n = 4)$data +#' +#' instance$eval_batch(xdt) +#' +#' surrogate = SurrogateGP$new(archive = instance$archive) +#' surrogate$param_set$set_values( +#' covtype = "matern5_2", +#' optim.method = "gen", +#' control = list(trace = FALSE), +#' nugget.stability = 10^-8 +#' ) +#' +#' surrogate$update() +#' +#' surrogate$learner$model +#' } +SurrogateRF = R6Class("SurrogateRF", + inherit = Surrogate, + + public = list( + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + #' + #' @template param_archive_surrogate + #' @template param_col_y_surrogate + #' @template param_cols_x_surrogate + initialize = function(archive = NULL, cols_x = NULL, col_y = NULL) { + assert_r6(archive, classes = "Archive", null.ok = TRUE) + + assert_character(cols_x, min.len = 1L, null.ok = TRUE) + assert_string(col_y, null.ok = TRUE) + + # https://github.com/mlr-org/mlr3learners/blob/51c15755438078fc99b11a9ca0147c7f2dbb96d8/R/LearnerRegrKM.R#L34 + ps = ps( + always.split.variables = p_uty(tags = "train"), + holdout = p_lgl(default = FALSE, tags = "train"), + importance = p_fct(c("none", "impurity", "impurity_corrected", "permutation"), tags = "train"), + keep.inbag = p_lgl(default = FALSE, tags = "train"), + max.depth = p_int(default = NULL, lower = 1L, special_vals = list(NULL), tags = "train"), + min.bucket = p_int(1L, default = 1L, tags = "train"), + min.node.size = p_int(1L, default = 5L, special_vals = list(NULL), tags = "train"), + mtry = p_int(lower = 1L, special_vals = list(NULL), tags = "train"), + mtry.ratio = p_dbl(lower = 0, upper = 1, tags = "train"), + na.action = p_fct(c("na.learn", "na.omit", "na.fail"), default = "na.learn", tags = "train"), + node.stats = p_lgl(default = FALSE, tags = "train"), + num.random.splits = p_int(1L, default = 1L, tags = "train", depends = quote(splitrule == "extratrees")), + num.threads = p_int(1L, default = 1L, tags = c("train", "predict", "threads")), + num.trees = p_int(1L, default = 500L, tags = c("train", "predict", "hotstart")), + oob.error = p_lgl(default = TRUE, tags = "train"), + poisson.tau = p_dbl(default = 1, tags = "train", depends = quote(splitrule == "poisson")), + regularization.factor = p_uty(default = 1, tags = "train"), + regularization.usedepth = p_lgl(default = FALSE, tags = "train"), + replace = p_lgl(default = TRUE, tags = "train"), + respect.unordered.factors = p_fct(c("ignore", "order", "partition"), tags = "train"), + sample.fraction = p_dbl(0L, 1L, tags = "train"), + save.memory = p_lgl(default = FALSE, tags = "train"), + scale.permutation.importance = p_lgl(default = FALSE, tags = "train", depends = quote(importance == "permutation")), + se.method = p_fct(c("jack", "infjack", "simple", "law_of_total_variance"), default = "infjack", tags = "predict"), + seed = p_int(default = NULL, special_vals = list(NULL), tags = c("train", "predict")), + split.select.weights = p_uty(default = NULL, tags = "train"), + splitrule = p_fct(c("variance", "extratrees", "maxstat", "beta", "poisson"), default = "variance", tags = "train"), + verbose = p_lgl(default = TRUE, tags = c("train", "predict")), + write.forest = p_lgl(default = TRUE, tags = "train"), + catch_errors = p_lgl(tags = "required"), + impute_method = p_fct(c("mean", "random"), tags = "required") + ) + ps$values = list(catch_errors = TRUE, impute_method = "random") + + super$initialize(learner = list(), archive = archive, cols_x = cols_x, cols_y = col_y, param_set = ps) + }, + + #' @description + #' Predict mean response and standard error. + #' + #' @param xdt ([data.table::data.table()])\cr + #' New data. One row per observation. + #' + #' @return [data.table::data.table()] with the columns `mean` and `se`. + predict = function(xdt) { + assert_xdt(xdt) + newdata = as_numeric_matrix(fix_xdt_missing(xdt, cols_x = self$cols_x, archive = self$archive)) + pv = self$param_set$get_values(tags = "predict") + + p = invoke(predict, private$.model, + data = newdata, + type = self$predict_type, + quantiles = private$.quantiles, + .args = pv) + + list(mean = p$predictions, se = p$se) + } + ), + + active = list( + + #' @template field_print_id + print_id = function(rhs) { + if (missing(rhs)) { + "ranger::ranger" + } else { + stop("$print_id is read-only.") + } + }, + + #' @template field_n_learner_surrogate + n_learner = function() { + 1L + }, + + + #' @template field_packages_surrogate + packages = function(rhs) { + if (missing(rhs)) { + "DiceKriging" + } else { + stop("$packages is read-only.") + } + }, + + #' @template field_feature_types_surrogate + feature_types = function(rhs) { + if (missing(rhs)) { + c("logical", "integer", "numeric") + } else { + stop("$feature_types is read-only.") + } + }, + + #' @template field_properties_surrogate + properties = function(rhs) { + if (missing(rhs)) { + character(0L) + } else { + stop("$properties is read-only.") + } + }, + + #' @template field_predict_type_surrogate + predict_type = function(rhs) { + if (missing(rhs)) { + "se" + } else { + stop("$predict_type is read-only.") + } + } + ), + + private = list( + + .model = NULL, + + # Train learner with new data. + .update = function() { + pv = self$param_set$get_values(tags = "train") + pv = convert_ratio(pv, "mtry", "mtry.ratio", length(self$cols_x)) + design = as_numeric_matrix(self$archive$data[, c(self$cols_x, self$cols_y), with = FALSE]) + + model = invoke(ranger::ranger, + dependent.variable.name = self$cols_y, + data = design, + .args = pv + ) + + private$.model = if (isTRUE(self$param_set$values$se.method %in% c("simple", "law_of_total_variance"))) { + prediction_nodes = mlr3misc::invoke(predict, model, data = design, type = "terminalNodes", .args = pv[setdiff(names(pv), "se.method")], predict.all = TRUE) + storage.mode(prediction_nodes$predictions) = "integer" + mu_sigma = .Call("c_ranger_mu_sigma", model, prediction_nodes$predictions, self$archive$data[[self$cols_y]]) + list(model = model, mu_sigma = mu_sigma) + } else { + list(model = model) + } + + self$learner$model = private$.model + invisible(NULL) + }, + + # Train learner with new data. + # Operates on an asynchronous archive and performs imputation as needed. + .update_async = function() { + xydt = self$archive$rush$fetch_tasks_with_state(states = c("queued", "running", "finished"))[, c(self$cols_x, self$cols_y, "state"), with = FALSE] + + if (self$param_set$values$impute_method == "mean") { + mean_y = mean(xydt[[self$cols_y]], na.rm = TRUE) + xydt[c("queued", "running"), (self$cols_y) := mean_y, on = "state"] + } else if (self$param_set$values$impute_method == "random") { + min_y = min(xydt[[self$cols_y]], na.rm = TRUE) + max_y = max(xydt[[self$cols_y]], na.rm = TRUE) + xydt[c("queued", "running"), (self$cols_y) := runif(.N, min = min_y, max = max_y), on = "state"] + } + set(xydt, j = "state", value = NULL) + + pv = self$param_set$get_values(tags = "train") + pv = convert_ratio(pv, "mtry", "mtry.ratio", length(self$cols_x)) + design = as_numeric_matrix(xydt[, c(self$cols_x, self$cols_y), with = FALSE]) + + model = invoke(ranger::ranger, + dependent.variable.name = self$cols_y, + data = design, + .args = pv + ) + + private$.model = if (isTRUE(self$param_set$values$se.method %in% c("simple", "law_of_total_variance"))) { + prediction_nodes = mlr3misc::invoke(predict, model, data = design, type = "terminalNodes", .args = pv[setdiff(names(pv), "se.method")], predict.all = TRUE) + storage.mode(prediction_nodes$predictions) = "integer" + mu_sigma = .Call("c_ranger_mu_sigma", model, prediction_nodes$predictions, xydt[[self$cols_y]]) + list(model = model, mu_sigma = mu_sigma) + } else { + list(model = model) + } + + self$learner$model = private$.model + invisible(NULL) + }, + + .reset = function() { + self$learner = list() + private$.model = NULL + }, + + deep_clone = function(name, value) { + switch(name, + .param_set = value$clone(deep = TRUE), + .archive = value$clone(deep = TRUE), + value + ) + } + ) +) + +# https://github.com/mlr-org/mlr3learners/blob/51c15755438078fc99b11a9ca0147c7f2dbb96d8/R/helpers.R#L16C1-L16C18 +as_numeric_matrix = function(x) { + x = as.matrix(x) + if (is.logical(x)) { + storage.mode(x) = "double" + } + x +} + +convert_ratio = function(pv, target, ratio, n) { + switch(to_decimal(c(target, ratio) %in% names(pv)) + 1L, + # !mtry && !mtry.ratio + pv, + + # !mtry && mtry.ratio + { + pv[[target]] = max(ceiling(pv[[ratio]] * n), 1) + remove_named(pv, ratio) + }, + + + # mtry && !mtry.ratio + pv, + + # mtry && mtry.ratio + stopf("Hyperparameters '%s' and '%s' are mutually exclusive", target, ratio) + ) +} From b945e2b8da3d166f44a5843da36fd9c98c30483a Mon Sep 17 00:00:00 2001 From: be-marc Date: Fri, 11 Jul 2025 12:06:40 +0200 Subject: [PATCH 096/126] ... --- DESCRIPTION | 2 + NAMESPACE | 2 + R/AcqOptimizerCmaes.R | 80 ++++++---- R/AcqOptimizerDirect.R | 2 +- R/AcqOptimizerDirect2.R | 123 +++++++++++++++ man/SurrogateRF.Rd | 184 ++++++++++++++++++++++ tests/testthat/test_AcqOptimizerCmaes.R | 16 +- tests/testthat/test_AcqOptimizerDirect2.R | 43 +++++ 8 files changed, 421 insertions(+), 31 deletions(-) create mode 100644 R/AcqOptimizerDirect2.R create mode 100644 man/SurrogateRF.Rd create mode 100644 tests/testthat/test_AcqOptimizerDirect2.R diff --git a/DESCRIPTION b/DESCRIPTION index d5b97a81..36f8a87a 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -96,6 +96,7 @@ Collate: 'AcqOptimizer.R' 'AcqOptimizerCmaes.R' 'AcqOptimizerDirect.R' + 'AcqOptimizerDirect2.R' 'mlr_input_trafos.R' 'InputTrafo.R' 'InputTrafoUnitcube.R' @@ -116,6 +117,7 @@ Collate: 'SurrogateGP.R' 'SurrogateLearner.R' 'SurrogateLearnerCollection.R' + 'SurrogateRF.R' 'TunerADBO.R' 'TunerAsyncMbo.R' 'TunerMbo.R' diff --git a/NAMESPACE b/NAMESPACE index 8cc27a7a..c8c99a2b 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -25,6 +25,7 @@ export(AcqFunctionStochasticEI) export(AcqOptimizer) export(AcqOptimizerCmaes) export(AcqOptimizerDirect) +export(AcqOptimizerDirect2) export(InputTrafo) export(InputTrafoUnitcube) export(LearnerRegrRangerMbo) @@ -41,6 +42,7 @@ export(Surrogate) export(SurrogateGP) export(SurrogateLearner) export(SurrogateLearnerCollection) +export(SurrogateRF) export(TunerADBO) export(TunerAsyncMbo) export(TunerMbo) diff --git a/R/AcqOptimizerCmaes.R b/R/AcqOptimizerCmaes.R index 2fd210d9..7264235c 100644 --- a/R/AcqOptimizerCmaes.R +++ b/R/AcqOptimizerCmaes.R @@ -10,24 +10,27 @@ AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", initialize = function(acq_function = NULL) { self$acq_function = assert_r6(acq_function, "AcqFunction", null.ok = TRUE) ps = ps( - fnscale = p_dbl(default = 1), - maxit = p_int(lower = 1L), - stopfitness = p_dbl(default = -Inf), - keep.best = p_lgl(default = TRUE), - sigma = p_uty(default = 0.5), - mu = p_int(lower = 1L), - lambda = p_int(lower = 1L), - weights = p_uty(), - damps = p_dbl(), - cs = p_dbl(), - ccum = p_dbl(), - ccov.1 = p_dbl(lower = 0), - ccov.mu = p_dbl(lower = 0), - diag.sigma = p_lgl(default = FALSE), - diag.eigen = p_lgl(default = FALSE), - diag.pop = p_lgl(default = FALSE), - diag.value = p_lgl(default = FALSE), - stop.tolx = p_dbl() # undocumented stop criterion + fnscale = p_dbl(default = 1), + maxit = p_int(lower = 1L), + stopfitness = p_dbl(default = -Inf), + keep.best = p_lgl(default = TRUE), + sigma = p_uty(default = 0.5), + mu = p_int(lower = 1L), + lambda = p_int(lower = 1L), + weights = p_uty(), + damps = p_dbl(), + cs = p_dbl(), + ccum = p_dbl(), + ccov.1 = p_dbl(lower = 0), + ccov.mu = p_dbl(lower = 0), + diag.sigma = p_lgl(default = FALSE), + diag.eigen = p_lgl(default = FALSE), + diag.pop = p_lgl(default = FALSE), + diag.value = p_lgl(default = FALSE), + stop.tolx = p_dbl(), # undocumented stop criterion + restart_strategy = p_fct(levels = c("none", "ipop"), init = "none"), + n_restarts = p_int(lower = 1L), + population_multiplier = p_int(lower = 1) # start_values = p_fct(default = "random", levels = c("random", "center", "custom")), # start = p_uty(default = NULL, depends = start_values == "custom"), # n_candidates = p_int(lower = 1, default = 1L), @@ -50,6 +53,12 @@ AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", pv = self$param_set$values pv$vectorized = TRUE par = set_names(as.numeric(self$acq_function$archive$best()[, self$acq_function$domain$ids(), with = FALSE]), self$acq_function$domain$ids()) + n_restarts = if (pv$restart_strategy == "ipop") pv$n_restarts else 1L + + # set package defaults if not set by user + # restarts needs lambda and mu to be set + if (is.null(pv$lambda)) pv$lambda = 4 + floor(3 * log(length(par))) + if (is.null(pv$mu)) pv$mu = floor(pv$lambda / 2) wrapper = function(xmat, fun, constants, direction) { xdt = as.data.table(t(xmat)) @@ -61,17 +70,30 @@ AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", constants = self$acq_function$constants$values direction = self$acq_function$codomain$direction - res = invoke(cmaes::cma_es, - par = par, - fn = wrapper, - lower = self$acq_function$domain$lower, - upper = self$acq_function$domain$upper, - control = pv, - fun = fun, - constants = constants, - direction = direction) - - as.data.table(as.list(set_names(c(res$par, res$value), c(self$acq_function$domain$ids(), self$acq_function$codomain$ids())))) + y = Inf + for (n in seq_len(n_restarts)) { + res = invoke(cmaes::cma_es, + par = par, + fn = wrapper, + lower = self$acq_function$domain$lower, + upper = self$acq_function$domain$upper, + control = pv, + fun = fun, + constants = constants, + direction = direction) + + if (res$value < y) { + y = res$value + par = set_names(res$par, self$acq_function$domain$ids()) + } + + if (n < n_restarts) { + pv$mu = pv$mu * pv$population_multiplier + pv$lambda = pv$lambda * pv$population_multiplier + } + } + + as.data.table(as.list(set_names(c(par, y * direction), c(self$acq_function$domain$ids(), self$acq_function$codomain$ids())))) }, #' @description diff --git a/R/AcqOptimizerDirect.R b/R/AcqOptimizerDirect.R index 86c1b0da..a89c912f 100644 --- a/R/AcqOptimizerDirect.R +++ b/R/AcqOptimizerDirect.R @@ -58,7 +58,7 @@ AcqOptimizerDirect = R6Class("AcqOptimizerDirect", constants = constants, direction = direction) - as.data.table(as.list(set_names(c(res$solution, res$objective), c(self$acq_function$domain$ids(), self$acq_function$codomain$ids())))) + as.data.table(as.list(set_names(c(res$solution, res$objective * direction), c(self$acq_function$domain$ids(), self$acq_function$codomain$ids())))) }, #' @description diff --git a/R/AcqOptimizerDirect2.R b/R/AcqOptimizerDirect2.R new file mode 100644 index 00000000..2a11ba02 --- /dev/null +++ b/R/AcqOptimizerDirect2.R @@ -0,0 +1,123 @@ +#' @export +AcqOptimizerDirect2 = R6Class("AcqOptimizerDirect2", + inherit = AcqOptimizer, + public = list( + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + #' + #' @param acq_function (`NULL` | [AcqFunction]). + initialize = function(acq_function = NULL) { + self$acq_function = assert_r6(acq_function, "AcqFunction", null.ok = TRUE) + ps = ps( + stopval = p_dbl(default = -Inf, lower = -Inf, upper = Inf), + xtol_rel = p_dbl(default = 1e-06, lower = 0, upper = Inf, special_vals = list(-1)), + xtol_abs = p_dbl(default = 0, lower = 0, upper = Inf, special_vals = list(-1)), + maxeval = p_int(lower = 1, default = 1000L, special_vals = list(-1)), + ftol_rel = p_dbl(default = 0, lower = 0, upper = Inf, special_vals = list(-1)), + ftol_abs = p_dbl(default = 0, lower = 0, upper = Inf, special_vals = list(-1)), + minf_max = p_dbl(default = -Inf), + random_restart_size = p_int(), + n_random_restarts = p_int() + # n_candidates = p_int(lower = 1, default = 1L), + # logging_level = p_fct(levels = c("fatal", "error", "warn", "info", "debug", "trace"), default = "warn"), + # warmstart = p_lgl(default = FALSE), + # warmstart_size = p_int(lower = 1L, special_vals = list("all")), + # skip_already_evaluated = p_lgl(default = TRUE), + # catch_errors = p_lgl(default = TRUE) + ) + # ps$values = list(n_candidates = 1, logging_level = "warn", warmstart = FALSE, skip_already_evaluated = TRUE, catch_errors = TRUE) + # ps$add_dep("warmstart_size", on = "warmstart", cond = CondEqual$new(TRUE)) + private$.param_set = ps + }, + + #' @description + #' Optimize the acquisition function. + #' + #' @return [data.table::data.table()] with 1 row per candidate. + optimize = function() { + pv = self$param_set$values + + + wrapper = function(x, fun, constants, direction) { + xdt = as.data.table(as.list(set_names(x, self$acq_function$domain$ids()))) + res = mlr3misc::invoke(fun, xdt = xdt, .args = constants)[[1]] + res * direction + } + + fun = get_private(self$acq_function)$.fun + constants = self$acq_function$constants$values + direction = self$acq_function$codomain$direction + + y = Inf + for (n in seq_len(pv$n_random_restarts)) { + # random restart + design = generate_design_random(self$acq_function$domain, n = pv$random_restart_size)$data + res = mlr3misc::invoke(fun, xdt = design, .args = constants)[[1]] * direction + i = which.min(res) + x0 = as.numeric(design[i, self$acq_function$domain$ids(), with = FALSE]) + + # optimize with nloptr + res = invoke(nloptr::nloptr, + eval_f = wrapper, + lb = self$acq_function$domain$lower, + ub = self$acq_function$domain$upper, + opts = c(pv, list(algorithm = "NLOPT_GN_DIRECT_L")), + eval_grad_f = NULL, + x0 = x0, + fun = fun, + constants = constants, + direction = direction) + + if (res$objective < y) { + y = res$objective + x = res$solution + } + } + as.data.table(as.list(set_names(c(x, y * direction), c(self$acq_function$domain$ids(), self$acq_function$codomain$ids())))) + }, + + #' @description + #' Reset the acquisition function optimizer. + #' + #' Currently not used. + reset = function() { + + } + ), + + active = list( + #' @template field_print_id + print_id = function(rhs) { + if (missing(rhs)) { + paste0("(", class(self$optimizer)[1L], " | ", class(self$terminator)[1L], ")") + } else { + stop("$print_id is read-only.") + } + }, + + #' @field param_set ([paradox::ParamSet])\cr + #' Set of hyperparameters. + param_set = function(rhs) { + if (!missing(rhs) && !identical(rhs, private$.param_set)) { + stop("$param_set is read-only.") + } + private$.param_set + } + ), + + private = list( + .param_set = NULL, + + deep_clone = function(name, value) { + switch(name, + optimizer = value$clone(deep = TRUE), + terminator = value$clone(deep = TRUE), + acq_function = if (!is.null(value)) value$clone(deep = TRUE) else NULL, + .param_set = value$clone(deep = TRUE), + value + ) + } + ) +) + diff --git a/man/SurrogateRF.Rd b/man/SurrogateRF.Rd new file mode 100644 index 00000000..368876f5 --- /dev/null +++ b/man/SurrogateRF.Rd @@ -0,0 +1,184 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/SurrogateRF.R +\name{SurrogateRF} +\alias{SurrogateRF} +\title{Surrogate Model Containing a Random Forest} +\description{ +Surrogate model containing a single Random Forest via \code{\link[ranger:ranger]{ranger::ranger()}} from package \CRANpkg{ranger}. +Update and predict methods are inspired from \link[mlr3learners:mlr_learners_regr.ranger]{mlr3learners::LearnerRegrRanger} from package \CRANpkg{mlr3learners}. + +Compared to using \link[mlr3learners:mlr_learners_regr.ranger]{mlr3learners::LearnerRegrRanger} within a \link{SurrogateLearner} the update and predict methods of this class are much more efficient +as they skip many assertions and checks naturally arising when using a \link{SurrogateLearner} wrapping a \link[mlr3learners:mlr_learners_regr.ranger]{mlr3learners::LearnerRegrRanger}. +} +\section{Parameters}{ + +\describe{ +\item{\code{catch_errors}}{\code{logical(1)}\cr +Should errors during updating the surrogate be caught and propagated to the \code{loop_function} which can then handle +the failed acquisition function optimization (as a result of the failed surrogate) appropriately by, e.g., proposing a randomly sampled point for evaluation? +Default is \code{TRUE}. +} +\item{\code{impute_method}}{\code{character(1)}\cr +Method to impute missing values in the case of updating on an asynchronous \link[bbotk:ArchiveAsync]{bbotk::ArchiveAsync} with pending evaluations. +Can be \code{"mean"} to use mean imputation or \code{"random"} to sample values uniformly at random between the empirical minimum and maximum. +Default is \code{"random"}. +} +} +For a description of all other parameters related to \code{\link[DiceKriging:km]{DiceKriging::km()}} directly, see the documentation of \code{\link[DiceKriging:km]{DiceKriging::km()}}. +\itemize{ +\item The predict type hyperparameter "type" defaults to "SK" (simple kriging). +\item The additional hyperparameter \code{nugget.stability} is used to overwrite the +hyperparameter \code{nugget} with \code{nugget.stability * var(y)} before training to +improve the numerical stability. We recommend a value of \code{1e-8}. +\item The additional hyperparameter \code{jitter} can be set to add +\verb{N(0, [jitter])}-distributed noise to the data before prediction to avoid +perfect interpolation. We recommend a value of \code{1e-12}. +} +} + +\examples{ +if (requireNamespace("DiceKriging") & + requireNamespace("rgenoud")) { + library(bbotk) + library(paradox) + + fun = function(xs) { + list(y = xs$x ^ 2) + } + domain = ps(x = p_dbl(lower = -10, upper = 10)) + codomain = ps(y = p_dbl(tags = "minimize")) + objective = ObjectiveRFun$new(fun = fun, domain = domain, codomain = codomain) + + instance = OptimInstanceBatchSingleCrit$new( + objective = objective, + terminator = trm("evals", n_evals = 5)) + + xdt = generate_design_random(instance$search_space, n = 4)$data + + instance$eval_batch(xdt) + + surrogate = SurrogateGP$new(archive = instance$archive) + surrogate$param_set$set_values( + covtype = "matern5_2", + optim.method = "gen", + control = list(trace = FALSE), + nugget.stability = 10^-8 + ) + + surrogate$update() + + surrogate$learner$model +} +} +\section{Super class}{ +\code{\link[mlr3mbo:Surrogate]{mlr3mbo::Surrogate}} -> \code{SurrogateRF} +} +\section{Active bindings}{ +\if{html}{\out{
    }} +\describe{ +\item{\code{print_id}}{(\code{character})\cr +Id used when printing.} + +\item{\code{n_learner}}{(\code{integer(1)})\cr +Returns the number of surrogate models.} + +\item{\code{packages}}{(\code{character()})\cr +Set of required packages. +A warning is signaled if at least one of the packages is not installed, but loaded (not attached) later on-demand via \code{\link[=requireNamespace]{requireNamespace()}}.} + +\item{\code{feature_types}}{(\code{character()})\cr +Stores the feature types the surrogate can handle, e.g. \code{"logical"}, \code{"numeric"}, or \code{"factor"}. +A complete list of candidate feature types, grouped by task type, is stored in \code{\link[mlr3:mlr_reflections]{mlr_reflections$task_feature_types}}.} + +\item{\code{properties}}{(\code{character()})\cr +Stores a set of properties/capabilities the surrogate has. +A complete list of candidate properties, grouped by task type, is stored in \code{\link[mlr3:mlr_reflections]{mlr_reflections$learner_properties}}.} + +\item{\code{predict_type}}{(\code{character(1)})\cr +Retrieves the currently active predict type, e.g. \code{"response"}.} +} +\if{html}{\out{
    }} +} +\section{Methods}{ +\subsection{Public methods}{ +\itemize{ +\item \href{#method-SurrogateRF-new}{\code{SurrogateRF$new()}} +\item \href{#method-SurrogateRF-predict}{\code{SurrogateRF$predict()}} +\item \href{#method-SurrogateRF-clone}{\code{SurrogateRF$clone()}} +} +} +\if{html}{\out{ +
    Inherited methods + +
    +}} +\if{html}{\out{
    }} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-SurrogateRF-new}{}}} +\subsection{Method \code{new()}}{ +Creates a new instance of this \link[R6:R6Class]{R6} class. +\subsection{Usage}{ +\if{html}{\out{
    }}\preformatted{SurrogateRF$new(archive = NULL, cols_x = NULL, col_y = NULL)}\if{html}{\out{
    }} +} + +\subsection{Arguments}{ +\if{html}{\out{
    }} +\describe{ +\item{\code{archive}}{(\link[bbotk:Archive]{bbotk::Archive} | \code{NULL})\cr +\link[bbotk:Archive]{bbotk::Archive} of the \link[bbotk:OptimInstance]{bbotk::OptimInstance}.} + +\item{\code{cols_x}}{(\code{character()} | \code{NULL})\cr +Column id's of variables that should be used as features. +By default, automatically inferred based on the archive.} + +\item{\code{col_y}}{(\code{character(1)} | \code{NULL})\cr +Column id of variable that should be used as a target. +By default, automatically inferred based on the archive.} +} +\if{html}{\out{
    }} +} +} +\if{html}{\out{
    }} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-SurrogateRF-predict}{}}} +\subsection{Method \code{predict()}}{ +Predict mean response and standard error. +\subsection{Usage}{ +\if{html}{\out{
    }}\preformatted{SurrogateRF$predict(xdt)}\if{html}{\out{
    }} +} + +\subsection{Arguments}{ +\if{html}{\out{
    }} +\describe{ +\item{\code{xdt}}{(\code{\link[data.table:data.table]{data.table::data.table()}})\cr +New data. One row per observation.} +} +\if{html}{\out{
    }} +} +\subsection{Returns}{ +\code{\link[data.table:data.table]{data.table::data.table()}} with the columns \code{mean} and \code{se}. +} +} +\if{html}{\out{
    }} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-SurrogateRF-clone}{}}} +\subsection{Method \code{clone()}}{ +The objects of this class are cloneable with this method. +\subsection{Usage}{ +\if{html}{\out{
    }}\preformatted{SurrogateRF$clone(deep = FALSE)}\if{html}{\out{
    }} +} + +\subsection{Arguments}{ +\if{html}{\out{
    }} +\describe{ +\item{\code{deep}}{Whether to make a deep clone.} +} +\if{html}{\out{
    }} +} +} +} diff --git a/tests/testthat/test_AcqOptimizerCmaes.R b/tests/testthat/test_AcqOptimizerCmaes.R index 56c3b502..fe6353f2 100644 --- a/tests/testthat/test_AcqOptimizerCmaes.R +++ b/tests/testthat/test_AcqOptimizerCmaes.R @@ -26,7 +26,7 @@ test_that("AcqOptimizerCmaes works with 2D", { expect_data_table(acqopt$optimize(), nrows = 1L) }) -test_that("AcqOptimizerCmaes works", { +test_that("AcqOptimizerCmaes works with instance", { instance = oi(OBJ_1D, terminator = trm("evals", n_evals = 10L)) design = generate_design_grid(instance$search_space, resolution = 4L)$data instance$eval_batch(design) @@ -41,3 +41,17 @@ test_that("AcqOptimizerCmaes works", { instance$archive$data }) + +test_that("AcqOptimizerCmaes works with ipop restart", { + instance = oi(OBJ_1D, terminator = trm("evals", n_evals = 10L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerCmaes$new(acq_function = acqfun) + acqopt$param_set$set_values(maxit = 10L, restart_strategy = "ipop", n_restarts = 3L, population_multiplier = 2L) + acqfun$surrogate$update() + acqfun$update() + expect_data_table(acqopt$optimize(), nrows = 1L) +}) diff --git a/tests/testthat/test_AcqOptimizerDirect2.R b/tests/testthat/test_AcqOptimizerDirect2.R new file mode 100644 index 00000000..de6eada5 --- /dev/null +++ b/tests/testthat/test_AcqOptimizerDirect2.R @@ -0,0 +1,43 @@ +test_that("AcqOptimizerDirect works", { + instance = oi(OBJ_1D, terminator = trm("evals", n_evals = 5L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerDirect2$new(acq_function = acqfun) + acqopt$param_set$set_values(maxeval = 200L, random_restart_size = 100L, n_random_restarts = 3L) + acqfun$surrogate$update() + acqfun$update() + expect_data_table(acqopt$optimize(), nrows = 1L) +}) + +test_that("AcqOptimizerDirect works with 2D", { + instance = oi(OBJ_2D, terminator = trm("evals", n_evals = 5L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerDirect$new(acq_function = acqfun) + acqopt$param_set$set_values(maxeval = 200L) + acqfun$surrogate$update() + acqfun$update() + expect_data_table(acqopt$optimize(), nrows = 1L) +}) + +test_that("AcqOptimizerDirect works", { + instance = oi(OBJ_1D, terminator = trm("evals", n_evals = 10L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerDirect$new(acq_function = acqfun) + acqopt$param_set$set_values(maxeval = 100L) + + optimizer = opt("mbo", acq_optimizer = acqopt, acq_function = acqfun, surrogate = surrogate) + optimizer$optimize(instance) + + instance$archive$data +}) From 2edf8d1b0f10ae7bb62cb6d6a34616e6b5f2f30b Mon Sep 17 00:00:00 2001 From: be-marc Date: Fri, 11 Jul 2025 12:59:36 +0200 Subject: [PATCH 097/126] ... --- DESCRIPTION | 1 + NAMESPACE | 1 + R/AcqOptimizerLbfgsb2.R | 123 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 125 insertions(+) create mode 100644 R/AcqOptimizerLbfgsb2.R diff --git a/DESCRIPTION b/DESCRIPTION index 36f8a87a..c2c3345d 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -97,6 +97,7 @@ Collate: 'AcqOptimizerCmaes.R' 'AcqOptimizerDirect.R' 'AcqOptimizerDirect2.R' + 'AcqOptimizerLbfgsb2.R' 'mlr_input_trafos.R' 'InputTrafo.R' 'InputTrafoUnitcube.R' diff --git a/NAMESPACE b/NAMESPACE index c8c99a2b..585d8a63 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -26,6 +26,7 @@ export(AcqOptimizer) export(AcqOptimizerCmaes) export(AcqOptimizerDirect) export(AcqOptimizerDirect2) +export(AcqOptimizerLbfgsb2) export(InputTrafo) export(InputTrafoUnitcube) export(LearnerRegrRangerMbo) diff --git a/R/AcqOptimizerLbfgsb2.R b/R/AcqOptimizerLbfgsb2.R new file mode 100644 index 00000000..80ac2af0 --- /dev/null +++ b/R/AcqOptimizerLbfgsb2.R @@ -0,0 +1,123 @@ +#' @export +AcqOptimizerLbfgsb2 = R6Class("AcqOptimizerLbfgsb2", + inherit = AcqOptimizer, + public = list( + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + #' + #' @param acq_function (`NULL` | [AcqFunction]). + initialize = function(acq_function = NULL) { + self$acq_function = assert_r6(acq_function, "AcqFunction", null.ok = TRUE) + ps = ps( + stopval = p_dbl(default = -Inf, lower = -Inf, upper = Inf), + xtol_rel = p_dbl(default = 1e-06, lower = 0, upper = Inf, special_vals = list(-1)), + xtol_abs = p_dbl(default = 0, lower = 0, upper = Inf, special_vals = list(-1)), + maxeval = p_int(lower = 1, default = 1000L, special_vals = list(-1)), + ftol_rel = p_dbl(default = 0, lower = 0, upper = Inf, special_vals = list(-1)), + ftol_abs = p_dbl(default = 0, lower = 0, upper = Inf, special_vals = list(-1)), + minf_max = p_dbl(default = -Inf), + random_restart_size = p_int(), + n_random_restarts = p_int() + # n_candidates = p_int(lower = 1, default = 1L), + # logging_level = p_fct(levels = c("fatal", "error", "warn", "info", "debug", "trace"), default = "warn"), + # warmstart = p_lgl(default = FALSE), + # warmstart_size = p_int(lower = 1L, special_vals = list("all")), + # skip_already_evaluated = p_lgl(default = TRUE), + # catch_errors = p_lgl(default = TRUE) + ) + # ps$values = list(n_candidates = 1, logging_level = "warn", warmstart = FALSE, skip_already_evaluated = TRUE, catch_errors = TRUE) + # ps$add_dep("warmstart_size", on = "warmstart", cond = CondEqual$new(TRUE)) + private$.param_set = ps + }, + + #' @description + #' Optimize the acquisition function. + #' + #' @return [data.table::data.table()] with 1 row per candidate. + optimize = function() { + pv = self$param_set$values + + + wrapper = function(x, fun, constants, direction) { + xdt = as.data.table(as.list(set_names(x, self$acq_function$domain$ids()))) + res = mlr3misc::invoke(fun, xdt = xdt, .args = constants)[[1]] + res * direction + } + + fun = get_private(self$acq_function)$.fun + constants = self$acq_function$constants$values + direction = self$acq_function$codomain$direction + + y = Inf + for (n in seq_len(pv$n_random_restarts)) { + # random restart + design = generate_design_random(self$acq_function$domain, n = pv$random_restart_size)$data + res = mlr3misc::invoke(fun, xdt = design, .args = constants)[[1]] * direction + i = which.min(res) + x0 = as.numeric(design[i, self$acq_function$domain$ids(), with = FALSE]) + + # optimize with nloptr + res = invoke(nloptr::nloptr, + eval_f = wrapper, + lb = self$acq_function$domain$lower, + ub = self$acq_function$domain$upper, + opts = c(pv, list(algorithm = "NLOPT_LD_LBFGS")), + eval_grad_f = NULL, + x0 = x0, + fun = fun, + constants = constants, + direction = direction) + + if (res$objective < y) { + y = res$objective + x = res$solution + } + } + as.data.table(as.list(set_names(c(x, y * direction), c(self$acq_function$domain$ids(), self$acq_function$codomain$ids())))) + }, + + #' @description + #' Reset the acquisition function optimizer. + #' + #' Currently not used. + reset = function() { + + } + ), + + active = list( + #' @template field_print_id + print_id = function(rhs) { + if (missing(rhs)) { + paste0("(", class(self$optimizer)[1L], " | ", class(self$terminator)[1L], ")") + } else { + stop("$print_id is read-only.") + } + }, + + #' @field param_set ([paradox::ParamSet])\cr + #' Set of hyperparameters. + param_set = function(rhs) { + if (!missing(rhs) && !identical(rhs, private$.param_set)) { + stop("$param_set is read-only.") + } + private$.param_set + } + ), + + private = list( + .param_set = NULL, + + deep_clone = function(name, value) { + switch(name, + optimizer = value$clone(deep = TRUE), + terminator = value$clone(deep = TRUE), + acq_function = if (!is.null(value)) value$clone(deep = TRUE) else NULL, + .param_set = value$clone(deep = TRUE), + value + ) + } + ) +) + From 4feda55e5c48571f26bae09f05bbd02eec090378 Mon Sep 17 00:00:00 2001 From: be-marc Date: Fri, 11 Jul 2025 14:11:43 +0200 Subject: [PATCH 098/126] ... --- NAMESPACE | 2 +- R/AcqOptimizerLbfgsb2.R | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/NAMESPACE b/NAMESPACE index 585d8a63..a3569f30 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -26,7 +26,7 @@ export(AcqOptimizer) export(AcqOptimizerCmaes) export(AcqOptimizerDirect) export(AcqOptimizerDirect2) -export(AcqOptimizerLbfgsb2) +export(AcqOptimizerLbfgsb) export(InputTrafo) export(InputTrafoUnitcube) export(LearnerRegrRangerMbo) diff --git a/R/AcqOptimizerLbfgsb2.R b/R/AcqOptimizerLbfgsb2.R index 80ac2af0..2fdd4c73 100644 --- a/R/AcqOptimizerLbfgsb2.R +++ b/R/AcqOptimizerLbfgsb2.R @@ -1,5 +1,5 @@ #' @export -AcqOptimizerLbfgsb2 = R6Class("AcqOptimizerLbfgsb2", +AcqOptimizerLbfgsb = R6Class("AcqOptimizerLbfgsb", inherit = AcqOptimizer, public = list( From 1745450706e4c2796536c47b534d8129e6a47d6d Mon Sep 17 00:00:00 2001 From: be-marc Date: Wed, 16 Jul 2025 14:12:09 +0200 Subject: [PATCH 099/126] ... --- DESCRIPTION | 3 +- NAMESPACE | 1 - R/AcqOptimizerCmaes.R | 36 +++-- R/AcqOptimizerDirect.R | 68 ++++++++-- R/AcqOptimizerDirect2.R | 123 ------------------ ...ptimizerLbfgsb2.R => AcqOptimizerLbfgsb.R} | 51 ++++++-- man/AcqOptimizerCmaes.Rd | 110 ++++++++++++++++ man/AcqOptimizerDirect.Rd | 112 ++++++++++++++++ man/AcqOptimizerLbfgsb.Rd | 112 ++++++++++++++++ man/mlr_learners_regr.ranger.Rd | 1 - tests/testthat/test_AcqOptimizerCmaes.R | 18 ++- tests/testthat/test_AcqOptimizerDirect.R | 34 ++++- tests/testthat/test_AcqOptimizerDirect2.R | 43 ------ tests/testthat/test_AcqOptimizerLbfgsb.R | 66 ++++++++++ 14 files changed, 562 insertions(+), 216 deletions(-) delete mode 100644 R/AcqOptimizerDirect2.R rename R/{AcqOptimizerLbfgsb2.R => AcqOptimizerLbfgsb.R} (66%) create mode 100644 man/AcqOptimizerCmaes.Rd create mode 100644 man/AcqOptimizerDirect.Rd create mode 100644 man/AcqOptimizerLbfgsb.Rd delete mode 100644 tests/testthat/test_AcqOptimizerDirect2.R create mode 100644 tests/testthat/test_AcqOptimizerLbfgsb.R diff --git a/DESCRIPTION b/DESCRIPTION index c2c3345d..86fb6940 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -96,8 +96,7 @@ Collate: 'AcqOptimizer.R' 'AcqOptimizerCmaes.R' 'AcqOptimizerDirect.R' - 'AcqOptimizerDirect2.R' - 'AcqOptimizerLbfgsb2.R' + 'AcqOptimizerLbfgsb.R' 'mlr_input_trafos.R' 'InputTrafo.R' 'InputTrafoUnitcube.R' diff --git a/NAMESPACE b/NAMESPACE index a3569f30..0af87a61 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -25,7 +25,6 @@ export(AcqFunctionStochasticEI) export(AcqOptimizer) export(AcqOptimizerCmaes) export(AcqOptimizerDirect) -export(AcqOptimizerDirect2) export(AcqOptimizerLbfgsb) export(InputTrafo) export(InputTrafoUnitcube) diff --git a/R/AcqOptimizerCmaes.R b/R/AcqOptimizerCmaes.R index 7264235c..a8fc3ae1 100644 --- a/R/AcqOptimizerCmaes.R +++ b/R/AcqOptimizerCmaes.R @@ -1,8 +1,20 @@ +#' @title CMA-ES Acquisition Function Optimizer +#' +#' @description +#' IPOP CMA-ES runs for `n_iterations` iterations. +#' The population size is increased by `population_multiplier` after each iteration. +#' In the first iteration, the start point is the best point in the archive. +#' In the subsequent iterations, the start point is a random point in the search space. +#' #' @export AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", inherit = AcqOptimizer, public = list( + #' @field state (`list()`)\cr + #' List of [cmaes::cma_es()] results. + state = NULL, + #' @description #' Creates a new instance of this [R6][R6::R6Class] class. #' @@ -29,8 +41,8 @@ AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", diag.value = p_lgl(default = FALSE), stop.tolx = p_dbl(), # undocumented stop criterion restart_strategy = p_fct(levels = c("none", "ipop"), init = "none"), - n_restarts = p_int(lower = 1L), - population_multiplier = p_int(lower = 1) + n_iterations = p_int(lower = 1L, init = 1L), + population_multiplier = p_int(lower = 1, init = 2L) # start_values = p_fct(default = "random", levels = c("random", "center", "custom")), # start = p_uty(default = NULL, depends = start_values == "custom"), # n_candidates = p_int(lower = 1, default = 1L), @@ -52,8 +64,8 @@ AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", optimize = function() { pv = self$param_set$values pv$vectorized = TRUE - par = set_names(as.numeric(self$acq_function$archive$best()[, self$acq_function$domain$ids(), with = FALSE]), self$acq_function$domain$ids()) - n_restarts = if (pv$restart_strategy == "ipop") pv$n_restarts else 1L + x = par = set_names(as.numeric(self$acq_function$archive$best()[, self$acq_function$domain$ids(), with = FALSE]), self$acq_function$domain$ids()) + n_iterations = if (pv$restart_strategy == "ipop") pv$n_iterations else 1L # set package defaults if not set by user # restarts needs lambda and mu to be set @@ -71,7 +83,7 @@ AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", direction = self$acq_function$codomain$direction y = Inf - for (n in seq_len(n_restarts)) { + for (n in seq_len(n_iterations)) { res = invoke(cmaes::cma_es, par = par, fn = wrapper, @@ -82,18 +94,20 @@ AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", constants = constants, direction = direction) + # set best if (res$value < y) { y = res$value - par = set_names(res$par, self$acq_function$domain$ids()) + x = set_names(res$par, self$acq_function$domain$ids()) } - if (n < n_restarts) { - pv$mu = pv$mu * pv$population_multiplier - pv$lambda = pv$lambda * pv$population_multiplier - } + pv$mu = pv$mu * pv$population_multiplier + pv$lambda = pv$lambda * pv$population_multiplier + par = unlist(generate_design_random(self$acq_function$domain, 1)$data[1, ]) + + self$state = c(self$state, set_names(list(res), paste0("iteration_", n))) } - as.data.table(as.list(set_names(c(par, y * direction), c(self$acq_function$domain$ids(), self$acq_function$codomain$ids())))) + as.data.table(as.list(set_names(c(x, y * direction), c(self$acq_function$domain$ids(), self$acq_function$codomain$ids())))) }, #' @description diff --git a/R/AcqOptimizerDirect.R b/R/AcqOptimizerDirect.R index a89c912f..04a2a597 100644 --- a/R/AcqOptimizerDirect.R +++ b/R/AcqOptimizerDirect.R @@ -1,8 +1,22 @@ +#' @title Direct Optimization Acquisition Function Optimizer +#' +#' @description +#' If `restart_strategy` is `"random"`, the optimizer runs for `n_iterations` iterations. +#' Each iteration starts with a random search of size `random_restart_size`. +#' The best point is used as the start point for the direct optimization. +#' +#' If `restart_strategy` is `"none"`, the only the direct optimization is performed. +#' The start point is the best point in the archive. +#' #' @export AcqOptimizerDirect = R6Class("AcqOptimizerDirect", inherit = AcqOptimizer, public = list( + #' @field state (`list()`)\cr + #' List of [nloptr::nloptr()] results. + state = NULL, + #' @description #' Creates a new instance of this [R6][R6::R6Class] class. #' @@ -16,7 +30,11 @@ AcqOptimizerDirect = R6Class("AcqOptimizerDirect", maxeval = p_int(lower = 1, default = 1000L, special_vals = list(-1)), ftol_rel = p_dbl(default = 0, lower = 0, upper = Inf, special_vals = list(-1)), ftol_abs = p_dbl(default = 0, lower = 0, upper = Inf, special_vals = list(-1)), - minf_max = p_dbl(default = -Inf) + minf_max = p_dbl(default = -Inf), + restart_strategy = p_fct(levels = c("none", "random"), init = "none"), + n_iterations = p_int(lower = 1, init = 1L), + random_restart_size = p_int(lower = 1, init = 100L) + # n_candidates = p_int(lower = 1, default = 1L), # logging_level = p_fct(levels = c("fatal", "error", "warn", "info", "debug", "trace"), default = "warn"), # warmstart = p_lgl(default = FALSE), @@ -35,7 +53,8 @@ AcqOptimizerDirect = R6Class("AcqOptimizerDirect", #' @return [data.table::data.table()] with 1 row per candidate. optimize = function() { pv = self$param_set$values - x0 = as.numeric(self$acq_function$archive$best()[, self$acq_function$domain$ids(), with = FALSE]) + n_iterations = if (pv$restart_strategy == "random") pv$n_iterations else 1L + wrapper = function(x, fun, constants, direction) { xdt = as.data.table(as.list(set_names(x, self$acq_function$domain$ids()))) @@ -47,18 +66,39 @@ AcqOptimizerDirect = R6Class("AcqOptimizerDirect", constants = self$acq_function$constants$values direction = self$acq_function$codomain$direction - res = invoke(nloptr::nloptr, - eval_f = wrapper, - lb = self$acq_function$domain$lower, - ub = self$acq_function$domain$upper, - opts = c(pv, list(algorithm = "NLOPT_GN_DIRECT_L")), - eval_grad_f = NULL, - x0 = x0, - fun = fun, - constants = constants, - direction = direction) - - as.data.table(as.list(set_names(c(res$solution, res$objective * direction), c(self$acq_function$domain$ids(), self$acq_function$codomain$ids())))) + y = Inf + for (n in seq_len(n_iterations)) { + + x0 = if (pv$restart_strategy == "none") { + as.numeric(self$acq_function$archive$best()[, self$acq_function$domain$ids(), with = FALSE]) + } else { + # random restart + design = generate_design_random(self$acq_function$domain, n = pv$random_restart_size)$data + res = mlr3misc::invoke(fun, xdt = design, .args = constants)[[1]] * direction + i = which.min(res) + as.numeric(design[i, self$acq_function$domain$ids(), with = FALSE]) + } + + # optimize with nloptr + res = invoke(nloptr::nloptr, + eval_f = wrapper, + lb = self$acq_function$domain$lower, + ub = self$acq_function$domain$upper, + opts = c(pv, list(algorithm = "NLOPT_GN_DIRECT_L")), + eval_grad_f = NULL, + x0 = x0, + fun = fun, + constants = constants, + direction = direction) + + if (res$objective < y) { + y = res$objective + x = res$solution + } + + self$state = c(self$state, set_names(list(res), paste0("iteration_", n))) + } + as.data.table(as.list(set_names(c(x, y * direction), c(self$acq_function$domain$ids(), self$acq_function$codomain$ids())))) }, #' @description diff --git a/R/AcqOptimizerDirect2.R b/R/AcqOptimizerDirect2.R deleted file mode 100644 index 2a11ba02..00000000 --- a/R/AcqOptimizerDirect2.R +++ /dev/null @@ -1,123 +0,0 @@ -#' @export -AcqOptimizerDirect2 = R6Class("AcqOptimizerDirect2", - inherit = AcqOptimizer, - public = list( - - #' @description - #' Creates a new instance of this [R6][R6::R6Class] class. - #' - #' @param acq_function (`NULL` | [AcqFunction]). - initialize = function(acq_function = NULL) { - self$acq_function = assert_r6(acq_function, "AcqFunction", null.ok = TRUE) - ps = ps( - stopval = p_dbl(default = -Inf, lower = -Inf, upper = Inf), - xtol_rel = p_dbl(default = 1e-06, lower = 0, upper = Inf, special_vals = list(-1)), - xtol_abs = p_dbl(default = 0, lower = 0, upper = Inf, special_vals = list(-1)), - maxeval = p_int(lower = 1, default = 1000L, special_vals = list(-1)), - ftol_rel = p_dbl(default = 0, lower = 0, upper = Inf, special_vals = list(-1)), - ftol_abs = p_dbl(default = 0, lower = 0, upper = Inf, special_vals = list(-1)), - minf_max = p_dbl(default = -Inf), - random_restart_size = p_int(), - n_random_restarts = p_int() - # n_candidates = p_int(lower = 1, default = 1L), - # logging_level = p_fct(levels = c("fatal", "error", "warn", "info", "debug", "trace"), default = "warn"), - # warmstart = p_lgl(default = FALSE), - # warmstart_size = p_int(lower = 1L, special_vals = list("all")), - # skip_already_evaluated = p_lgl(default = TRUE), - # catch_errors = p_lgl(default = TRUE) - ) - # ps$values = list(n_candidates = 1, logging_level = "warn", warmstart = FALSE, skip_already_evaluated = TRUE, catch_errors = TRUE) - # ps$add_dep("warmstart_size", on = "warmstart", cond = CondEqual$new(TRUE)) - private$.param_set = ps - }, - - #' @description - #' Optimize the acquisition function. - #' - #' @return [data.table::data.table()] with 1 row per candidate. - optimize = function() { - pv = self$param_set$values - - - wrapper = function(x, fun, constants, direction) { - xdt = as.data.table(as.list(set_names(x, self$acq_function$domain$ids()))) - res = mlr3misc::invoke(fun, xdt = xdt, .args = constants)[[1]] - res * direction - } - - fun = get_private(self$acq_function)$.fun - constants = self$acq_function$constants$values - direction = self$acq_function$codomain$direction - - y = Inf - for (n in seq_len(pv$n_random_restarts)) { - # random restart - design = generate_design_random(self$acq_function$domain, n = pv$random_restart_size)$data - res = mlr3misc::invoke(fun, xdt = design, .args = constants)[[1]] * direction - i = which.min(res) - x0 = as.numeric(design[i, self$acq_function$domain$ids(), with = FALSE]) - - # optimize with nloptr - res = invoke(nloptr::nloptr, - eval_f = wrapper, - lb = self$acq_function$domain$lower, - ub = self$acq_function$domain$upper, - opts = c(pv, list(algorithm = "NLOPT_GN_DIRECT_L")), - eval_grad_f = NULL, - x0 = x0, - fun = fun, - constants = constants, - direction = direction) - - if (res$objective < y) { - y = res$objective - x = res$solution - } - } - as.data.table(as.list(set_names(c(x, y * direction), c(self$acq_function$domain$ids(), self$acq_function$codomain$ids())))) - }, - - #' @description - #' Reset the acquisition function optimizer. - #' - #' Currently not used. - reset = function() { - - } - ), - - active = list( - #' @template field_print_id - print_id = function(rhs) { - if (missing(rhs)) { - paste0("(", class(self$optimizer)[1L], " | ", class(self$terminator)[1L], ")") - } else { - stop("$print_id is read-only.") - } - }, - - #' @field param_set ([paradox::ParamSet])\cr - #' Set of hyperparameters. - param_set = function(rhs) { - if (!missing(rhs) && !identical(rhs, private$.param_set)) { - stop("$param_set is read-only.") - } - private$.param_set - } - ), - - private = list( - .param_set = NULL, - - deep_clone = function(name, value) { - switch(name, - optimizer = value$clone(deep = TRUE), - terminator = value$clone(deep = TRUE), - acq_function = if (!is.null(value)) value$clone(deep = TRUE) else NULL, - .param_set = value$clone(deep = TRUE), - value - ) - } - ) -) - diff --git a/R/AcqOptimizerLbfgsb2.R b/R/AcqOptimizerLbfgsb.R similarity index 66% rename from R/AcqOptimizerLbfgsb2.R rename to R/AcqOptimizerLbfgsb.R index 2fdd4c73..933d4ee0 100644 --- a/R/AcqOptimizerLbfgsb2.R +++ b/R/AcqOptimizerLbfgsb.R @@ -1,8 +1,23 @@ +#' @title L-BFGS-B Acquisition Function Optimizer +#' +#' @description +#' If `restart_strategy` is `"random"`, the optimizer runs for `n_iterations` iterations. +#' Each iteration starts with a random search of size `random_restart_size`. +#' The best point is used as the start point for the L-BFGS-B optimization. +#' +#' If `restart_strategy` is `"none"`, the only the L-BFGS-B optimization is performed. +#' The start point is the best point in the archive. +#' +#' @export #' @export AcqOptimizerLbfgsb = R6Class("AcqOptimizerLbfgsb", inherit = AcqOptimizer, public = list( + #' @field state (`list()`)\cr + #' List of [nloptr::nloptr()] results. + state = NULL, + #' @description #' Creates a new instance of this [R6][R6::R6Class] class. #' @@ -17,8 +32,9 @@ AcqOptimizerLbfgsb = R6Class("AcqOptimizerLbfgsb", ftol_rel = p_dbl(default = 0, lower = 0, upper = Inf, special_vals = list(-1)), ftol_abs = p_dbl(default = 0, lower = 0, upper = Inf, special_vals = list(-1)), minf_max = p_dbl(default = -Inf), - random_restart_size = p_int(), - n_random_restarts = p_int() + restart_strategy = p_fct(levels = c("none", "random"), init = "none"), + n_iterations = p_int(lower = 1, init = 1L), + random_restart_size = p_int(lower = 1, init = 100L) # n_candidates = p_int(lower = 1, default = 1L), # logging_level = p_fct(levels = c("fatal", "error", "warn", "info", "debug", "trace"), default = "warn"), # warmstart = p_lgl(default = FALSE), @@ -37,6 +53,7 @@ AcqOptimizerLbfgsb = R6Class("AcqOptimizerLbfgsb", #' @return [data.table::data.table()] with 1 row per candidate. optimize = function() { pv = self$param_set$values + n_iterations = if (pv$restart_strategy == "random") pv$n_iterations else 1L wrapper = function(x, fun, constants, direction) { @@ -50,20 +67,30 @@ AcqOptimizerLbfgsb = R6Class("AcqOptimizerLbfgsb", direction = self$acq_function$codomain$direction y = Inf - for (n in seq_len(pv$n_random_restarts)) { - # random restart - design = generate_design_random(self$acq_function$domain, n = pv$random_restart_size)$data - res = mlr3misc::invoke(fun, xdt = design, .args = constants)[[1]] * direction - i = which.min(res) - x0 = as.numeric(design[i, self$acq_function$domain$ids(), with = FALSE]) + for (n in seq_len(n_iterations)) { + + x0 = if (pv$restart_strategy == "none") { + as.numeric(self$acq_function$archive$best()[, self$acq_function$domain$ids(), with = FALSE]) + } else { + # random restart + design = generate_design_random(self$acq_function$domain, n = pv$random_restart_size)$data + res = mlr3misc::invoke(fun, xdt = design, .args = constants)[[1]] * direction + i = which.min(res) + as.numeric(design[i, self$acq_function$domain$ids(), with = FALSE]) + } + + eval_grad_f = function(x, fun, constants, direction) { + invoke(nloptr::nl.grad, x0 = x, fn = wrapper, fun = fun, constants = constants, direction = direction) + } + saveguard_epsilon = 1e-5 # optimize with nloptr res = invoke(nloptr::nloptr, eval_f = wrapper, - lb = self$acq_function$domain$lower, - ub = self$acq_function$domain$upper, + lb = self$acq_function$domain$lower + saveguard_epsilon, + ub = self$acq_function$domain$upper - saveguard_epsilon, opts = c(pv, list(algorithm = "NLOPT_LD_LBFGS")), - eval_grad_f = NULL, + eval_grad_f = eval_grad_f, x0 = x0, fun = fun, constants = constants, @@ -73,6 +100,8 @@ AcqOptimizerLbfgsb = R6Class("AcqOptimizerLbfgsb", y = res$objective x = res$solution } + + self$state = c(self$state, set_names(list(res), paste0("iteration_", n))) } as.data.table(as.list(set_names(c(x, y * direction), c(self$acq_function$domain$ids(), self$acq_function$codomain$ids())))) }, diff --git a/man/AcqOptimizerCmaes.Rd b/man/AcqOptimizerCmaes.Rd new file mode 100644 index 00000000..11122ef3 --- /dev/null +++ b/man/AcqOptimizerCmaes.Rd @@ -0,0 +1,110 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/AcqOptimizerCmaes.R +\name{AcqOptimizerCmaes} +\alias{AcqOptimizerCmaes} +\title{CMA-ES Acquisition Function Optimizer} +\description{ +IPOP CMA-ES runs for \code{n_iterations} iterations. +The population size is increased by \code{population_multiplier} after each iteration. +In the first iteration, the start point is the best point in the archive. +In the subsequent iterations, the start point is a random point in the search space. +} +\section{Super class}{ +\code{\link[mlr3mbo:AcqOptimizer]{mlr3mbo::AcqOptimizer}} -> \code{AcqOptimizerCmaes} +} +\section{Public fields}{ +\if{html}{\out{
    }} +\describe{ +\item{\code{state}}{(\code{list()})\cr +List of \code{\link[cmaes:cma_es]{cmaes::cma_es()}} results.} +} +\if{html}{\out{
    }} +} +\section{Active bindings}{ +\if{html}{\out{
    }} +\describe{ +\item{\code{print_id}}{(\code{character})\cr +Id used when printing.} + +\item{\code{param_set}}{(\link[paradox:ParamSet]{paradox::ParamSet})\cr +Set of hyperparameters.} +} +\if{html}{\out{
    }} +} +\section{Methods}{ +\subsection{Public methods}{ +\itemize{ +\item \href{#method-AcqOptimizerCmaes-new}{\code{AcqOptimizerCmaes$new()}} +\item \href{#method-AcqOptimizerCmaes-optimize}{\code{AcqOptimizerCmaes$optimize()}} +\item \href{#method-AcqOptimizerCmaes-reset}{\code{AcqOptimizerCmaes$reset()}} +\item \href{#method-AcqOptimizerCmaes-clone}{\code{AcqOptimizerCmaes$clone()}} +} +} +\if{html}{\out{ +
    Inherited methods + +
    +}} +\if{html}{\out{
    }} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerCmaes-new}{}}} +\subsection{Method \code{new()}}{ +Creates a new instance of this \link[R6:R6Class]{R6} class. +\subsection{Usage}{ +\if{html}{\out{
    }}\preformatted{AcqOptimizerCmaes$new(acq_function = NULL)}\if{html}{\out{
    }} +} + +\subsection{Arguments}{ +\if{html}{\out{
    }} +\describe{ +\item{\code{acq_function}}{(\code{NULL} | \link{AcqFunction}).} +} +\if{html}{\out{
    }} +} +} +\if{html}{\out{
    }} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerCmaes-optimize}{}}} +\subsection{Method \code{optimize()}}{ +Optimize the acquisition function. +\subsection{Usage}{ +\if{html}{\out{
    }}\preformatted{AcqOptimizerCmaes$optimize()}\if{html}{\out{
    }} +} + +\subsection{Returns}{ +\code{\link[data.table:data.table]{data.table::data.table()}} with 1 row per candidate. +} +} +\if{html}{\out{
    }} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerCmaes-reset}{}}} +\subsection{Method \code{reset()}}{ +Reset the acquisition function optimizer. + +Currently not used. +\subsection{Usage}{ +\if{html}{\out{
    }}\preformatted{AcqOptimizerCmaes$reset()}\if{html}{\out{
    }} +} + +} +\if{html}{\out{
    }} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerCmaes-clone}{}}} +\subsection{Method \code{clone()}}{ +The objects of this class are cloneable with this method. +\subsection{Usage}{ +\if{html}{\out{
    }}\preformatted{AcqOptimizerCmaes$clone(deep = FALSE)}\if{html}{\out{
    }} +} + +\subsection{Arguments}{ +\if{html}{\out{
    }} +\describe{ +\item{\code{deep}}{Whether to make a deep clone.} +} +\if{html}{\out{
    }} +} +} +} diff --git a/man/AcqOptimizerDirect.Rd b/man/AcqOptimizerDirect.Rd new file mode 100644 index 00000000..71719b78 --- /dev/null +++ b/man/AcqOptimizerDirect.Rd @@ -0,0 +1,112 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/AcqOptimizerDirect.R +\name{AcqOptimizerDirect} +\alias{AcqOptimizerDirect} +\title{Direct Optimization Acquisition Function Optimizer} +\description{ +If \code{restart_strategy} is \code{"random"}, the optimizer runs for \code{n_iterations} iterations. +Each iteration starts with a random search of size \code{random_restart_size}. +The best point is used as the start point for the direct optimization. + +If \code{restart_strategy} is \code{"none"}, the only the direct optimization is performed. +The start point is the best point in the archive. +} +\section{Super class}{ +\code{\link[mlr3mbo:AcqOptimizer]{mlr3mbo::AcqOptimizer}} -> \code{AcqOptimizerDirect} +} +\section{Public fields}{ +\if{html}{\out{
    }} +\describe{ +\item{\code{state}}{(\code{list()})\cr +List of \code{\link[nloptr:nloptr]{nloptr::nloptr()}} results.} +} +\if{html}{\out{
    }} +} +\section{Active bindings}{ +\if{html}{\out{
    }} +\describe{ +\item{\code{print_id}}{(\code{character})\cr +Id used when printing.} + +\item{\code{param_set}}{(\link[paradox:ParamSet]{paradox::ParamSet})\cr +Set of hyperparameters.} +} +\if{html}{\out{
    }} +} +\section{Methods}{ +\subsection{Public methods}{ +\itemize{ +\item \href{#method-AcqOptimizerDirect-new}{\code{AcqOptimizerDirect$new()}} +\item \href{#method-AcqOptimizerDirect-optimize}{\code{AcqOptimizerDirect$optimize()}} +\item \href{#method-AcqOptimizerDirect-reset}{\code{AcqOptimizerDirect$reset()}} +\item \href{#method-AcqOptimizerDirect-clone}{\code{AcqOptimizerDirect$clone()}} +} +} +\if{html}{\out{ +
    Inherited methods + +
    +}} +\if{html}{\out{
    }} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerDirect-new}{}}} +\subsection{Method \code{new()}}{ +Creates a new instance of this \link[R6:R6Class]{R6} class. +\subsection{Usage}{ +\if{html}{\out{
    }}\preformatted{AcqOptimizerDirect$new(acq_function = NULL)}\if{html}{\out{
    }} +} + +\subsection{Arguments}{ +\if{html}{\out{
    }} +\describe{ +\item{\code{acq_function}}{(\code{NULL} | \link{AcqFunction}).} +} +\if{html}{\out{
    }} +} +} +\if{html}{\out{
    }} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerDirect-optimize}{}}} +\subsection{Method \code{optimize()}}{ +Optimize the acquisition function. +\subsection{Usage}{ +\if{html}{\out{
    }}\preformatted{AcqOptimizerDirect$optimize()}\if{html}{\out{
    }} +} + +\subsection{Returns}{ +\code{\link[data.table:data.table]{data.table::data.table()}} with 1 row per candidate. +} +} +\if{html}{\out{
    }} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerDirect-reset}{}}} +\subsection{Method \code{reset()}}{ +Reset the acquisition function optimizer. + +Currently not used. +\subsection{Usage}{ +\if{html}{\out{
    }}\preformatted{AcqOptimizerDirect$reset()}\if{html}{\out{
    }} +} + +} +\if{html}{\out{
    }} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerDirect-clone}{}}} +\subsection{Method \code{clone()}}{ +The objects of this class are cloneable with this method. +\subsection{Usage}{ +\if{html}{\out{
    }}\preformatted{AcqOptimizerDirect$clone(deep = FALSE)}\if{html}{\out{
    }} +} + +\subsection{Arguments}{ +\if{html}{\out{
    }} +\describe{ +\item{\code{deep}}{Whether to make a deep clone.} +} +\if{html}{\out{
    }} +} +} +} diff --git a/man/AcqOptimizerLbfgsb.Rd b/man/AcqOptimizerLbfgsb.Rd new file mode 100644 index 00000000..c4e4954b --- /dev/null +++ b/man/AcqOptimizerLbfgsb.Rd @@ -0,0 +1,112 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/AcqOptimizerLbfgsb.R +\name{AcqOptimizerLbfgsb} +\alias{AcqOptimizerLbfgsb} +\title{L-BFGS-B Acquisition Function Optimizer} +\description{ +If \code{restart_strategy} is \code{"random"}, the optimizer runs for \code{n_iterations} iterations. +Each iteration starts with a random search of size \code{random_restart_size}. +The best point is used as the start point for the L-BFGS-B optimization. + +If \code{restart_strategy} is \code{"none"}, the only the L-BFGS-B optimization is performed. +The start point is the best point in the archive. +} +\section{Super class}{ +\code{\link[mlr3mbo:AcqOptimizer]{mlr3mbo::AcqOptimizer}} -> \code{AcqOptimizerLbfgsb} +} +\section{Public fields}{ +\if{html}{\out{
    }} +\describe{ +\item{\code{state}}{(\code{list()})\cr +List of \code{\link[nloptr:nloptr]{nloptr::nloptr()}} results.} +} +\if{html}{\out{
    }} +} +\section{Active bindings}{ +\if{html}{\out{
    }} +\describe{ +\item{\code{print_id}}{(\code{character})\cr +Id used when printing.} + +\item{\code{param_set}}{(\link[paradox:ParamSet]{paradox::ParamSet})\cr +Set of hyperparameters.} +} +\if{html}{\out{
    }} +} +\section{Methods}{ +\subsection{Public methods}{ +\itemize{ +\item \href{#method-AcqOptimizerLbfgsb-new}{\code{AcqOptimizerLbfgsb$new()}} +\item \href{#method-AcqOptimizerLbfgsb-optimize}{\code{AcqOptimizerLbfgsb$optimize()}} +\item \href{#method-AcqOptimizerLbfgsb-reset}{\code{AcqOptimizerLbfgsb$reset()}} +\item \href{#method-AcqOptimizerLbfgsb-clone}{\code{AcqOptimizerLbfgsb$clone()}} +} +} +\if{html}{\out{ +
    Inherited methods + +
    +}} +\if{html}{\out{
    }} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerLbfgsb-new}{}}} +\subsection{Method \code{new()}}{ +Creates a new instance of this \link[R6:R6Class]{R6} class. +\subsection{Usage}{ +\if{html}{\out{
    }}\preformatted{AcqOptimizerLbfgsb$new(acq_function = NULL)}\if{html}{\out{
    }} +} + +\subsection{Arguments}{ +\if{html}{\out{
    }} +\describe{ +\item{\code{acq_function}}{(\code{NULL} | \link{AcqFunction}).} +} +\if{html}{\out{
    }} +} +} +\if{html}{\out{
    }} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerLbfgsb-optimize}{}}} +\subsection{Method \code{optimize()}}{ +Optimize the acquisition function. +\subsection{Usage}{ +\if{html}{\out{
    }}\preformatted{AcqOptimizerLbfgsb$optimize()}\if{html}{\out{
    }} +} + +\subsection{Returns}{ +\code{\link[data.table:data.table]{data.table::data.table()}} with 1 row per candidate. +} +} +\if{html}{\out{
    }} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerLbfgsb-reset}{}}} +\subsection{Method \code{reset()}}{ +Reset the acquisition function optimizer. + +Currently not used. +\subsection{Usage}{ +\if{html}{\out{
    }}\preformatted{AcqOptimizerLbfgsb$reset()}\if{html}{\out{
    }} +} + +} +\if{html}{\out{
    }} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerLbfgsb-clone}{}}} +\subsection{Method \code{clone()}}{ +The objects of this class are cloneable with this method. +\subsection{Usage}{ +\if{html}{\out{
    }}\preformatted{AcqOptimizerLbfgsb$clone(deep = FALSE)}\if{html}{\out{
    }} +} + +\subsection{Arguments}{ +\if{html}{\out{
    }} +\describe{ +\item{\code{deep}}{Whether to make a deep clone.} +} +\if{html}{\out{
    }} +} +} +} diff --git a/man/mlr_learners_regr.ranger.Rd b/man/mlr_learners_regr.ranger.Rd index f0eb53ee..f0c824b1 100644 --- a/man/mlr_learners_regr.ranger.Rd +++ b/man/mlr_learners_regr.ranger.Rd @@ -30,7 +30,6 @@ Calls \code{\link[ranger:ranger]{ranger::ranger()}} from package \CRANpkg{ranger
  • mlr3::Learner$help()
  • mlr3::Learner$predict()
  • mlr3::Learner$predict_newdata()
    • -
  • mlr3::Learner$predict_newdata_fast()
  • mlr3::Learner$print()
  • mlr3::Learner$reset()
  • mlr3::Learner$selected_features()
    • diff --git a/tests/testthat/test_AcqOptimizerCmaes.R b/tests/testthat/test_AcqOptimizerCmaes.R index fe6353f2..bdef6545 100644 --- a/tests/testthat/test_AcqOptimizerCmaes.R +++ b/tests/testthat/test_AcqOptimizerCmaes.R @@ -6,10 +6,13 @@ test_that("AcqOptimizerCmaes works", { surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) acqfun = acqf("ei", surrogate = surrogate) acqopt = AcqOptimizerCmaes$new(acq_function = acqfun) - acqopt$param_set$set_values(maxit = 10L) + acqopt$param_set$set_values(maxit = 1000L) acqfun$surrogate$update() acqfun$update() expect_data_table(acqopt$optimize(), nrows = 1L) + expect_list(acqopt$state) + expect_names(names(acqopt$state), must.include = "iteration_1") + expect_class(acqopt$state$iteration_1, "cma_es.result") }) test_that("AcqOptimizerCmaes works with 2D", { @@ -24,6 +27,9 @@ test_that("AcqOptimizerCmaes works with 2D", { acqfun$surrogate$update() acqfun$update() expect_data_table(acqopt$optimize(), nrows = 1L) + expect_list(acqopt$state) + expect_names(names(acqopt$state), must.include = "iteration_1") + expect_class(acqopt$state$iteration_1, "cma_es.result") }) test_that("AcqOptimizerCmaes works with instance", { @@ -37,9 +43,7 @@ test_that("AcqOptimizerCmaes works with instance", { acqopt$param_set$set_values(maxit = 10L) optimizer = opt("mbo", acq_optimizer = acqopt, acq_function = acqfun, surrogate = surrogate) - optimizer$optimize(instance) - - instance$archive$data + expect_data_table(optimizer$optimize(instance), nrow = 1L) }) test_that("AcqOptimizerCmaes works with ipop restart", { @@ -50,8 +54,12 @@ test_that("AcqOptimizerCmaes works with ipop restart", { surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) acqfun = acqf("ei", surrogate = surrogate) acqopt = AcqOptimizerCmaes$new(acq_function = acqfun) - acqopt$param_set$set_values(maxit = 10L, restart_strategy = "ipop", n_restarts = 3L, population_multiplier = 2L) + acqopt$param_set$set_values(maxit = 10L, restart_strategy = "ipop", n_iterations = 3L, population_multiplier = 2L) acqfun$surrogate$update() acqfun$update() + expect_data_table(acqopt$optimize(), nrows = 1L) + expect_list(acqopt$state, len = 3L) + expect_names(names(acqopt$state), identical.to = c("iteration_1", "iteration_2", "iteration_3")) + walk(acqopt$state, function(x) expect_class(x, "cma_es.result")) }) diff --git a/tests/testthat/test_AcqOptimizerDirect.R b/tests/testthat/test_AcqOptimizerDirect.R index 16da3a2d..06f4be71 100644 --- a/tests/testthat/test_AcqOptimizerDirect.R +++ b/tests/testthat/test_AcqOptimizerDirect.R @@ -6,10 +6,14 @@ test_that("AcqOptimizerDirect works", { surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) acqfun = acqf("ei", surrogate = surrogate) acqopt = AcqOptimizerDirect$new(acq_function = acqfun) - acqopt$param_set$set_values(maxeval = 200L) + acqopt$param_set$set_values(maxeval = 200L, restart_strategy = "none") acqfun$surrogate$update() acqfun$update() + expect_data_table(acqopt$optimize(), nrows = 1L) + expect_list(acqopt$state) + expect_names(names(acqopt$state), must.include = "iteration_1") + expect_class(acqopt$state$iteration_1, "nloptr") }) test_that("AcqOptimizerDirect works with 2D", { @@ -23,10 +27,14 @@ test_that("AcqOptimizerDirect works with 2D", { acqopt$param_set$set_values(maxeval = 200L) acqfun$surrogate$update() acqfun$update() + expect_data_table(acqopt$optimize(), nrows = 1L) + expect_list(acqopt$state) + expect_names(names(acqopt$state), must.include = "iteration_1") + expect_class(acqopt$state$iteration_1, "nloptr") }) -test_that("AcqOptimizerDirect works", { +test_that("AcqOptimizerDirect works with instance", { instance = oi(OBJ_1D, terminator = trm("evals", n_evals = 10L)) design = generate_design_grid(instance$search_space, resolution = 4L)$data instance$eval_batch(design) @@ -34,10 +42,26 @@ test_that("AcqOptimizerDirect works", { surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) acqfun = acqf("ei", surrogate = surrogate) acqopt = AcqOptimizerDirect$new(acq_function = acqfun) - acqopt$param_set$set_values(maxeval = 100L) + acqopt$param_set$set_values(maxeval = 10L) optimizer = opt("mbo", acq_optimizer = acqopt, acq_function = acqfun, surrogate = surrogate) - optimizer$optimize(instance) + expect_data_table(optimizer$optimize(instance), nrow = 1L) +}) + +test_that("AcqOptimizerDirect works with random restart", { + instance = oi(OBJ_2D, terminator = trm("evals", n_evals = 5L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerDirect$new(acq_function = acqfun) + acqopt$param_set$set_values(maxeval = 200L, restart_strategy = "random", n_restarts = 3L, random_restart_size = 20L) + acqfun$surrogate$update() + acqfun$update() - instance$archive$data + expect_data_table(acqopt$optimize(), nrows = 1L) + expect_list(acqopt$state, len = 3L) + expect_names(names(acqopt$state), identical.to = c("iteration_1", "iteration_2", "iteration_3")) + walk(acqopt$state, function(x) expect_class(x, "nloptr")) }) diff --git a/tests/testthat/test_AcqOptimizerDirect2.R b/tests/testthat/test_AcqOptimizerDirect2.R deleted file mode 100644 index de6eada5..00000000 --- a/tests/testthat/test_AcqOptimizerDirect2.R +++ /dev/null @@ -1,43 +0,0 @@ -test_that("AcqOptimizerDirect works", { - instance = oi(OBJ_1D, terminator = trm("evals", n_evals = 5L)) - design = generate_design_grid(instance$search_space, resolution = 4L)$data - instance$eval_batch(design) - - surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) - acqfun = acqf("ei", surrogate = surrogate) - acqopt = AcqOptimizerDirect2$new(acq_function = acqfun) - acqopt$param_set$set_values(maxeval = 200L, random_restart_size = 100L, n_random_restarts = 3L) - acqfun$surrogate$update() - acqfun$update() - expect_data_table(acqopt$optimize(), nrows = 1L) -}) - -test_that("AcqOptimizerDirect works with 2D", { - instance = oi(OBJ_2D, terminator = trm("evals", n_evals = 5L)) - design = generate_design_grid(instance$search_space, resolution = 4L)$data - instance$eval_batch(design) - - surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) - acqfun = acqf("ei", surrogate = surrogate) - acqopt = AcqOptimizerDirect$new(acq_function = acqfun) - acqopt$param_set$set_values(maxeval = 200L) - acqfun$surrogate$update() - acqfun$update() - expect_data_table(acqopt$optimize(), nrows = 1L) -}) - -test_that("AcqOptimizerDirect works", { - instance = oi(OBJ_1D, terminator = trm("evals", n_evals = 10L)) - design = generate_design_grid(instance$search_space, resolution = 4L)$data - instance$eval_batch(design) - - surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) - acqfun = acqf("ei", surrogate = surrogate) - acqopt = AcqOptimizerDirect$new(acq_function = acqfun) - acqopt$param_set$set_values(maxeval = 100L) - - optimizer = opt("mbo", acq_optimizer = acqopt, acq_function = acqfun, surrogate = surrogate) - optimizer$optimize(instance) - - instance$archive$data -}) diff --git a/tests/testthat/test_AcqOptimizerLbfgsb.R b/tests/testthat/test_AcqOptimizerLbfgsb.R new file mode 100644 index 00000000..be453689 --- /dev/null +++ b/tests/testthat/test_AcqOptimizerLbfgsb.R @@ -0,0 +1,66 @@ +test_that("AcqOptimizerLbfgsb works", { + instance = oi(OBJ_1D, terminator = trm("evals", n_evals = 5L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerLbfgsb$new(acq_function = acqfun) + acqopt$param_set$set_values(maxeval = 200L, restart_strategy = "none") + acqfun$surrogate$update() + acqfun$update() + + expect_data_table(acqopt$optimize(), nrows = 1L) + expect_list(acqopt$result) + expect_names(names(acqopt$result), must.include = "iteration_1") + expect_class(acqopt$result$iteration_1, "nloptr") +}) + +test_that("AcqOptimizerLbfgsb works with 2D", { + instance = oi(OBJ_2D, terminator = trm("evals", n_evals = 5L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerLbfgsb$new(acq_function = acqfun) + acqopt$param_set$set_values(maxeval = 200L) + acqfun$surrogate$update() + acqfun$update() + + expect_data_table(acqopt$optimize(), nrows = 1L) + expect_list(acqopt$result) + expect_names(names(acqopt$result), must.include = "iteration_1") + expect_class(acqopt$result$iteration_1, "nloptr") +}) + +test_that("AcqOptimizerLbfgsb works with instance", { + instance = oi(OBJ_1D, terminator = trm("evals", n_evals = 10L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerLbfgsb$new(acq_function = acqfun) + acqopt$param_set$set_values(maxeval = 10L) + optimizer = opt("mbo", acq_optimizer = acqopt, acq_function = acqfun, surrogate = surrogate) + expect_data_table(optimizer$optimize(instance), nrow = 1L) +}) + +test_that("AcqOptimizerLbfgsb works with random restart", { + instance = oi(OBJ_2D, terminator = trm("evals", n_evals = 5L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerLbfgsb$new(acq_function = acqfun) + acqopt$param_set$set_values(maxeval = 200L, restart_strategy = "random", n_restarts = 3L, random_restart_size = 20L) + acqfun$surrogate$update() + acqfun$update() + + expect_data_table(acqopt$optimize(), nrows = 1L) + expect_list(acqopt$result, len = 3L) + expect_names(names(acqopt$result), identical.to = c("iteration_1", "iteration_2", "iteration_3")) + walk(acqopt$result, function(x) expect_class(x, "nloptr")) +}) From df421cdd65c60e047a9a9ff15b2ac1c9027eb98d Mon Sep 17 00:00:00 2001 From: be-marc Date: Tue, 5 Aug 2025 12:46:59 +0200 Subject: [PATCH 100/126] ... --- DESCRIPTION | 2 +- R/AcqOptimizerDirect.R | 25 ++++++++++++------------ tests/testthat/test_AcqOptimizerDirect.R | 15 ++++++++------ 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 86fb6940..ab0cafbf 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -39,6 +39,7 @@ License: LGPL-3 URL: https://mlr3mbo.mlr-org.com, https://github.com/mlr-org/mlr3mbo BugReports: https://github.com/mlr-org/mlr3mbo/issues Depends: + mlr3 (>= 1.1.0), mlr3tuning (>= 1.4.0), R (>= 3.1.0) Imports: @@ -67,7 +68,6 @@ Suggests: stringi, testthat (>= 3.0.0), Remotes: - mlr-org/mlr3@predict_newdata_fast, mlr-org/mlr3learners@ranger_se ByteCompile: no Encoding: UTF-8 diff --git a/R/AcqOptimizerDirect.R b/R/AcqOptimizerDirect.R index 04a2a597..c123752d 100644 --- a/R/AcqOptimizerDirect.R +++ b/R/AcqOptimizerDirect.R @@ -27,14 +27,13 @@ AcqOptimizerDirect = R6Class("AcqOptimizerDirect", stopval = p_dbl(default = -Inf, lower = -Inf, upper = Inf), xtol_rel = p_dbl(default = 1e-06, lower = 0, upper = Inf, special_vals = list(-1)), xtol_abs = p_dbl(default = 0, lower = 0, upper = Inf, special_vals = list(-1)), - maxeval = p_int(lower = 1, default = 1000L, special_vals = list(-1)), + #maxeval = p_int(lower = 1, default = 1000L, special_vals = list(-1)), ftol_rel = p_dbl(default = 0, lower = 0, upper = Inf, special_vals = list(-1)), ftol_abs = p_dbl(default = 0, lower = 0, upper = Inf, special_vals = list(-1)), minf_max = p_dbl(default = -Inf), restart_strategy = p_fct(levels = c("none", "random"), init = "none"), n_iterations = p_int(lower = 1, init = 1L), - random_restart_size = p_int(lower = 1, init = 100L) - + n_evals = p_int(lower = 1, init = 1L) # n_candidates = p_int(lower = 1, default = 1L), # logging_level = p_fct(levels = c("fatal", "error", "warn", "info", "debug", "trace"), default = "warn"), # warmstart = p_lgl(default = FALSE), @@ -55,7 +54,6 @@ AcqOptimizerDirect = R6Class("AcqOptimizerDirect", pv = self$param_set$values n_iterations = if (pv$restart_strategy == "random") pv$n_iterations else 1L - wrapper = function(x, fun, constants, direction) { xdt = as.data.table(as.list(set_names(x, self$acq_function$domain$ids()))) res = mlr3misc::invoke(fun, xdt = xdt, .args = constants)[[1]] @@ -67,16 +65,17 @@ AcqOptimizerDirect = R6Class("AcqOptimizerDirect", direction = self$acq_function$codomain$direction y = Inf - for (n in seq_len(n_iterations)) { + n = 0L + i = 0L + maxeval = ceiling(pv$n_evals / n_iterations) + while (n < pv$n_evals) { + i = i + 1L - x0 = if (pv$restart_strategy == "none") { + x0 = if (i == 1L) { as.numeric(self$acq_function$archive$best()[, self$acq_function$domain$ids(), with = FALSE]) } else { # random restart - design = generate_design_random(self$acq_function$domain, n = pv$random_restart_size)$data - res = mlr3misc::invoke(fun, xdt = design, .args = constants)[[1]] * direction - i = which.min(res) - as.numeric(design[i, self$acq_function$domain$ids(), with = FALSE]) + as.numeric(generate_design_random(self$acq_function$domain, n = 1)$data) } # optimize with nloptr @@ -84,7 +83,7 @@ AcqOptimizerDirect = R6Class("AcqOptimizerDirect", eval_f = wrapper, lb = self$acq_function$domain$lower, ub = self$acq_function$domain$upper, - opts = c(pv, list(algorithm = "NLOPT_GN_DIRECT_L")), + opts = c(pv, list(algorithm = "NLOPT_GN_DIRECT_L", maxeval = min(maxeval, pv$n_evals - n))), eval_grad_f = NULL, x0 = x0, fun = fun, @@ -96,7 +95,9 @@ AcqOptimizerDirect = R6Class("AcqOptimizerDirect", x = res$solution } - self$state = c(self$state, set_names(list(res), paste0("iteration_", n))) + n = n + res$iterations + + self$state = c(self$state, set_names(list(res), paste0("iteration_", i))) } as.data.table(as.list(set_names(c(x, y * direction), c(self$acq_function$domain$ids(), self$acq_function$codomain$ids())))) }, diff --git a/tests/testthat/test_AcqOptimizerDirect.R b/tests/testthat/test_AcqOptimizerDirect.R index 06f4be71..df653074 100644 --- a/tests/testthat/test_AcqOptimizerDirect.R +++ b/tests/testthat/test_AcqOptimizerDirect.R @@ -6,7 +6,7 @@ test_that("AcqOptimizerDirect works", { surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) acqfun = acqf("ei", surrogate = surrogate) acqopt = AcqOptimizerDirect$new(acq_function = acqfun) - acqopt$param_set$set_values(maxeval = 200L, restart_strategy = "none") + acqopt$param_set$set_values(n_evals = 200L, restart_strategy = "none") acqfun$surrogate$update() acqfun$update() @@ -14,6 +14,7 @@ test_that("AcqOptimizerDirect works", { expect_list(acqopt$state) expect_names(names(acqopt$state), must.include = "iteration_1") expect_class(acqopt$state$iteration_1, "nloptr") + expect_true(acqopt$state$iteration_1$iterations <= 200L) }) test_that("AcqOptimizerDirect works with 2D", { @@ -24,7 +25,7 @@ test_that("AcqOptimizerDirect works with 2D", { surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) acqfun = acqf("ei", surrogate = surrogate) acqopt = AcqOptimizerDirect$new(acq_function = acqfun) - acqopt$param_set$set_values(maxeval = 200L) + acqopt$param_set$set_values(n_evals = 200L) acqfun$surrogate$update() acqfun$update() @@ -32,6 +33,7 @@ test_that("AcqOptimizerDirect works with 2D", { expect_list(acqopt$state) expect_names(names(acqopt$state), must.include = "iteration_1") expect_class(acqopt$state$iteration_1, "nloptr") + expect_true(acqopt$state$iteration_1$iterations <= 200L) }) test_that("AcqOptimizerDirect works with instance", { @@ -42,7 +44,7 @@ test_that("AcqOptimizerDirect works with instance", { surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) acqfun = acqf("ei", surrogate = surrogate) acqopt = AcqOptimizerDirect$new(acq_function = acqfun) - acqopt$param_set$set_values(maxeval = 10L) + acqopt$param_set$set_values(n_evals = 10L) optimizer = opt("mbo", acq_optimizer = acqopt, acq_function = acqfun, surrogate = surrogate) expect_data_table(optimizer$optimize(instance), nrow = 1L) @@ -56,12 +58,13 @@ test_that("AcqOptimizerDirect works with random restart", { surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) acqfun = acqf("ei", surrogate = surrogate) acqopt = AcqOptimizerDirect$new(acq_function = acqfun) - acqopt$param_set$set_values(maxeval = 200L, restart_strategy = "random", n_restarts = 3L, random_restart_size = 20L) + acqopt$param_set$set_values(n_evals = 200L, restart_strategy = "random", n_iterations = 4L) acqfun$surrogate$update() acqfun$update() expect_data_table(acqopt$optimize(), nrows = 1L) - expect_list(acqopt$state, len = 3L) - expect_names(names(acqopt$state), identical.to = c("iteration_1", "iteration_2", "iteration_3")) + expect_list(acqopt$state, len = 4L) + expect_names(names(acqopt$state), identical.to = c("iteration_1", "iteration_2", "iteration_3", "iteration_4")) walk(acqopt$state, function(x) expect_class(x, "nloptr")) + expect_true(all(sapply(acqopt$state, function(x) x$iterations) <= 50L)) }) From 0d7ad5890d32b604084381ad99cc58fba4a065df Mon Sep 17 00:00:00 2001 From: be-marc Date: Tue, 5 Aug 2025 13:29:03 +0200 Subject: [PATCH 101/126] ... --- R/AcqOptimizerLocalSearch.R | 93 +++++++++++++++++++ tests/testthat/test_AcqOptimizerLocalSearch.R | 46 +++++++++ 2 files changed, 139 insertions(+) create mode 100644 R/AcqOptimizerLocalSearch.R create mode 100644 tests/testthat/test_AcqOptimizerLocalSearch.R diff --git a/R/AcqOptimizerLocalSearch.R b/R/AcqOptimizerLocalSearch.R new file mode 100644 index 00000000..95b7b29b --- /dev/null +++ b/R/AcqOptimizerLocalSearch.R @@ -0,0 +1,93 @@ +#' @title Local Search Acquisition Function Optimizer +#' +#' @export +AcqOptimizerLocalSearch = R6Class("AcqOptimizerLocalSearch", + inherit = AcqOptimizer, + public = list( + + #' @field state (`list()`)\cr + #' List of [cmaes::cma_es()] results. + state = NULL, + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + #' + #' @param acq_function (`NULL` | [AcqFunction]). + initialize = function(acq_function = NULL) { + self$acq_function = assert_r6(acq_function, "AcqFunction", null.ok = TRUE) + param_set = ps( + n_searches = p_int(lower = 1L, default = 10L), + n_steps = p_int(lower = 0L, default = 5L), + n_neighs = p_int(lower = 1L, default = 10L), + mut_sd = p_dbl(lower = 0, default = 0.1), + stagnate_max = p_int(lower = 1L, default = 10L) + ) + private$.param_set = param_set + }, + + #' @description + #' Optimize the acquisition function. + #' + #' @return [data.table::data.table()] with 1 row per candidate. + optimize = function() { + pv = self$param_set$get_values() + + local_search_control = invoke(local_search_control, minimize = self$acq_function$direction == "minimize", .args = pv) + + wrapper = function(xdt) { + mlr3misc::invoke(self$acq_function$fun, xdt = xdt, .args = self$acq_function$constants$values)[[1]] + } + + res = invoke(local_search, + objective = wrapper, + search_space = self$acq_function$domain, + control = local_search_control + ) + + as.data.table(as.list(set_names(c(res$x, res$y), c(self$acq_function$domain$ids(), self$acq_function$codomain$ids())))) + }, + + #' @description + #' Reset the acquisition function optimizer. + #' + #' Currently not used. + reset = function() { + + } + ), + + active = list( + #' @template field_print_id + print_id = function(rhs) { + if (missing(rhs)) { + paste0("(", class(self$optimizer)[1L], " | ", class(self$terminator)[1L], ")") + } else { + stop("$print_id is read-only.") + } + }, + + #' @field param_set ([paradox::ParamSet])\cr + #' Set of hyperparameters. + param_set = function(rhs) { + if (!missing(rhs) && !identical(rhs, private$.param_set)) { + stop("$param_set is read-only.") + } + private$.param_set + } + ), + + private = list( + .param_set = NULL, + + deep_clone = function(name, value) { + switch(name, + optimizer = value$clone(deep = TRUE), + terminator = value$clone(deep = TRUE), + acq_function = if (!is.null(value)) value$clone(deep = TRUE) else NULL, + .param_set = value$clone(deep = TRUE), + value + ) + } + ) +) + diff --git a/tests/testthat/test_AcqOptimizerLocalSearch.R b/tests/testthat/test_AcqOptimizerLocalSearch.R new file mode 100644 index 00000000..32218318 --- /dev/null +++ b/tests/testthat/test_AcqOptimizerLocalSearch.R @@ -0,0 +1,46 @@ +test_that("AcqOptimizerLocalSearch works", { + instance = oi(OBJ_1D, terminator = trm("evals", n_evals = 5L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerLocalSearch$new(acq_function = acqfun) + acqfun$surrogate$update() + acqfun$update() + + res = acqopt$optimize() + expect_data_table(res, nrows = 1L, ncols = 2L) + expect_names(names(res), must.include = c("x", "acq_ei")) +}) + +test_that("AcqOptimizerDirect works with 2D", { + instance = oi(OBJ_2D, terminator = trm("evals", n_evals = 5L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerLocalSearch$new(acq_function = acqfun) + acqfun$surrogate$update() + acqfun$update() + + res = acqopt$optimize() + expect_data_table(res, nrows = 1L, ncols = 3L) + expect_names(names(res), must.include = c("x1", "x2", "acq_ei")) +}) + +test_that("AcqOptimizerDirect works with instance", { + instance = oi(OBJ_1D, terminator = trm("evals", n_evals = 10L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerLocalSearch$new(acq_function = acqfun) + + optimizer = opt("mbo", acq_optimizer = acqopt, acq_function = acqfun, surrogate = surrogate) + expect_data_table(optimizer$optimize(instance), nrow = 1L) +}) + + From 24d538d0bd077c9496ecf3920964700097c32116 Mon Sep 17 00:00:00 2001 From: be-marc Date: Wed, 6 Aug 2025 11:52:02 +0200 Subject: [PATCH 102/126] ... --- R/AcqOptimizerCmaes.R | 36 +++++++++++++------ R/AcqOptimizerDirect.R | 31 ++++++++-------- R/AcqOptimizerLbfgsb.R | 31 ++++++++-------- tests/testthat/test_AcqOptimizerCmaes.R | 12 +++---- tests/testthat/test_AcqOptimizerDirect.R | 8 ++--- tests/testthat/test_AcqOptimizerLbfgsb.R | 24 +++++++------ tests/testthat/test_AcqOptimizerLocalSearch.R | 5 ++- 7 files changed, 84 insertions(+), 63 deletions(-) diff --git a/R/AcqOptimizerCmaes.R b/R/AcqOptimizerCmaes.R index a8fc3ae1..1613aeb8 100644 --- a/R/AcqOptimizerCmaes.R +++ b/R/AcqOptimizerCmaes.R @@ -21,9 +21,10 @@ AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", #' @param acq_function (`NULL` | [AcqFunction]). initialize = function(acq_function = NULL) { self$acq_function = assert_r6(acq_function, "AcqFunction", null.ok = TRUE) - ps = ps( + param_set = ps( + maxeval = p_int(lower = 1, init = 1000L, special_vals = list(-1)), fnscale = p_dbl(default = 1), - maxit = p_int(lower = 1L), + #maxit = p_int(lower = 1L), stopfitness = p_dbl(default = -Inf), keep.best = p_lgl(default = TRUE), sigma = p_uty(default = 0.5), @@ -41,7 +42,7 @@ AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", diag.value = p_lgl(default = FALSE), stop.tolx = p_dbl(), # undocumented stop criterion restart_strategy = p_fct(levels = c("none", "ipop"), init = "none"), - n_iterations = p_int(lower = 1L, init = 1L), + n_restarts = p_int(lower = 0L, init = 0L), population_multiplier = p_int(lower = 1, init = 2L) # start_values = p_fct(default = "random", levels = c("random", "center", "custom")), # start = p_uty(default = NULL, depends = start_values == "custom"), @@ -54,7 +55,7 @@ AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", ) # ps$values = list(n_candidates = 1, logging_level = "warn", warmstart = FALSE, skip_already_evaluated = TRUE, catch_errors = TRUE) # ps$add_dep("warmstart_size", on = "warmstart", cond = CondEqual$new(TRUE)) - private$.param_set = ps + private$.param_set = param_set }, #' @description @@ -63,14 +64,16 @@ AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", #' @return [data.table::data.table()] with 1 row per candidate. optimize = function() { pv = self$param_set$values + min_iterations = if (pv$restart_strategy == "ipop") pv$n_restarts + 1L else 1L + pv$n_restarts = NULL + pv$restart_strategy = NULL pv$vectorized = TRUE - x = par = set_names(as.numeric(self$acq_function$archive$best()[, self$acq_function$domain$ids(), with = FALSE]), self$acq_function$domain$ids()) - n_iterations = if (pv$restart_strategy == "ipop") pv$n_iterations else 1L # set package defaults if not set by user # restarts needs lambda and mu to be set if (is.null(pv$lambda)) pv$lambda = 4 + floor(3 * log(length(par))) if (is.null(pv$mu)) pv$mu = floor(pv$lambda / 2) + if (is.null(pv$maxit)) pv$maxit = 100 * length(par)^2 * min_iterations wrapper = function(xmat, fun, constants, direction) { xdt = as.data.table(t(xmat)) @@ -83,13 +86,25 @@ AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", direction = self$acq_function$codomain$direction y = Inf - for (n in seq_len(n_iterations)) { + n = 0L + i = 0L + maxeval_i = floor(pv$maxeval / min_iterations) + while (n < pv$maxeval) { + i = i + 1L + + par = if (i == 1L) { + set_names(as.numeric(self$acq_function$archive$best()[, self$acq_function$domain$ids(), with = FALSE]), self$acq_function$domain$ids()) + } else { + # random restart + set_names(as.numeric(generate_design_random(self$acq_function$domain, n = 1)$data), self$acq_function$domain$ids()) + } + res = invoke(cmaes::cma_es, par = par, fn = wrapper, lower = self$acq_function$domain$lower, upper = self$acq_function$domain$upper, - control = pv, + control = insert_named(pv, list(maxit = ceiling(min(maxeval_i / pv$lambda, (pv$maxeval - n) / pv$lambda)))), fun = fun, constants = constants, direction = direction) @@ -102,9 +117,10 @@ AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", pv$mu = pv$mu * pv$population_multiplier pv$lambda = pv$lambda * pv$population_multiplier - par = unlist(generate_design_random(self$acq_function$domain, 1)$data[1, ]) - self$state = c(self$state, set_names(list(res), paste0("iteration_", n))) + n = n + res$counts[1] + + self$state = c(self$state, set_names(list(res), paste0("iteration_", i))) } as.data.table(as.list(set_names(c(x, y * direction), c(self$acq_function$domain$ids(), self$acq_function$codomain$ids())))) diff --git a/R/AcqOptimizerDirect.R b/R/AcqOptimizerDirect.R index c123752d..5af87e6d 100644 --- a/R/AcqOptimizerDirect.R +++ b/R/AcqOptimizerDirect.R @@ -1,12 +1,12 @@ #' @title Direct Optimization Acquisition Function Optimizer #' #' @description -#' If `restart_strategy` is `"random"`, the optimizer runs for `n_iterations` iterations. -#' Each iteration starts with a random search of size `random_restart_size`. -#' The best point is used as the start point for the direct optimization. +#' If the restart strategy is `"none"`, the optimizer starts with the best point in the archive. +#' The optimization stops when one of the stopping criteria is met. #' -#' If `restart_strategy` is `"none"`, the only the direct optimization is performed. -#' The start point is the best point in the archive. +#' If `restart_strategy` is `"random"`, the optimizer runs at least for `maxeval / n_restarts` iterations. +#' The first iteration starts with the best point in the archive. +#' The next iterations start from a random point. #' #' @export AcqOptimizerDirect = R6Class("AcqOptimizerDirect", @@ -23,17 +23,16 @@ AcqOptimizerDirect = R6Class("AcqOptimizerDirect", #' @param acq_function (`NULL` | [AcqFunction]). initialize = function(acq_function = NULL) { self$acq_function = assert_r6(acq_function, "AcqFunction", null.ok = TRUE) - ps = ps( + param_set = ps( + maxeval = p_int(lower = 1, init = 1000L, special_vals = list(-1)), stopval = p_dbl(default = -Inf, lower = -Inf, upper = Inf), - xtol_rel = p_dbl(default = 1e-06, lower = 0, upper = Inf, special_vals = list(-1)), + xtol_rel = p_dbl(default = 1e-04, lower = 0, upper = Inf, special_vals = list(-1)), xtol_abs = p_dbl(default = 0, lower = 0, upper = Inf, special_vals = list(-1)), - #maxeval = p_int(lower = 1, default = 1000L, special_vals = list(-1)), ftol_rel = p_dbl(default = 0, lower = 0, upper = Inf, special_vals = list(-1)), ftol_abs = p_dbl(default = 0, lower = 0, upper = Inf, special_vals = list(-1)), minf_max = p_dbl(default = -Inf), restart_strategy = p_fct(levels = c("none", "random"), init = "none"), - n_iterations = p_int(lower = 1, init = 1L), - n_evals = p_int(lower = 1, init = 1L) + n_restarts = p_int(lower = 0L, init = 0L) # n_candidates = p_int(lower = 1, default = 1L), # logging_level = p_fct(levels = c("fatal", "error", "warn", "info", "debug", "trace"), default = "warn"), # warmstart = p_lgl(default = FALSE), @@ -43,7 +42,7 @@ AcqOptimizerDirect = R6Class("AcqOptimizerDirect", ) # ps$values = list(n_candidates = 1, logging_level = "warn", warmstart = FALSE, skip_already_evaluated = TRUE, catch_errors = TRUE) # ps$add_dep("warmstart_size", on = "warmstart", cond = CondEqual$new(TRUE)) - private$.param_set = ps + private$.param_set = param_set }, #' @description @@ -52,7 +51,9 @@ AcqOptimizerDirect = R6Class("AcqOptimizerDirect", #' @return [data.table::data.table()] with 1 row per candidate. optimize = function() { pv = self$param_set$values - n_iterations = if (pv$restart_strategy == "random") pv$n_iterations else 1L + min_iterations = if (pv$restart_strategy == "random") pv$n_restarts + 1L else 1L + pv$n_restarts = NULL + pv$restart_strategy = NULL wrapper = function(x, fun, constants, direction) { xdt = as.data.table(as.list(set_names(x, self$acq_function$domain$ids()))) @@ -67,8 +68,8 @@ AcqOptimizerDirect = R6Class("AcqOptimizerDirect", y = Inf n = 0L i = 0L - maxeval = ceiling(pv$n_evals / n_iterations) - while (n < pv$n_evals) { + maxeval_i = ceiling(pv$maxeval / min_iterations) + while (n < pv$maxeval) { i = i + 1L x0 = if (i == 1L) { @@ -83,7 +84,7 @@ AcqOptimizerDirect = R6Class("AcqOptimizerDirect", eval_f = wrapper, lb = self$acq_function$domain$lower, ub = self$acq_function$domain$upper, - opts = c(pv, list(algorithm = "NLOPT_GN_DIRECT_L", maxeval = min(maxeval, pv$n_evals - n))), + opts = insert_named(pv, list(algorithm = "NLOPT_GN_DIRECT_L", maxeval = min(maxeval_i, pv$maxeval - n))), eval_grad_f = NULL, x0 = x0, fun = fun, diff --git a/R/AcqOptimizerLbfgsb.R b/R/AcqOptimizerLbfgsb.R index 933d4ee0..1552d1b0 100644 --- a/R/AcqOptimizerLbfgsb.R +++ b/R/AcqOptimizerLbfgsb.R @@ -24,17 +24,16 @@ AcqOptimizerLbfgsb = R6Class("AcqOptimizerLbfgsb", #' @param acq_function (`NULL` | [AcqFunction]). initialize = function(acq_function = NULL) { self$acq_function = assert_r6(acq_function, "AcqFunction", null.ok = TRUE) - ps = ps( + param_set = ps( + maxeval = p_int(lower = 1, init = 1000L, special_vals = list(-1)), stopval = p_dbl(default = -Inf, lower = -Inf, upper = Inf), - xtol_rel = p_dbl(default = 1e-06, lower = 0, upper = Inf, special_vals = list(-1)), + xtol_rel = p_dbl(default = 1e-04, lower = 0, upper = Inf, special_vals = list(-1)), xtol_abs = p_dbl(default = 0, lower = 0, upper = Inf, special_vals = list(-1)), - maxeval = p_int(lower = 1, default = 1000L, special_vals = list(-1)), ftol_rel = p_dbl(default = 0, lower = 0, upper = Inf, special_vals = list(-1)), ftol_abs = p_dbl(default = 0, lower = 0, upper = Inf, special_vals = list(-1)), minf_max = p_dbl(default = -Inf), restart_strategy = p_fct(levels = c("none", "random"), init = "none"), - n_iterations = p_int(lower = 1, init = 1L), - random_restart_size = p_int(lower = 1, init = 100L) + n_restarts = p_int(lower = 0L, init = 0L) # n_candidates = p_int(lower = 1, default = 1L), # logging_level = p_fct(levels = c("fatal", "error", "warn", "info", "debug", "trace"), default = "warn"), # warmstart = p_lgl(default = FALSE), @@ -44,7 +43,7 @@ AcqOptimizerLbfgsb = R6Class("AcqOptimizerLbfgsb", ) # ps$values = list(n_candidates = 1, logging_level = "warn", warmstart = FALSE, skip_already_evaluated = TRUE, catch_errors = TRUE) # ps$add_dep("warmstart_size", on = "warmstart", cond = CondEqual$new(TRUE)) - private$.param_set = ps + private$.param_set = param_set }, #' @description @@ -53,8 +52,9 @@ AcqOptimizerLbfgsb = R6Class("AcqOptimizerLbfgsb", #' @return [data.table::data.table()] with 1 row per candidate. optimize = function() { pv = self$param_set$values - n_iterations = if (pv$restart_strategy == "random") pv$n_iterations else 1L - + min_iterations = if (pv$restart_strategy == "random") pv$n_restarts + 1L else 1L + pv$n_restarts = NULL + pv$restart_strategy = NULL wrapper = function(x, fun, constants, direction) { xdt = as.data.table(as.list(set_names(x, self$acq_function$domain$ids()))) @@ -67,16 +67,17 @@ AcqOptimizerLbfgsb = R6Class("AcqOptimizerLbfgsb", direction = self$acq_function$codomain$direction y = Inf - for (n in seq_len(n_iterations)) { + n = 0L + i = 0L + maxeval_i = ceiling(pv$maxeval / min_iterations) + while (n < pv$maxeval) { + i = i + 1L - x0 = if (pv$restart_strategy == "none") { + x0 = if (i == 1L) { as.numeric(self$acq_function$archive$best()[, self$acq_function$domain$ids(), with = FALSE]) } else { # random restart - design = generate_design_random(self$acq_function$domain, n = pv$random_restart_size)$data - res = mlr3misc::invoke(fun, xdt = design, .args = constants)[[1]] * direction - i = which.min(res) - as.numeric(design[i, self$acq_function$domain$ids(), with = FALSE]) + as.numeric(generate_design_random(self$acq_function$domain, n = 1)$data) } eval_grad_f = function(x, fun, constants, direction) { @@ -89,7 +90,7 @@ AcqOptimizerLbfgsb = R6Class("AcqOptimizerLbfgsb", eval_f = wrapper, lb = self$acq_function$domain$lower + saveguard_epsilon, ub = self$acq_function$domain$upper - saveguard_epsilon, - opts = c(pv, list(algorithm = "NLOPT_LD_LBFGS")), + opts = insert_named(pv, list(algorithm = "NLOPT_LD_LBFGS", maxeval = min(maxeval_i, pv$maxeval - n))), eval_grad_f = eval_grad_f, x0 = x0, fun = fun, diff --git a/tests/testthat/test_AcqOptimizerCmaes.R b/tests/testthat/test_AcqOptimizerCmaes.R index bdef6545..971ce24d 100644 --- a/tests/testthat/test_AcqOptimizerCmaes.R +++ b/tests/testthat/test_AcqOptimizerCmaes.R @@ -6,7 +6,7 @@ test_that("AcqOptimizerCmaes works", { surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) acqfun = acqf("ei", surrogate = surrogate) acqopt = AcqOptimizerCmaes$new(acq_function = acqfun) - acqopt$param_set$set_values(maxit = 1000L) + acqopt$param_set$set_values(maxeval = 100L) acqfun$surrogate$update() acqfun$update() expect_data_table(acqopt$optimize(), nrows = 1L) @@ -23,7 +23,7 @@ test_that("AcqOptimizerCmaes works with 2D", { surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) acqfun = acqf("ei", surrogate = surrogate) acqopt = AcqOptimizerCmaes$new(acq_function = acqfun) - acqopt$param_set$set_values(maxit = 10L) + acqopt$param_set$set_values(maxeval = 100L) acqfun$surrogate$update() acqfun$update() expect_data_table(acqopt$optimize(), nrows = 1L) @@ -40,7 +40,7 @@ test_that("AcqOptimizerCmaes works with instance", { surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) acqfun = acqf("ei", surrogate = surrogate) acqopt = AcqOptimizerCmaes$new(acq_function = acqfun) - acqopt$param_set$set_values(maxit = 10L) + acqopt$param_set$set_values(maxeval = 100L) optimizer = opt("mbo", acq_optimizer = acqopt, acq_function = acqfun, surrogate = surrogate) expect_data_table(optimizer$optimize(instance), nrow = 1L) @@ -54,12 +54,12 @@ test_that("AcqOptimizerCmaes works with ipop restart", { surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) acqfun = acqf("ei", surrogate = surrogate) acqopt = AcqOptimizerCmaes$new(acq_function = acqfun) - acqopt$param_set$set_values(maxit = 10L, restart_strategy = "ipop", n_iterations = 3L, population_multiplier = 2L) + acqopt$param_set$set_values(maxeval = 200L, restart_strategy = "ipop", n_restarts = 3L, population_multiplier = 2L) acqfun$surrogate$update() acqfun$update() expect_data_table(acqopt$optimize(), nrows = 1L) - expect_list(acqopt$state, len = 3L) - expect_names(names(acqopt$state), identical.to = c("iteration_1", "iteration_2", "iteration_3")) + expect_list(acqopt$state, len = 4L) + expect_names(names(acqopt$state), identical.to = c("iteration_1", "iteration_2", "iteration_3", "iteration_4")) walk(acqopt$state, function(x) expect_class(x, "cma_es.result")) }) diff --git a/tests/testthat/test_AcqOptimizerDirect.R b/tests/testthat/test_AcqOptimizerDirect.R index df653074..12cba437 100644 --- a/tests/testthat/test_AcqOptimizerDirect.R +++ b/tests/testthat/test_AcqOptimizerDirect.R @@ -6,7 +6,7 @@ test_that("AcqOptimizerDirect works", { surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) acqfun = acqf("ei", surrogate = surrogate) acqopt = AcqOptimizerDirect$new(acq_function = acqfun) - acqopt$param_set$set_values(n_evals = 200L, restart_strategy = "none") + acqopt$param_set$set_values(maxeval = 200L, restart_strategy = "none") acqfun$surrogate$update() acqfun$update() @@ -25,7 +25,7 @@ test_that("AcqOptimizerDirect works with 2D", { surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) acqfun = acqf("ei", surrogate = surrogate) acqopt = AcqOptimizerDirect$new(acq_function = acqfun) - acqopt$param_set$set_values(n_evals = 200L) + acqopt$param_set$set_values(maxeval = 200L) acqfun$surrogate$update() acqfun$update() @@ -44,7 +44,7 @@ test_that("AcqOptimizerDirect works with instance", { surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) acqfun = acqf("ei", surrogate = surrogate) acqopt = AcqOptimizerDirect$new(acq_function = acqfun) - acqopt$param_set$set_values(n_evals = 10L) + acqopt$param_set$set_values(maxeval = 10L) optimizer = opt("mbo", acq_optimizer = acqopt, acq_function = acqfun, surrogate = surrogate) expect_data_table(optimizer$optimize(instance), nrow = 1L) @@ -58,7 +58,7 @@ test_that("AcqOptimizerDirect works with random restart", { surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) acqfun = acqf("ei", surrogate = surrogate) acqopt = AcqOptimizerDirect$new(acq_function = acqfun) - acqopt$param_set$set_values(n_evals = 200L, restart_strategy = "random", n_iterations = 4L) + acqopt$param_set$set_values(maxeval = 200L, restart_strategy = "random", n_restarts = 3L) acqfun$surrogate$update() acqfun$update() diff --git a/tests/testthat/test_AcqOptimizerLbfgsb.R b/tests/testthat/test_AcqOptimizerLbfgsb.R index be453689..3ffc37e9 100644 --- a/tests/testthat/test_AcqOptimizerLbfgsb.R +++ b/tests/testthat/test_AcqOptimizerLbfgsb.R @@ -11,9 +11,10 @@ test_that("AcqOptimizerLbfgsb works", { acqfun$update() expect_data_table(acqopt$optimize(), nrows = 1L) - expect_list(acqopt$result) - expect_names(names(acqopt$result), must.include = "iteration_1") - expect_class(acqopt$result$iteration_1, "nloptr") + expect_list(acqopt$state) + expect_names(names(acqopt$state), must.include = "iteration_1") + expect_class(acqopt$state$iteration_1, "nloptr") + expect_true(acqopt$state$iteration_1$iterations <= 200L) }) test_that("AcqOptimizerLbfgsb works with 2D", { @@ -29,9 +30,10 @@ test_that("AcqOptimizerLbfgsb works with 2D", { acqfun$update() expect_data_table(acqopt$optimize(), nrows = 1L) - expect_list(acqopt$result) - expect_names(names(acqopt$result), must.include = "iteration_1") - expect_class(acqopt$result$iteration_1, "nloptr") + expect_list(acqopt$state) + expect_names(names(acqopt$state), must.include = "iteration_1") + expect_class(acqopt$state$iteration_1, "nloptr") + expect_true(acqopt$state$iteration_1$iterations <= 200L) }) test_that("AcqOptimizerLbfgsb works with instance", { @@ -43,6 +45,7 @@ test_that("AcqOptimizerLbfgsb works with instance", { acqfun = acqf("ei", surrogate = surrogate) acqopt = AcqOptimizerLbfgsb$new(acq_function = acqfun) acqopt$param_set$set_values(maxeval = 10L) + optimizer = opt("mbo", acq_optimizer = acqopt, acq_function = acqfun, surrogate = surrogate) expect_data_table(optimizer$optimize(instance), nrow = 1L) }) @@ -55,12 +58,13 @@ test_that("AcqOptimizerLbfgsb works with random restart", { surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) acqfun = acqf("ei", surrogate = surrogate) acqopt = AcqOptimizerLbfgsb$new(acq_function = acqfun) - acqopt$param_set$set_values(maxeval = 200L, restart_strategy = "random", n_restarts = 3L, random_restart_size = 20L) + acqopt$param_set$set_values(maxeval = 200L, restart_strategy = "random", n_restarts = 3L) acqfun$surrogate$update() acqfun$update() expect_data_table(acqopt$optimize(), nrows = 1L) - expect_list(acqopt$result, len = 3L) - expect_names(names(acqopt$result), identical.to = c("iteration_1", "iteration_2", "iteration_3")) - walk(acqopt$result, function(x) expect_class(x, "nloptr")) + expect_list(acqopt$state, len = 4L) + expect_names(names(acqopt$state), identical.to = c("iteration_1", "iteration_2", "iteration_3", "iteration_4")) + walk(acqopt$state, function(x) expect_class(x, "nloptr")) + expect_true(all(sapply(acqopt$state, function(x) x$iterations) <= 50L)) }) diff --git a/tests/testthat/test_AcqOptimizerLocalSearch.R b/tests/testthat/test_AcqOptimizerLocalSearch.R index 32218318..bc3e4f56 100644 --- a/tests/testthat/test_AcqOptimizerLocalSearch.R +++ b/tests/testthat/test_AcqOptimizerLocalSearch.R @@ -14,7 +14,7 @@ test_that("AcqOptimizerLocalSearch works", { expect_names(names(res), must.include = c("x", "acq_ei")) }) -test_that("AcqOptimizerDirect works with 2D", { +test_that("AcqOptimizerLocalSearch works with 2D", { instance = oi(OBJ_2D, terminator = trm("evals", n_evals = 5L)) design = generate_design_grid(instance$search_space, resolution = 4L)$data instance$eval_batch(design) @@ -30,7 +30,7 @@ test_that("AcqOptimizerDirect works with 2D", { expect_names(names(res), must.include = c("x1", "x2", "acq_ei")) }) -test_that("AcqOptimizerDirect works with instance", { +test_that("AcqOptimizerLocalSearch works with instance", { instance = oi(OBJ_1D, terminator = trm("evals", n_evals = 10L)) design = generate_design_grid(instance$search_space, resolution = 4L)$data instance$eval_batch(design) @@ -43,4 +43,3 @@ test_that("AcqOptimizerDirect works with instance", { expect_data_table(optimizer$optimize(instance), nrow = 1L) }) - From b4ceab2314682da6acc239cb4114ee3ca965c8a6 Mon Sep 17 00:00:00 2001 From: be-marc Date: Wed, 6 Aug 2025 11:52:24 +0200 Subject: [PATCH 103/126] ... --- DESCRIPTION | 1 + 1 file changed, 1 insertion(+) diff --git a/DESCRIPTION b/DESCRIPTION index ab0cafbf..dc6edf6a 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -69,6 +69,7 @@ Suggests: testthat (>= 3.0.0), Remotes: mlr-org/mlr3learners@ranger_se + mlr-org/bbotk@ls-in-c ByteCompile: no Encoding: UTF-8 Config/testthat/edition: 3 From b3e38dfa77b07b0e4a0727d23ff16e9b8bf33805 Mon Sep 17 00:00:00 2001 From: be-marc Date: Wed, 6 Aug 2025 11:53:07 +0200 Subject: [PATCH 104/126] ... --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index dc6edf6a..c8d01e56 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -68,7 +68,7 @@ Suggests: stringi, testthat (>= 3.0.0), Remotes: - mlr-org/mlr3learners@ranger_se + mlr-org/mlr3learners@ranger_se, mlr-org/bbotk@ls-in-c ByteCompile: no Encoding: UTF-8 From c0d689be161adee93d444680c039d74729181c02 Mon Sep 17 00:00:00 2001 From: be-marc Date: Wed, 6 Aug 2025 12:21:13 +0200 Subject: [PATCH 105/126] ... --- DESCRIPTION | 1 + NAMESPACE | 1 + man/AcqOptimizerDirect.Rd | 10 +-- man/AcqOptimizerLocalSearch.Rd | 109 ++++++++++++++++++++++++++++++++ man/mlr_learners_regr.ranger.Rd | 1 + 5 files changed, 117 insertions(+), 5 deletions(-) create mode 100644 man/AcqOptimizerLocalSearch.Rd diff --git a/DESCRIPTION b/DESCRIPTION index c8d01e56..c90e2e4a 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -98,6 +98,7 @@ Collate: 'AcqOptimizerCmaes.R' 'AcqOptimizerDirect.R' 'AcqOptimizerLbfgsb.R' + 'AcqOptimizerLocalSearch.R' 'mlr_input_trafos.R' 'InputTrafo.R' 'InputTrafoUnitcube.R' diff --git a/NAMESPACE b/NAMESPACE index 0af87a61..57dfb79c 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -26,6 +26,7 @@ export(AcqOptimizer) export(AcqOptimizerCmaes) export(AcqOptimizerDirect) export(AcqOptimizerLbfgsb) +export(AcqOptimizerLocalSearch) export(InputTrafo) export(InputTrafoUnitcube) export(LearnerRegrRangerMbo) diff --git a/man/AcqOptimizerDirect.Rd b/man/AcqOptimizerDirect.Rd index 71719b78..f2cf9b58 100644 --- a/man/AcqOptimizerDirect.Rd +++ b/man/AcqOptimizerDirect.Rd @@ -4,12 +4,12 @@ \alias{AcqOptimizerDirect} \title{Direct Optimization Acquisition Function Optimizer} \description{ -If \code{restart_strategy} is \code{"random"}, the optimizer runs for \code{n_iterations} iterations. -Each iteration starts with a random search of size \code{random_restart_size}. -The best point is used as the start point for the direct optimization. +If the restart strategy is \code{"none"}, the optimizer starts with the best point in the archive. +The optimization stops when one of the stopping criteria is met. -If \code{restart_strategy} is \code{"none"}, the only the direct optimization is performed. -The start point is the best point in the archive. +If \code{restart_strategy} is \code{"random"}, the optimizer runs at least for \code{maxeval / n_restarts} iterations. +The first iteration starts with the best point in the archive. +The next iterations start from a random point. } \section{Super class}{ \code{\link[mlr3mbo:AcqOptimizer]{mlr3mbo::AcqOptimizer}} -> \code{AcqOptimizerDirect} diff --git a/man/AcqOptimizerLocalSearch.Rd b/man/AcqOptimizerLocalSearch.Rd new file mode 100644 index 00000000..37a96b30 --- /dev/null +++ b/man/AcqOptimizerLocalSearch.Rd @@ -0,0 +1,109 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/AcqOptimizerLocalSearch.R +\name{AcqOptimizerLocalSearch} +\alias{AcqOptimizerLocalSearch} +\title{Local Search Acquisition Function Optimizer} +\description{ +Local Search Acquisition Function Optimizer + +Local Search Acquisition Function Optimizer +} +\section{Super class}{ +\code{\link[mlr3mbo:AcqOptimizer]{mlr3mbo::AcqOptimizer}} -> \code{AcqOptimizerLocalSearch} +} +\section{Public fields}{ +\if{html}{\out{
    }} +\describe{ +\item{\code{state}}{(\code{list()})\cr +List of \code{\link[cmaes:cma_es]{cmaes::cma_es()}} results.} +} +\if{html}{\out{
    }} +} +\section{Active bindings}{ +\if{html}{\out{
    }} +\describe{ +\item{\code{print_id}}{(\code{character})\cr +Id used when printing.} + +\item{\code{param_set}}{(\link[paradox:ParamSet]{paradox::ParamSet})\cr +Set of hyperparameters.} +} +\if{html}{\out{
    }} +} +\section{Methods}{ +\subsection{Public methods}{ +\itemize{ +\item \href{#method-AcqOptimizerLocalSearch-new}{\code{AcqOptimizerLocalSearch$new()}} +\item \href{#method-AcqOptimizerLocalSearch-optimize}{\code{AcqOptimizerLocalSearch$optimize()}} +\item \href{#method-AcqOptimizerLocalSearch-reset}{\code{AcqOptimizerLocalSearch$reset()}} +\item \href{#method-AcqOptimizerLocalSearch-clone}{\code{AcqOptimizerLocalSearch$clone()}} +} +} +\if{html}{\out{ +
    Inherited methods + +
    +}} +\if{html}{\out{
    }} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerLocalSearch-new}{}}} +\subsection{Method \code{new()}}{ +Creates a new instance of this \link[R6:R6Class]{R6} class. +\subsection{Usage}{ +\if{html}{\out{
    }}\preformatted{AcqOptimizerLocalSearch$new(acq_function = NULL)}\if{html}{\out{
    }} +} + +\subsection{Arguments}{ +\if{html}{\out{
    }} +\describe{ +\item{\code{acq_function}}{(\code{NULL} | \link{AcqFunction}).} +} +\if{html}{\out{
    }} +} +} +\if{html}{\out{
    }} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerLocalSearch-optimize}{}}} +\subsection{Method \code{optimize()}}{ +Optimize the acquisition function. +\subsection{Usage}{ +\if{html}{\out{
    }}\preformatted{AcqOptimizerLocalSearch$optimize()}\if{html}{\out{
    }} +} + +\subsection{Returns}{ +\code{\link[data.table:data.table]{data.table::data.table()}} with 1 row per candidate. +} +} +\if{html}{\out{
    }} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerLocalSearch-reset}{}}} +\subsection{Method \code{reset()}}{ +Reset the acquisition function optimizer. + +Currently not used. +\subsection{Usage}{ +\if{html}{\out{
    }}\preformatted{AcqOptimizerLocalSearch$reset()}\if{html}{\out{
    }} +} + +} +\if{html}{\out{
    }} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerLocalSearch-clone}{}}} +\subsection{Method \code{clone()}}{ +The objects of this class are cloneable with this method. +\subsection{Usage}{ +\if{html}{\out{
    }}\preformatted{AcqOptimizerLocalSearch$clone(deep = FALSE)}\if{html}{\out{
    }} +} + +\subsection{Arguments}{ +\if{html}{\out{
    }} +\describe{ +\item{\code{deep}}{Whether to make a deep clone.} +} +\if{html}{\out{
    }} +} +} +} diff --git a/man/mlr_learners_regr.ranger.Rd b/man/mlr_learners_regr.ranger.Rd index f0c824b1..f96d5a80 100644 --- a/man/mlr_learners_regr.ranger.Rd +++ b/man/mlr_learners_regr.ranger.Rd @@ -34,6 +34,7 @@ Calls \code{\link[ranger:ranger]{ranger::ranger()}} from package \CRANpkg{ranger
  • mlr3::Learner$reset()
  • mlr3::Learner$selected_features()
  • mlr3::Learner$train()
    • +
  • mlr3::LearnerRegr$predict_newdata_fast()
    • }} From 46eb09f202f6df90922b32b5593db6d6cf935e3b Mon Sep 17 00:00:00 2001 From: be-marc Date: Wed, 6 Aug 2025 16:59:23 +0200 Subject: [PATCH 106/126] ... --- R/AcqOptimizerCmaes.R | 133 ++++++++++++----------------- attic/AcqOptimizerCmaes.R | 173 ++++++++++++++++++++++++++++++++++++++ attic/test_cmaes.R | 168 ++++++++++++++++++++++++++++++++++++ 3 files changed, 393 insertions(+), 81 deletions(-) create mode 100644 attic/AcqOptimizerCmaes.R create mode 100644 attic/test_cmaes.R diff --git a/R/AcqOptimizerCmaes.R b/R/AcqOptimizerCmaes.R index 1613aeb8..3f311f9b 100644 --- a/R/AcqOptimizerCmaes.R +++ b/R/AcqOptimizerCmaes.R @@ -1,11 +1,5 @@ #' @title CMA-ES Acquisition Function Optimizer #' -#' @description -#' IPOP CMA-ES runs for `n_iterations` iterations. -#' The population size is increased by `population_multiplier` after each iteration. -#' In the first iteration, the start point is the best point in the archive. -#' In the subsequent iterations, the start point is a random point in the search space. -#' #' @export AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", inherit = AcqOptimizer, @@ -22,28 +16,30 @@ AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", initialize = function(acq_function = NULL) { self$acq_function = assert_r6(acq_function, "AcqFunction", null.ok = TRUE) param_set = ps( - maxeval = p_int(lower = 1, init = 1000L, special_vals = list(-1)), - fnscale = p_dbl(default = 1), - #maxit = p_int(lower = 1L), - stopfitness = p_dbl(default = -Inf), - keep.best = p_lgl(default = TRUE), - sigma = p_uty(default = 0.5), - mu = p_int(lower = 1L), - lambda = p_int(lower = 1L), - weights = p_uty(), - damps = p_dbl(), - cs = p_dbl(), - ccum = p_dbl(), - ccov.1 = p_dbl(lower = 0), - ccov.mu = p_dbl(lower = 0), - diag.sigma = p_lgl(default = FALSE), - diag.eigen = p_lgl(default = FALSE), - diag.pop = p_lgl(default = FALSE), - diag.value = p_lgl(default = FALSE), - stop.tolx = p_dbl(), # undocumented stop criterion - restart_strategy = p_fct(levels = c("none", "ipop"), init = "none"), - n_restarts = p_int(lower = 0L, init = 0L), - population_multiplier = p_int(lower = 1, init = 2L) + maxEvals = p_int(lower = 1, init = 1000L), + xtol = p_dbl(init = 1e-3) + # maxeval = p_int(lower = 1, init = 1000L, special_vals = list(-1)), + # fnscale = p_dbl(default = 1), + # #maxit = p_int(lower = 1L), + # stopfitness = p_dbl(default = -Inf), + # keep.best = p_lgl(default = TRUE), + # sigma = p_uty(default = 0.5), + # mu = p_int(lower = 1L), + # lambda = p_int(lower = 1L), + # weights = p_uty(), + # damps = p_dbl(), + # cs = p_dbl(), + # ccum = p_dbl(), + # ccov.1 = p_dbl(lower = 0), + # ccov.mu = p_dbl(lower = 0), + # diag.sigma = p_lgl(default = FALSE), + # diag.eigen = p_lgl(default = FALSE), + # diag.pop = p_lgl(default = FALSE), + # diag.value = p_lgl(default = FALSE), + # stop.tolx = p_dbl(), # undocumented stop criterion + # restart_strategy = p_fct(levels = c("none", "ipop"), init = "none"), + # n_restarts = p_int(lower = 0L, init = 0L), + # population_multiplier = p_int(lower = 1, init = 2L) # start_values = p_fct(default = "random", levels = c("random", "center", "custom")), # start = p_uty(default = NULL, depends = start_values == "custom"), # n_candidates = p_int(lower = 1, default = 1L), @@ -64,66 +60,41 @@ AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", #' @return [data.table::data.table()] with 1 row per candidate. optimize = function() { pv = self$param_set$values - min_iterations = if (pv$restart_strategy == "ipop") pv$n_restarts + 1L else 1L - pv$n_restarts = NULL - pv$restart_strategy = NULL - pv$vectorized = TRUE - - # set package defaults if not set by user - # restarts needs lambda and mu to be set - if (is.null(pv$lambda)) pv$lambda = 4 + floor(3 * log(length(par))) - if (is.null(pv$mu)) pv$mu = floor(pv$lambda / 2) - if (is.null(pv$maxit)) pv$maxit = 100 * length(par)^2 * min_iterations - - wrapper = function(xmat, fun, constants, direction) { - xdt = as.data.table(t(xmat)) - res = mlr3misc::invoke(fun, xdt = xdt, .args = constants)[[1]] - res * direction - } fun = get_private(self$acq_function)$.fun constants = self$acq_function$constants$values direction = self$acq_function$codomain$direction - y = Inf - n = 0L - i = 0L - maxeval_i = floor(pv$maxeval / min_iterations) - while (n < pv$maxeval) { - i = i + 1L - - par = if (i == 1L) { - set_names(as.numeric(self$acq_function$archive$best()[, self$acq_function$domain$ids(), with = FALSE]), self$acq_function$domain$ids()) - } else { - # random restart - set_names(as.numeric(generate_design_random(self$acq_function$domain, n = 1)$data), self$acq_function$domain$ids()) - } - - res = invoke(cmaes::cma_es, - par = par, - fn = wrapper, - lower = self$acq_function$domain$lower, - upper = self$acq_function$domain$upper, - control = insert_named(pv, list(maxit = ceiling(min(maxeval_i / pv$lambda, (pv$maxeval - n) / pv$lambda)))), - fun = fun, - constants = constants, - direction = direction) - - # set best - if (res$value < y) { - y = res$value - x = set_names(res$par, self$acq_function$domain$ids()) - } - - pv$mu = pv$mu * pv$population_multiplier - pv$lambda = pv$lambda * pv$population_multiplier - - n = n + res$counts[1] - - self$state = c(self$state, set_names(list(res), paste0("iteration_", i))) + + wrapper = function(xmat) { + xdt = set_names(as.data.table(t(xmat)), self$acq_function$domain$ids()) + res = mlr3misc::invoke(fun, xdt = xdt, .args = constants)[[1]] + res * direction + } + + optimFunBlock = function(x) { + wrapper(x) } - as.data.table(as.list(set_names(c(x, y * direction), c(self$acq_function$domain$ids(), self$acq_function$codomain$ids())))) + lower = acq_function$domain$lower + upper = acq_function$domain$upper + + res = Rlibcmaes::cmaesOptim( + x0 = x0, + sigma = median(upper - lower) / 4, + optimFun = wrapper, + optimFunBlock = optimFunBlock, + lower = lower, + upper = upper, + cmaAlgo = as.integer(cmaEsAlgo()$IPOP_CMAES), + lambda = -1, + maxEvals = pv$maxEvals, + xtol = pv$xtol, + ) + + y = wrapper(res) + + as.data.table(as.list(set_names(c(res, y * direction), c(self$acq_function$domain$ids(), self$acq_function$codomain$ids())))) }, #' @description diff --git a/attic/AcqOptimizerCmaes.R b/attic/AcqOptimizerCmaes.R new file mode 100644 index 00000000..8c2b193e --- /dev/null +++ b/attic/AcqOptimizerCmaes.R @@ -0,0 +1,173 @@ +#' @title CMA-ES Acquisition Function Optimizer +#' +#' @description +#' IPOP CMA-ES runs for `n_iterations` iterations. +#' The population size is increased by `population_multiplier` after each iteration. +#' In the first iteration, the start point is the best point in the archive. +#' In the subsequent iterations, the start point is a random point in the search space. +#' +#' @export +AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", + inherit = AcqOptimizer, + public = list( + + #' @field state (`list()`)\cr + #' List of [cmaes::cma_es()] results. + state = NULL, + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + #' + #' @param acq_function (`NULL` | [AcqFunction]). + initialize = function(acq_function = NULL) { + self$acq_function = assert_r6(acq_function, "AcqFunction", null.ok = TRUE) + param_set = ps( + maxeval = p_int(lower = 1, init = 1000L, special_vals = list(-1)), + fnscale = p_dbl(default = 1), + #maxit = p_int(lower = 1L), + stopfitness = p_dbl(default = -Inf), + keep.best = p_lgl(default = TRUE), + sigma = p_uty(default = 0.5), + mu = p_int(lower = 1L), + lambda = p_int(lower = 1L), + weights = p_uty(), + damps = p_dbl(), + cs = p_dbl(), + ccum = p_dbl(), + ccov.1 = p_dbl(lower = 0), + ccov.mu = p_dbl(lower = 0), + diag.sigma = p_lgl(default = FALSE), + diag.eigen = p_lgl(default = FALSE), + diag.pop = p_lgl(default = FALSE), + diag.value = p_lgl(default = FALSE), + stop.tolx = p_dbl(), # undocumented stop criterion + restart_strategy = p_fct(levels = c("none", "ipop"), init = "none"), + n_restarts = p_int(lower = 0L, init = 0L), + population_multiplier = p_int(lower = 1, init = 2L) + # start_values = p_fct(default = "random", levels = c("random", "center", "custom")), + # start = p_uty(default = NULL, depends = start_values == "custom"), + # n_candidates = p_int(lower = 1, default = 1L), + # logging_level = p_fct(levels = c("fatal", "error", "warn", "info", "debug", "trace"), default = "warn"), + # warmstart = p_lgl(default = FALSE), + # warmstart_size = p_int(lower = 1L, special_vals = list("all")), + # skip_already_evaluated = p_lgl(default = TRUE), + # catch_errors = p_lgl(default = TRUE) + ) + # ps$values = list(n_candidates = 1, logging_level = "warn", warmstart = FALSE, skip_already_evaluated = TRUE, catch_errors = TRUE) + # ps$add_dep("warmstart_size", on = "warmstart", cond = CondEqual$new(TRUE)) + private$.param_set = param_set + }, + + #' @description + #' Optimize the acquisition function. + #' + #' @return [data.table::data.table()] with 1 row per candidate. + optimize = function() { + pv = self$param_set$values + min_iterations = if (pv$restart_strategy == "ipop") pv$n_restarts + 1L else 1L + pv$n_restarts = NULL + pv$restart_strategy = NULL + pv$vectorized = TRUE + + # set package defaults if not set by user + # restarts needs lambda and mu to be set + if (is.null(pv$lambda)) pv$lambda = 4 + floor(3 * log(length(par))) + if (is.null(pv$mu)) pv$mu = floor(pv$lambda / 2) + + wrapper = function(xmat, fun, constants, direction) { + xdt = as.data.table(t(xmat)) + res = mlr3misc::invoke(fun, xdt = xdt, .args = constants)[[1]] + res * direction + } + + fun = get_private(self$acq_function)$.fun + constants = self$acq_function$constants$values + direction = self$acq_function$codomain$direction + + y = Inf + n = 0L + i = 0L + maxeval_i = floor(pv$maxeval / min_iterations) + while (n < pv$maxeval) { + i = i + 1L + + par = if (i == 1L) { + set_names(as.numeric(self$acq_function$archive$best()[, self$acq_function$domain$ids(), with = FALSE]), self$acq_function$domain$ids()) + } else { + # random restart + set_names(as.numeric(generate_design_random(self$acq_function$domain, n = 1)$data), self$acq_function$domain$ids()) + } + + res = invoke(cmaes::cma_es, + par = par, + fn = wrapper, + lower = self$acq_function$domain$lower, + upper = self$acq_function$domain$upper, + control = pv, # ceiling(min(maxeval_i / pv$lambda, (pv$maxeval - n) / pv$lambda))) + fun = fun, + constants = constants, + direction = direction) + + browser() + + # set best + if (res$value < y) { + y = res$value + x = set_names(res$par, self$acq_function$domain$ids()) + } + + pv$mu = pv$mu * pv$population_multiplier + pv$lambda = pv$lambda * pv$population_multiplier + + n = n + res$counts[1] + + self$state = c(self$state, set_names(list(res), paste0("iteration_", i))) + } + + as.data.table(as.list(set_names(c(x, y * direction), c(self$acq_function$domain$ids(), self$acq_function$codomain$ids())))) + }, + + #' @description + #' Reset the acquisition function optimizer. + #' + #' Currently not used. + reset = function() { + + } + ), + + active = list( + #' @template field_print_id + print_id = function(rhs) { + if (missing(rhs)) { + paste0("(", class(self$optimizer)[1L], " | ", class(self$terminator)[1L], ")") + } else { + stop("$print_id is read-only.") + } + }, + + #' @field param_set ([paradox::ParamSet])\cr + #' Set of hyperparameters. + param_set = function(rhs) { + if (!missing(rhs) && !identical(rhs, private$.param_set)) { + stop("$param_set is read-only.") + } + private$.param_set + } + ), + + private = list( + .param_set = NULL, + + deep_clone = function(name, value) { + switch(name, + optimizer = value$clone(deep = TRUE), + terminator = value$clone(deep = TRUE), + acq_function = if (!is.null(value)) value$clone(deep = TRUE) else NULL, + .param_set = value$clone(deep = TRUE), + value + ) + } + ) +) + diff --git a/attic/test_cmaes.R b/attic/test_cmaes.R new file mode 100644 index 00000000..811ba337 --- /dev/null +++ b/attic/test_cmaes.R @@ -0,0 +1,168 @@ +library(batchtools) +library(data.table) +library(mlr3) +library(mlr3misc) +library(mlr3mbo) +library(mlr3pipelines) +library(bbotk) +library(paradox) +library(R6) +library(checkmate) +library(reticulate) +library(mlr3learners) +library(yahpogym) + +data.table::setDTthreads(1L) + + +use_condaenv("yahpo_gym", required = TRUE) +yahpo_gym = import("yahpo_gym") + +setup = mlr3misc::rowwise_table( + ~benchmark, ~scenario, ~instance, ~target_variable, ~direction, + "pure_numeric", "lcbench", "167168", "val_accuracy", "maximize", + "pure_numeric", "lcbench", "189873", "val_accuracy", "maximize", + "pure_numeric", "lcbench", "189906", "val_accuracy", "maximize", + "pure_numeric", "rbv2_rpart", "14", "acc", "maximize", + "pure_numeric", "rbv2_rpart", "40499", "acc", "maximize", + "pure_numeric", "rbv2_xgboost", "12", "acc", "maximize", + "pure_numeric", "rbv2_xgboost", "1501", "acc", "maximize", + "pure_numeric", "rbv2_xgboost", "40499", "acc", "maximize" +) + + +benchmark = BenchmarkSet$new("lcbench", instance = "167168") +benchmark$subset_codomain("val_accuracy") +objective = benchmark$get_objective("167168", multifidelity = FALSE) +search_space = benchmark$get_search_space(drop_fidelity_params = TRUE) + +optim_instance = oi(objective, search_space = search_space, terminator = trm("none")) + +init = generate_design_lhs(search_space, n = 50)$data # 10% to full budget + +optim_instance$eval_batch(init) + +learner = lrn("regr.ranger", + num.trees = 100L, + se.method = "simple", + splitrule = "variance", + predict_type = "se", + keep.inbag = TRUE, + sample.fraction = 1, + min.node.size = 3, + min.bucket = 3, + mtry.ratio = 5/6 + ) + +surrogate = SurrogateLearner$new(learner) +acq_function = AcqFunctionEI$new() + +surrogate$archive = optim_instance$archive +acq_function$surrogate = surrogate +acq_function$surrogate$update() +acq_function$update() + + +sigma = (objective$domain$upper - objective$domain$lower) / 3 + +# cmaes +acq_optimizer_cmaes = AcqOptimizerCmaes$new() +acq_optimizer_cmaes$param_set$set_values( + maxeval = 1000L, + restart_strategy = "none", + n_restarts = 0L +) +acq_optimizer_cmaes$acq_function = acq_function +acq_optimizer_cmaes$optimize() + + + + + +########## + +library(Rlibcmaes) + +cmaes = function(x0, optimFun, lower, upper, params=cmaEsParams(), cl=NULL) { + stopifnot(all(upper > lower)) + + # set default value for sigma + if (is.null(params$sigma)) { + params$sigma <- stats::median(upper-lower) / 4 + } + + optimFunBlock <- function(x) { + browser() + if (is.null(cl)) + return(apply(x,2,optimFun)) + else + return(parallel::parApply(cl,x,2,optimFun)) + } + Rlibcmaes::cmaesOptim(x0, params$sigma, optimFun, optimFunBlock,lower, upper, cmaAlgo = as.integer(params$cmaAlgorithm), lambda = ifelse(is.null(params$lambda),-1,params$lambda), maxEvals = params$maxEvals, xtol=params$xtol, ftol=params$ftol, traceFreq =params$trace, seed = params$seed, quietRun=params$quiet) +} + + +fun = get_private(acq_function)$.fun +constants = acq_function$constants$values +direction = acq_function$codomain$direction + +wrapper = function(x) { + xdt = set_names(as.data.table(t(xmat)), acq_function$domain$ids()) + res = mlr3misc::invoke(fun, xdt = xdt, .args = constants)[[1]] + res * acq_function$codomain$direction +} + +x0 = set_names(as.numeric(generate_design_random(acq_function$domain, n = 1)$data), acq_function$domain$ids()) + + +res = cmaes(x0 = x0, optimFun = wrapper, lower = acq_function$domain$lower, upper = acq_function$domain$upper, params = cmaEsParams(xtol = 1e-3,ftol = 1e-3)) + +res = cmaes(x0 = x0, optimFun = wrapper, lower = acq_function$domain$lower, upper = acq_function$domain$upper, params = cmaEsParams(cmaAlgorithm = cmaEsAlgo()$IPOP_CMAES,maxEvals=1e4)) + + +wrapper = function(xmat) { # fun, constants, direction + xdt = set_names(as.data.table(t(xmat)), acq_function$domain$ids()) + res = mlr3misc::invoke(fun, xdt = xdt, .args = constants)[[1]] + res * direction +} + +optimFunBlock = function(x) { + wrapper(x) +} + +lower = acq_function$domain$lower +upper = acq_function$domain$upper + +res = Rlibcmaes::cmaesOptim( + x0 = x0, + sigma = median(upper - lower) / 4, + optimFun = wrapper, + optimFunBlock = optimFunBlock, + lower = lower, + upper = upper, + cmaAlgo = as.integer(cmaEsAlgo()$IPOP_CMAES), + lambda = -1, + maxEvals = 1e4, + xtol = 1e-3, +) + +wrapper(res) + + +## random search +acq_optimizer_random_search = AcqOptimizer$new(opt("random_search", batch_size = 1000L), trm("evals", n_evals = 1000L)) +acq_optimizer_random_search$acq_function = acq_function +res_random_search = acq_optimizer_random_search$optimize() + +res_random_search + + + +# cmaes +acq_optimizer_cmaes = AcqOptimizerCmaes$new() +acq_optimizer_cmaes$param_set$set_values( + maxEvals = 30000L, + xtol = 1e-4 +) +acq_optimizer_cmaes$acq_function = acq_function +acq_optimizer_cmaes$optimize() From 56744d287fd9e021770b241262bc200c0698c3eb Mon Sep 17 00:00:00 2001 From: be-marc Date: Wed, 6 Aug 2025 17:16:26 +0200 Subject: [PATCH 107/126] ... --- R/AcqOptimizerCmaes.R | 7 ++++--- tests/testthat/test_AcqOptimizerCmaes.R | 14 +++++++++----- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/R/AcqOptimizerCmaes.R b/R/AcqOptimizerCmaes.R index 3f311f9b..edf33bfe 100644 --- a/R/AcqOptimizerCmaes.R +++ b/R/AcqOptimizerCmaes.R @@ -65,7 +65,6 @@ AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", constants = self$acq_function$constants$values direction = self$acq_function$codomain$direction - wrapper = function(xmat) { xdt = set_names(as.data.table(t(xmat)), self$acq_function$domain$ids()) res = mlr3misc::invoke(fun, xdt = xdt, .args = constants)[[1]] @@ -76,8 +75,10 @@ AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", wrapper(x) } - lower = acq_function$domain$lower - upper = acq_function$domain$upper + lower = self$acq_function$domain$lower + upper = self$acq_function$domain$upper + + x0 = as.numeric(self$acq_function$archive$best()[, self$acq_function$domain$ids(), with = FALSE]) res = Rlibcmaes::cmaesOptim( x0 = x0, diff --git a/tests/testthat/test_AcqOptimizerCmaes.R b/tests/testthat/test_AcqOptimizerCmaes.R index 971ce24d..25a2ac3b 100644 --- a/tests/testthat/test_AcqOptimizerCmaes.R +++ b/tests/testthat/test_AcqOptimizerCmaes.R @@ -6,13 +6,13 @@ test_that("AcqOptimizerCmaes works", { surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) acqfun = acqf("ei", surrogate = surrogate) acqopt = AcqOptimizerCmaes$new(acq_function = acqfun) - acqopt$param_set$set_values(maxeval = 100L) + acqopt$param_set$set_values(maxEvals = 100L) acqfun$surrogate$update() acqfun$update() expect_data_table(acqopt$optimize(), nrows = 1L) - expect_list(acqopt$state) - expect_names(names(acqopt$state), must.include = "iteration_1") - expect_class(acqopt$state$iteration_1, "cma_es.result") + # expect_list(acqopt$state) + # expect_names(names(acqopt$state), must.include = "iteration_1") + # expect_class(acqopt$state$iteration_1, "cma_es.result") }) test_that("AcqOptimizerCmaes works with 2D", { @@ -58,8 +58,12 @@ test_that("AcqOptimizerCmaes works with ipop restart", { acqfun$surrogate$update() acqfun$update() - expect_data_table(acqopt$optimize(), nrows = 1L) + res = acqopt$optimize() + expect_data_table(res, nrows = 1L) expect_list(acqopt$state, len = 4L) expect_names(names(acqopt$state), identical.to = c("iteration_1", "iteration_2", "iteration_3", "iteration_4")) walk(acqopt$state, function(x) expect_class(x, "cma_es.result")) + + ys = map_dbl(acqopt$state, function(x) x$value) * -1 + expect_equal(res$acq_ei, max(ys)) }) From 51ccaf3437e64e333054655fe2399d2705406f87 Mon Sep 17 00:00:00 2001 From: be-marc Date: Wed, 6 Aug 2025 17:17:43 +0200 Subject: [PATCH 108/126] ... --- R/AcqOptimizerCmaes.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/AcqOptimizerCmaes.R b/R/AcqOptimizerCmaes.R index edf33bfe..5c90959c 100644 --- a/R/AcqOptimizerCmaes.R +++ b/R/AcqOptimizerCmaes.R @@ -87,7 +87,7 @@ AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", optimFunBlock = optimFunBlock, lower = lower, upper = upper, - cmaAlgo = as.integer(cmaEsAlgo()$IPOP_CMAES), + cmaAlgo = as.integer(Rlibcmaes::cmaEsAlgo()$IPOP_CMAES), lambda = -1, maxEvals = pv$maxEvals, xtol = pv$xtol, From d9d665f9adef02a1f40f200f50c93d47857ea49a Mon Sep 17 00:00:00 2001 From: be-marc Date: Fri, 8 Aug 2025 12:28:46 +0200 Subject: [PATCH 109/126] ... --- R/AcqOptimizerDirect.R | 5 ++++- R/AcqOptimizerLbfgsb.R | 7 ++++++- tests/testthat/test_AcqOptimizerDirect.R | 16 ++++++++-------- tests/testthat/test_AcqOptimizerLbfgsb.R | 16 ++++++++-------- 4 files changed, 26 insertions(+), 18 deletions(-) diff --git a/R/AcqOptimizerDirect.R b/R/AcqOptimizerDirect.R index 5af87e6d..a18a697d 100644 --- a/R/AcqOptimizerDirect.R +++ b/R/AcqOptimizerDirect.R @@ -52,6 +52,7 @@ AcqOptimizerDirect = R6Class("AcqOptimizerDirect", optimize = function() { pv = self$param_set$values min_iterations = if (pv$restart_strategy == "random") pv$n_restarts + 1L else 1L + restart_strategy = pv$restart_strategy pv$n_restarts = NULL pv$restart_strategy = NULL @@ -98,7 +99,9 @@ AcqOptimizerDirect = R6Class("AcqOptimizerDirect", n = n + res$iterations - self$state = c(self$state, set_names(list(res), paste0("iteration_", i))) + self$state = c(self$state, set_names(list(list(model = res, start = x0)), paste0("iteration_", i))) + + if (restart_strategy == "none") break } as.data.table(as.list(set_names(c(x, y * direction), c(self$acq_function$domain$ids(), self$acq_function$codomain$ids())))) }, diff --git a/R/AcqOptimizerLbfgsb.R b/R/AcqOptimizerLbfgsb.R index 1552d1b0..8eb43e4c 100644 --- a/R/AcqOptimizerLbfgsb.R +++ b/R/AcqOptimizerLbfgsb.R @@ -53,6 +53,7 @@ AcqOptimizerLbfgsb = R6Class("AcqOptimizerLbfgsb", optimize = function() { pv = self$param_set$values min_iterations = if (pv$restart_strategy == "random") pv$n_restarts + 1L else 1L + restart_strategy = pv$restart_strategy pv$n_restarts = NULL pv$restart_strategy = NULL @@ -102,7 +103,11 @@ AcqOptimizerLbfgsb = R6Class("AcqOptimizerLbfgsb", x = res$solution } - self$state = c(self$state, set_names(list(res), paste0("iteration_", n))) + n = n + res$iterations + + self$state = c(self$state, set_names(list(list(model = res, start = x0)), paste0("iteration_", i))) + + if (restart_strategy == "none") break } as.data.table(as.list(set_names(c(x, y * direction), c(self$acq_function$domain$ids(), self$acq_function$codomain$ids())))) }, diff --git a/tests/testthat/test_AcqOptimizerDirect.R b/tests/testthat/test_AcqOptimizerDirect.R index 12cba437..f53d6e9c 100644 --- a/tests/testthat/test_AcqOptimizerDirect.R +++ b/tests/testthat/test_AcqOptimizerDirect.R @@ -13,8 +13,8 @@ test_that("AcqOptimizerDirect works", { expect_data_table(acqopt$optimize(), nrows = 1L) expect_list(acqopt$state) expect_names(names(acqopt$state), must.include = "iteration_1") - expect_class(acqopt$state$iteration_1, "nloptr") - expect_true(acqopt$state$iteration_1$iterations <= 200L) + expect_class(acqopt$state$iteration_1$model, "nloptr") + expect_true(acqopt$state$iteration_1$model$iterations <= 200L) }) test_that("AcqOptimizerDirect works with 2D", { @@ -32,8 +32,8 @@ test_that("AcqOptimizerDirect works with 2D", { expect_data_table(acqopt$optimize(), nrows = 1L) expect_list(acqopt$state) expect_names(names(acqopt$state), must.include = "iteration_1") - expect_class(acqopt$state$iteration_1, "nloptr") - expect_true(acqopt$state$iteration_1$iterations <= 200L) + expect_class(acqopt$state$iteration_1$model, "nloptr") + expect_true(acqopt$state$iteration_1$model$iterations <= 200L) }) test_that("AcqOptimizerDirect works with instance", { @@ -63,8 +63,8 @@ test_that("AcqOptimizerDirect works with random restart", { acqfun$update() expect_data_table(acqopt$optimize(), nrows = 1L) - expect_list(acqopt$state, len = 4L) - expect_names(names(acqopt$state), identical.to = c("iteration_1", "iteration_2", "iteration_3", "iteration_4")) - walk(acqopt$state, function(x) expect_class(x, "nloptr")) - expect_true(all(sapply(acqopt$state, function(x) x$iterations) <= 50L)) + expect_list(acqopt$state, min.len = 4L) + expect_names(names(acqopt$state), must.include = c("iteration_1", "iteration_2", "iteration_3", "iteration_4")) + walk(acqopt$state, function(x) expect_class(x$model, "nloptr")) + expect_true(all(sapply(acqopt$state, function(x) x$model$iterations) <= 50L)) }) diff --git a/tests/testthat/test_AcqOptimizerLbfgsb.R b/tests/testthat/test_AcqOptimizerLbfgsb.R index 3ffc37e9..9a1efbd4 100644 --- a/tests/testthat/test_AcqOptimizerLbfgsb.R +++ b/tests/testthat/test_AcqOptimizerLbfgsb.R @@ -13,8 +13,8 @@ test_that("AcqOptimizerLbfgsb works", { expect_data_table(acqopt$optimize(), nrows = 1L) expect_list(acqopt$state) expect_names(names(acqopt$state), must.include = "iteration_1") - expect_class(acqopt$state$iteration_1, "nloptr") - expect_true(acqopt$state$iteration_1$iterations <= 200L) + expect_class(acqopt$state$iteration_1$model, "nloptr") + expect_true(acqopt$state$iteration_1$model$iterations <= 200L) }) test_that("AcqOptimizerLbfgsb works with 2D", { @@ -32,8 +32,8 @@ test_that("AcqOptimizerLbfgsb works with 2D", { expect_data_table(acqopt$optimize(), nrows = 1L) expect_list(acqopt$state) expect_names(names(acqopt$state), must.include = "iteration_1") - expect_class(acqopt$state$iteration_1, "nloptr") - expect_true(acqopt$state$iteration_1$iterations <= 200L) + expect_class(acqopt$state$iteration_1$model, "nloptr") + expect_true(acqopt$state$iteration_1$model$iterations <= 200L) }) test_that("AcqOptimizerLbfgsb works with instance", { @@ -63,8 +63,8 @@ test_that("AcqOptimizerLbfgsb works with random restart", { acqfun$update() expect_data_table(acqopt$optimize(), nrows = 1L) - expect_list(acqopt$state, len = 4L) - expect_names(names(acqopt$state), identical.to = c("iteration_1", "iteration_2", "iteration_3", "iteration_4")) - walk(acqopt$state, function(x) expect_class(x, "nloptr")) - expect_true(all(sapply(acqopt$state, function(x) x$iterations) <= 50L)) + expect_list(acqopt$state, min.len = 4L) + expect_names(names(acqopt$state), must.include = c("iteration_1", "iteration_2", "iteration_3", "iteration_4")) + walk(acqopt$state, function(x) expect_class(x$model, "nloptr")) + expect_true(all(sapply(acqopt$state, function(x) x$model$iterations) <= 50L)) }) From b0080987654aa3782c86aee682e146f56e0ed648 Mon Sep 17 00:00:00 2001 From: be-marc Date: Fri, 8 Aug 2025 12:42:15 +0200 Subject: [PATCH 110/126] ... --- R/AcqOptimizerCmaes.R | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/R/AcqOptimizerCmaes.R b/R/AcqOptimizerCmaes.R index 5c90959c..a7c7b5ff 100644 --- a/R/AcqOptimizerCmaes.R +++ b/R/AcqOptimizerCmaes.R @@ -17,7 +17,8 @@ AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", self$acq_function = assert_r6(acq_function, "AcqFunction", null.ok = TRUE) param_set = ps( maxEvals = p_int(lower = 1, init = 1000L), - xtol = p_dbl(init = 1e-3) + xtol = p_dbl(init = 1e-12), + ftol = p_dbl(init = 1e-12) # maxeval = p_int(lower = 1, init = 1000L, special_vals = list(-1)), # fnscale = p_dbl(default = 1), # #maxit = p_int(lower = 1L), @@ -91,6 +92,9 @@ AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", lambda = -1, maxEvals = pv$maxEvals, xtol = pv$xtol, + ftol = pv$ftol, + quiet = TRUE, + trace = pv$maxEvals + 1L ) y = wrapper(res) From d49a80a3512277683e7e75884130a83720472e80 Mon Sep 17 00:00:00 2001 From: be-marc Date: Mon, 15 Sep 2025 18:16:54 +0200 Subject: [PATCH 111/126] add libcmaesr --- DESCRIPTION | 8 ++++-- R/AcqOptimizerCmaes.R | 37 ++++++++++--------------- man/AcqOptimizerCmaes.Rd | 9 +++--- tests/testthat/test_AcqOptimizerCmaes.R | 8 ++---- 4 files changed, 26 insertions(+), 36 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index c90e2e4a..9d6753bb 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -57,6 +57,7 @@ Suggests: emoa, fastGHQuad, lhs, + libcmaesr, mlr3learners (>= 0.7.0), mlr3pipelines (>= 0.5.2), nloptr, @@ -68,15 +69,16 @@ Suggests: stringi, testthat (>= 3.0.0), Remotes: - mlr-org/mlr3learners@ranger_se, - mlr-org/bbotk@ls-in-c + mlr-org/mlr3learners, + mlr-org/bbotk@ls-in-c, + mlr-org/libcmaesr ByteCompile: no Encoding: UTF-8 Config/testthat/edition: 3 Config/testthat/parallel: false NeedsCompilation: yes Roxygen: list(markdown = TRUE, r6 = TRUE) -RoxygenNote: 7.3.2 +RoxygenNote: 7.3.3 Collate: 'mlr_acqfunctions.R' 'AcqFunction.R' diff --git a/R/AcqOptimizerCmaes.R b/R/AcqOptimizerCmaes.R index a7c7b5ff..658b9b59 100644 --- a/R/AcqOptimizerCmaes.R +++ b/R/AcqOptimizerCmaes.R @@ -6,7 +6,7 @@ AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", public = list( #' @field state (`list()`)\cr - #' List of [cmaes::cma_es()] results. + #' [libcmaesr::cmaes()] results. state = NULL, #' @description @@ -16,9 +16,7 @@ AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", initialize = function(acq_function = NULL) { self$acq_function = assert_r6(acq_function, "AcqFunction", null.ok = TRUE) param_set = ps( - maxEvals = p_int(lower = 1, init = 1000L), - xtol = p_dbl(init = 1e-12), - ftol = p_dbl(init = 1e-12) + max_fevals = p_int(lower = 1, init = 1000L) # maxeval = p_int(lower = 1, init = 1000L, special_vals = list(-1)), # fnscale = p_dbl(default = 1), # #maxit = p_int(lower = 1L), @@ -66,40 +64,33 @@ AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", constants = self$acq_function$constants$values direction = self$acq_function$codomain$direction + control = libcmaesr::cmaes_control( + maximize = direction == -1L, + algo = "abipop", + max_fevals = pv$max_fevals + ) + + wrapper = function(xmat) { xdt = set_names(as.data.table(t(xmat)), self$acq_function$domain$ids()) res = mlr3misc::invoke(fun, xdt = xdt, .args = constants)[[1]] res * direction } - optimFunBlock = function(x) { - wrapper(x) - } - lower = self$acq_function$domain$lower upper = self$acq_function$domain$upper - x0 = as.numeric(self$acq_function$archive$best()[, self$acq_function$domain$ids(), with = FALSE]) - res = Rlibcmaes::cmaesOptim( + res = libcmaesr::cmaes( + objective = wrapper, x0 = x0, - sigma = median(upper - lower) / 4, - optimFun = wrapper, - optimFunBlock = optimFunBlock, lower = lower, upper = upper, - cmaAlgo = as.integer(Rlibcmaes::cmaEsAlgo()$IPOP_CMAES), - lambda = -1, - maxEvals = pv$maxEvals, - xtol = pv$xtol, - ftol = pv$ftol, - quiet = TRUE, - trace = pv$maxEvals + 1L - ) + control = control) - y = wrapper(res) + self$state = res - as.data.table(as.list(set_names(c(res, y * direction), c(self$acq_function$domain$ids(), self$acq_function$codomain$ids())))) + as.data.table(as.list(set_names(c(res$x, res$y * direction), c(self$acq_function$domain$ids(), self$acq_function$codomain$ids())))) }, #' @description diff --git a/man/AcqOptimizerCmaes.Rd b/man/AcqOptimizerCmaes.Rd index 11122ef3..2a5a35dc 100644 --- a/man/AcqOptimizerCmaes.Rd +++ b/man/AcqOptimizerCmaes.Rd @@ -4,10 +4,9 @@ \alias{AcqOptimizerCmaes} \title{CMA-ES Acquisition Function Optimizer} \description{ -IPOP CMA-ES runs for \code{n_iterations} iterations. -The population size is increased by \code{population_multiplier} after each iteration. -In the first iteration, the start point is the best point in the archive. -In the subsequent iterations, the start point is a random point in the search space. +CMA-ES Acquisition Function Optimizer + +CMA-ES Acquisition Function Optimizer } \section{Super class}{ \code{\link[mlr3mbo:AcqOptimizer]{mlr3mbo::AcqOptimizer}} -> \code{AcqOptimizerCmaes} @@ -16,7 +15,7 @@ In the subsequent iterations, the start point is a random point in the search sp \if{html}{\out{
    }} \describe{ \item{\code{state}}{(\code{list()})\cr -List of \code{\link[cmaes:cma_es]{cmaes::cma_es()}} results.} +\code{\link[libcmaesr:cmaes]{libcmaesr::cmaes()}} results.} } \if{html}{\out{
    }} } diff --git a/tests/testthat/test_AcqOptimizerCmaes.R b/tests/testthat/test_AcqOptimizerCmaes.R index 25a2ac3b..51cf276e 100644 --- a/tests/testthat/test_AcqOptimizerCmaes.R +++ b/tests/testthat/test_AcqOptimizerCmaes.R @@ -6,17 +6,15 @@ test_that("AcqOptimizerCmaes works", { surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) acqfun = acqf("ei", surrogate = surrogate) acqopt = AcqOptimizerCmaes$new(acq_function = acqfun) - acqopt$param_set$set_values(maxEvals = 100L) + acqopt$param_set$set_values(max_fevals = 100L) acqfun$surrogate$update() acqfun$update() expect_data_table(acqopt$optimize(), nrows = 1L) - # expect_list(acqopt$state) - # expect_names(names(acqopt$state), must.include = "iteration_1") - # expect_class(acqopt$state$iteration_1, "cma_es.result") + expect_list(acqopt$state) }) test_that("AcqOptimizerCmaes works with 2D", { - instance = oi(OBJ_2D, terminator = trm("evals", n_evals = 5L)) + instance = oi(OBJ_2D, terminator = trm("evals", n_evals = 5L))^ design = generate_design_grid(instance$search_space, resolution = 4L)$data instance$eval_batch(design) From 8386a48933fdaa5d40136bcad795abff7b835fba Mon Sep 17 00:00:00 2001 From: be-marc Date: Mon, 15 Sep 2025 19:18:47 +0200 Subject: [PATCH 112/126] ... --- R/AcqOptimizerCmaes.R | 1 - tests/testthat/test_AcqOptimizerCmaes.R | 30 ++++--------------------- 2 files changed, 4 insertions(+), 27 deletions(-) diff --git a/R/AcqOptimizerCmaes.R b/R/AcqOptimizerCmaes.R index 658b9b59..754ec20d 100644 --- a/R/AcqOptimizerCmaes.R +++ b/R/AcqOptimizerCmaes.R @@ -70,7 +70,6 @@ AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", max_fevals = pv$max_fevals ) - wrapper = function(xmat) { xdt = set_names(as.data.table(t(xmat)), self$acq_function$domain$ids()) res = mlr3misc::invoke(fun, xdt = xdt, .args = constants)[[1]] diff --git a/tests/testthat/test_AcqOptimizerCmaes.R b/tests/testthat/test_AcqOptimizerCmaes.R index 51cf276e..3a1e9baf 100644 --- a/tests/testthat/test_AcqOptimizerCmaes.R +++ b/tests/testthat/test_AcqOptimizerCmaes.R @@ -14,20 +14,19 @@ test_that("AcqOptimizerCmaes works", { }) test_that("AcqOptimizerCmaes works with 2D", { - instance = oi(OBJ_2D, terminator = trm("evals", n_evals = 5L))^ + instance = oi(OBJ_2D, terminator = trm("evals", n_evals = 5L)) design = generate_design_grid(instance$search_space, resolution = 4L)$data instance$eval_batch(design) surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) acqfun = acqf("ei", surrogate = surrogate) acqopt = AcqOptimizerCmaes$new(acq_function = acqfun) - acqopt$param_set$set_values(maxeval = 100L) + acqopt$param_set$set_values(max_fevals = 100L) acqfun$surrogate$update() acqfun$update() expect_data_table(acqopt$optimize(), nrows = 1L) expect_list(acqopt$state) - expect_names(names(acqopt$state), must.include = "iteration_1") - expect_class(acqopt$state$iteration_1, "cma_es.result") + expect_list(acqopt$state) }) test_that("AcqOptimizerCmaes works with instance", { @@ -38,30 +37,9 @@ test_that("AcqOptimizerCmaes works with instance", { surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) acqfun = acqf("ei", surrogate = surrogate) acqopt = AcqOptimizerCmaes$new(acq_function = acqfun) - acqopt$param_set$set_values(maxeval = 100L) + acqopt$param_set$set_values(max_fevals = 100L) optimizer = opt("mbo", acq_optimizer = acqopt, acq_function = acqfun, surrogate = surrogate) expect_data_table(optimizer$optimize(instance), nrow = 1L) }) -test_that("AcqOptimizerCmaes works with ipop restart", { - instance = oi(OBJ_1D, terminator = trm("evals", n_evals = 10L)) - design = generate_design_grid(instance$search_space, resolution = 4L)$data - instance$eval_batch(design) - - surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) - acqfun = acqf("ei", surrogate = surrogate) - acqopt = AcqOptimizerCmaes$new(acq_function = acqfun) - acqopt$param_set$set_values(maxeval = 200L, restart_strategy = "ipop", n_restarts = 3L, population_multiplier = 2L) - acqfun$surrogate$update() - acqfun$update() - - res = acqopt$optimize() - expect_data_table(res, nrows = 1L) - expect_list(acqopt$state, len = 4L) - expect_names(names(acqopt$state), identical.to = c("iteration_1", "iteration_2", "iteration_3", "iteration_4")) - walk(acqopt$state, function(x) expect_class(x, "cma_es.result")) - - ys = map_dbl(acqopt$state, function(x) x$value) * -1 - expect_equal(res$acq_ei, max(ys)) -}) From 4bf295e48bc0a899f14574462a3dd34c9383c420 Mon Sep 17 00:00:00 2001 From: be-marc Date: Mon, 15 Sep 2025 22:19:26 +0200 Subject: [PATCH 113/126] ... --- R/AcqOptimizerCmaes.R | 6 +++--- R/SurrogateLearner.R | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/R/AcqOptimizerCmaes.R b/R/AcqOptimizerCmaes.R index 754ec20d..5fb2b886 100644 --- a/R/AcqOptimizerCmaes.R +++ b/R/AcqOptimizerCmaes.R @@ -71,9 +71,8 @@ AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", ) wrapper = function(xmat) { - xdt = set_names(as.data.table(t(xmat)), self$acq_function$domain$ids()) - res = mlr3misc::invoke(fun, xdt = xdt, .args = constants)[[1]] - res * direction + xdt = set_names(as.data.table(xmat), self$acq_function$domain$ids()) + mlr3misc::invoke(fun, xdt = xdt, .args = constants)[[1]] } lower = self$acq_function$domain$lower @@ -85,6 +84,7 @@ AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", x0 = x0, lower = lower, upper = upper, + batch = TRUE, control = control) self$state = res diff --git a/R/SurrogateLearner.R b/R/SurrogateLearner.R index a452b6a6..42deff5d 100644 --- a/R/SurrogateLearner.R +++ b/R/SurrogateLearner.R @@ -115,7 +115,6 @@ SurrogateLearner = R6Class("SurrogateLearner", xdt = self$input_trafo$transform(xdt) } - pred = self$learner$predict_newdata_fast(xdt) pred = set_names(pred, c("mean", "se")) From be65653dc5838af965e854f170f766896784a44f Mon Sep 17 00:00:00 2001 From: be-marc Date: Thu, 25 Sep 2025 16:45:54 +0200 Subject: [PATCH 114/126] ... --- DESCRIPTION | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 9d6753bb..e9490e3f 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -39,7 +39,7 @@ License: LGPL-3 URL: https://mlr3mbo.mlr-org.com, https://github.com/mlr-org/mlr3mbo BugReports: https://github.com/mlr-org/mlr3mbo/issues Depends: - mlr3 (>= 1.1.0), + mlr3 (>= 1.2.0), mlr3tuning (>= 1.4.0), R (>= 3.1.0) Imports: @@ -58,7 +58,7 @@ Suggests: fastGHQuad, lhs, libcmaesr, - mlr3learners (>= 0.7.0), + mlr3learners (>= 0.12.0), mlr3pipelines (>= 0.5.2), nloptr, ranger, @@ -69,8 +69,7 @@ Suggests: stringi, testthat (>= 3.0.0), Remotes: - mlr-org/mlr3learners, - mlr-org/bbotk@ls-in-c, + mlr-org/bbotk@benchmark, mlr-org/libcmaesr ByteCompile: no Encoding: UTF-8 From ab400d195a378f33a93e6fda04da6d94cd871672 Mon Sep 17 00:00:00 2001 From: be-marc Date: Fri, 26 Sep 2025 14:13:31 +0200 Subject: [PATCH 115/126] ... --- R/SurrogateLearner.R | 40 +++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/R/SurrogateLearner.R b/R/SurrogateLearner.R index 42deff5d..a79f179c 100644 --- a/R/SurrogateLearner.R +++ b/R/SurrogateLearner.R @@ -115,27 +115,33 @@ SurrogateLearner = R6Class("SurrogateLearner", xdt = self$input_trafo$transform(xdt) } - pred = self$learner$predict_newdata_fast(xdt) - pred = set_names(pred, c("mean", "se")) - - # speeding up some checks by constructing the predict task directly instead of relying on predict_newdata - # task = self$learner$state$train_task$clone() - # set(xdt, j = task$target_names, value = NA_real_) # tasks only have features and the target but we have to set the target to NA - # newdata = as_data_backend(xdt) - # task$backend = newdata - # task$row_roles$use = task$backend$rownames - # pred = self$learner$predict(task) - + if (!inherits(self$learner, "GraphLearner")) { + pred = self$learner$predict_newdata_fast(xdt) + pred = set_names(pred, c("mean", "se")) + } else { + # speeding up some checks by constructing the predict task directly instead of relying on predict_newdata + task = self$learner$state$train_task$clone() + set(xdt, j = task$target_names, value = NA_real_) # tasks only have features and the target but we have to set the target to NA + newdata = as_data_backend(xdt) + task$backend = newdata + task$row_roles$use = task$backend$rownames + pred = self$learner$predict(task) + + pred = if (self$learner$predict_type == "se") { + list(mean = pred$response, se = pred$se) + } else { + list(mean = pred$response) + } + } # slow #pred = self$learner$predict_newdata(newdata = xdt) - # pred = if (self$learner$predict_type == "se") { - # data.table(mean = pred$response, se = pred$se) - # } else { - # data.table(mean = pred$response) - # } + + + + if (!is.null(self$output_trafo) && self$output_trafo$invert_posterior) { - pred = self$output_trafo$inverse_transform_posterior(pred) + pred = self$output_trafo$inverse_transform_posterior(as.data.table(pred)) } pred } From 9656e5b21713fc27b3d83f5930a775bac18c4c63 Mon Sep 17 00:00:00 2001 From: be-marc Date: Sun, 28 Sep 2025 19:31:10 +0200 Subject: [PATCH 116/126] ... --- DESCRIPTION | 1 + NAMESPACE | 1 + R/AcqOptimzerRandomSearch.R | 118 ++++++++++++++++++ man/AcqOptimizerRandomSearch.Rd | 101 +++++++++++++++ .../testthat/test_AcqOptimizerRandomSearch.R | 70 +++++++++++ 5 files changed, 291 insertions(+) create mode 100644 R/AcqOptimzerRandomSearch.R create mode 100644 man/AcqOptimizerRandomSearch.Rd create mode 100644 tests/testthat/test_AcqOptimizerRandomSearch.R diff --git a/DESCRIPTION b/DESCRIPTION index e9490e3f..ef1ebf78 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -100,6 +100,7 @@ Collate: 'AcqOptimizerDirect.R' 'AcqOptimizerLbfgsb.R' 'AcqOptimizerLocalSearch.R' + 'AcqOptimzerRandomSearch.R' 'mlr_input_trafos.R' 'InputTrafo.R' 'InputTrafoUnitcube.R' diff --git a/NAMESPACE b/NAMESPACE index 57dfb79c..58a83d8d 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -27,6 +27,7 @@ export(AcqOptimizerCmaes) export(AcqOptimizerDirect) export(AcqOptimizerLbfgsb) export(AcqOptimizerLocalSearch) +export(AcqOptimizerRandomSearch) export(InputTrafo) export(InputTrafoUnitcube) export(LearnerRegrRangerMbo) diff --git a/R/AcqOptimzerRandomSearch.R b/R/AcqOptimzerRandomSearch.R new file mode 100644 index 00000000..f96f49b2 --- /dev/null +++ b/R/AcqOptimzerRandomSearch.R @@ -0,0 +1,118 @@ +#' @title Random Search Acquisition Function Optimizer +#' +#' @export +AcqOptimizerRandomSearch = R6Class("AcqOptimizerRandomSearch", + inherit = AcqOptimizer, + public = list( + + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + #' + #' @param acq_function (`NULL` | [AcqFunction]). + initialize = function(acq_function = NULL) { + self$acq_function = assert_r6(acq_function, "AcqFunction", null.ok = TRUE) + param_set = ps( + max_fevals = p_int(lower = 1, init = 1000L) + # maxeval = p_int(lower = 1, init = 1000L, special_vals = list(-1)), + # fnscale = p_dbl(default = 1), + # #maxit = p_int(lower = 1L), + # stopfitness = p_dbl(default = -Inf), + # keep.best = p_lgl(default = TRUE), + # sigma = p_uty(default = 0.5), + # mu = p_int(lower = 1L), + # lambda = p_int(lower = 1L), + # weights = p_uty(), + # damps = p_dbl(), + # cs = p_dbl(), + # ccum = p_dbl(), + # ccov.1 = p_dbl(lower = 0), + # ccov.mu = p_dbl(lower = 0), + # diag.sigma = p_lgl(default = FALSE), + # diag.eigen = p_lgl(default = FALSE), + # diag.pop = p_lgl(default = FALSE), + # diag.value = p_lgl(default = FALSE), + # stop.tolx = p_dbl(), # undocumented stop criterion + # restart_strategy = p_fct(levels = c("none", "ipop"), init = "none"), + # n_restarts = p_int(lower = 0L, init = 0L), + # population_multiplier = p_int(lower = 1, init = 2L) + # start_values = p_fct(default = "random", levels = c("random", "center", "custom")), + # start = p_uty(default = NULL, depends = start_values == "custom"), + # n_candidates = p_int(lower = 1, default = 1L), + # logging_level = p_fct(levels = c("fatal", "error", "warn", "info", "debug", "trace"), default = "warn"), + # warmstart = p_lgl(default = FALSE), + # warmstart_size = p_int(lower = 1L, special_vals = list("all")), + # skip_already_evaluated = p_lgl(default = TRUE), + # catch_errors = p_lgl(default = TRUE) + ) + # ps$values = list(n_candidates = 1, logging_level = "warn", warmstart = FALSE, skip_already_evaluated = TRUE, catch_errors = TRUE) + # ps$add_dep("warmstart_size", on = "warmstart", cond = CondEqual$new(TRUE)) + private$.param_set = param_set + }, + + #' @description + #' Optimize the acquisition function. + #' + #' @return [data.table::data.table()] with 1 row per candidate. + optimize = function() { + pv = self$param_set$values + + fun = get_private(self$acq_function)$.fun + constants = self$acq_function$constants$values + direction = self$acq_function$codomain$direction + + xdt = generate_design_random(self$acq_function$domain, n = pv$max_fevals)$data + + ys = mlr3misc::invoke(fun, xdt = xdt, .args = constants)[[1]] + + id = if (direction == 1) which.min(ys) else which.max(ys) + x = xdt[id, ] + y = ys[id] + + set(x, j = self$acq_function$codomain$ids(), value = y) + x + }, + + #' @description + #' Reset the acquisition function optimizer. + #' + #' Currently not used. + reset = function() { + + } + ), + + active = list( + #' @template field_print_id + print_id = function(rhs) { + if (missing(rhs)) { + paste0("(", class(self$optimizer)[1L], " | ", class(self$terminator)[1L], ")") + } else { + stop("$print_id is read-only.") + } + }, + + #' @field param_set ([paradox::ParamSet])\cr + #' Set of hyperparameters. + param_set = function(rhs) { + if (!missing(rhs) && !identical(rhs, private$.param_set)) { + stop("$param_set is read-only.") + } + private$.param_set + } + ), + + private = list( + .param_set = NULL, + + deep_clone = function(name, value) { + switch(name, + optimizer = value$clone(deep = TRUE), + terminator = value$clone(deep = TRUE), + acq_function = if (!is.null(value)) value$clone(deep = TRUE) else NULL, + .param_set = value$clone(deep = TRUE), + value + ) + } + ) +) + diff --git a/man/AcqOptimizerRandomSearch.Rd b/man/AcqOptimizerRandomSearch.Rd new file mode 100644 index 00000000..60a07970 --- /dev/null +++ b/man/AcqOptimizerRandomSearch.Rd @@ -0,0 +1,101 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/AcqOptimzerRandomSearch.R +\name{AcqOptimizerRandomSearch} +\alias{AcqOptimizerRandomSearch} +\title{Random Search Acquisition Function Optimizer} +\description{ +Random Search Acquisition Function Optimizer + +Random Search Acquisition Function Optimizer +} +\section{Super class}{ +\code{\link[mlr3mbo:AcqOptimizer]{mlr3mbo::AcqOptimizer}} -> \code{AcqOptimizerRandomSearch} +} +\section{Active bindings}{ +\if{html}{\out{
    }} +\describe{ +\item{\code{print_id}}{(\code{character})\cr +Id used when printing.} + +\item{\code{param_set}}{(\link[paradox:ParamSet]{paradox::ParamSet})\cr +Set of hyperparameters.} +} +\if{html}{\out{
    }} +} +\section{Methods}{ +\subsection{Public methods}{ +\itemize{ +\item \href{#method-AcqOptimizerRandomSearch-new}{\code{AcqOptimizerRandomSearch$new()}} +\item \href{#method-AcqOptimizerRandomSearch-optimize}{\code{AcqOptimizerRandomSearch$optimize()}} +\item \href{#method-AcqOptimizerRandomSearch-reset}{\code{AcqOptimizerRandomSearch$reset()}} +\item \href{#method-AcqOptimizerRandomSearch-clone}{\code{AcqOptimizerRandomSearch$clone()}} +} +} +\if{html}{\out{ +
    Inherited methods + +
    +}} +\if{html}{\out{
    }} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerRandomSearch-new}{}}} +\subsection{Method \code{new()}}{ +Creates a new instance of this \link[R6:R6Class]{R6} class. +\subsection{Usage}{ +\if{html}{\out{
    }}\preformatted{AcqOptimizerRandomSearch$new(acq_function = NULL)}\if{html}{\out{
    }} +} + +\subsection{Arguments}{ +\if{html}{\out{
    }} +\describe{ +\item{\code{acq_function}}{(\code{NULL} | \link{AcqFunction}).} +} +\if{html}{\out{
    }} +} +} +\if{html}{\out{
    }} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerRandomSearch-optimize}{}}} +\subsection{Method \code{optimize()}}{ +Optimize the acquisition function. +\subsection{Usage}{ +\if{html}{\out{
    }}\preformatted{AcqOptimizerRandomSearch$optimize()}\if{html}{\out{
    }} +} + +\subsection{Returns}{ +\code{\link[data.table:data.table]{data.table::data.table()}} with 1 row per candidate. +} +} +\if{html}{\out{
    }} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerRandomSearch-reset}{}}} +\subsection{Method \code{reset()}}{ +Reset the acquisition function optimizer. + +Currently not used. +\subsection{Usage}{ +\if{html}{\out{
    }}\preformatted{AcqOptimizerRandomSearch$reset()}\if{html}{\out{
    }} +} + +} +\if{html}{\out{
    }} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-AcqOptimizerRandomSearch-clone}{}}} +\subsection{Method \code{clone()}}{ +The objects of this class are cloneable with this method. +\subsection{Usage}{ +\if{html}{\out{
    }}\preformatted{AcqOptimizerRandomSearch$clone(deep = FALSE)}\if{html}{\out{
    }} +} + +\subsection{Arguments}{ +\if{html}{\out{
    }} +\describe{ +\item{\code{deep}}{Whether to make a deep clone.} +} +\if{html}{\out{
    }} +} +} +} diff --git a/tests/testthat/test_AcqOptimizerRandomSearch.R b/tests/testthat/test_AcqOptimizerRandomSearch.R new file mode 100644 index 00000000..5d30ae20 --- /dev/null +++ b/tests/testthat/test_AcqOptimizerRandomSearch.R @@ -0,0 +1,70 @@ +test_that("AcqOptimizerRandomSearch works", { + instance = oi(OBJ_1D, terminator = trm("evals", n_evals = 5L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerRandomSearch$new(acq_function = acqfun) + acqopt$param_set$set_values(max_fevals = 200L) + acqfun$surrogate$update() + acqfun$update() + + expect_data_table(acqopt$optimize(), nrows = 1L) + expect_list(acqopt$state) + expect_names(names(acqopt$state), must.include = "iteration_1") + expect_class(acqopt$state$iteration_1$model, "nloptr") + expect_true(acqopt$state$iteration_1$model$iterations <= 200L) +}) + +test_that("AcqOptimizerLbfgsb works with 2D", { + instance = oi(OBJ_2D, terminator = trm("evals", n_evals = 5L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerLbfgsb$new(acq_function = acqfun) + acqopt$param_set$set_values(maxeval = 200L) + acqfun$surrogate$update() + acqfun$update() + + expect_data_table(acqopt$optimize(), nrows = 1L) + expect_list(acqopt$state) + expect_names(names(acqopt$state), must.include = "iteration_1") + expect_class(acqopt$state$iteration_1$model, "nloptr") + expect_true(acqopt$state$iteration_1$model$iterations <= 200L) +}) + +test_that("AcqOptimizerLbfgsb works with instance", { + instance = oi(OBJ_1D, terminator = trm("evals", n_evals = 10L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerLbfgsb$new(acq_function = acqfun) + acqopt$param_set$set_values(maxeval = 10L) + + optimizer = opt("mbo", acq_optimizer = acqopt, acq_function = acqfun, surrogate = surrogate) + expect_data_table(optimizer$optimize(instance), nrow = 1L) +}) + +test_that("AcqOptimizerLbfgsb works with random restart", { + instance = oi(OBJ_2D, terminator = trm("evals", n_evals = 5L)) + design = generate_design_grid(instance$search_space, resolution = 4L)$data + instance$eval_batch(design) + + surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) + acqfun = acqf("ei", surrogate = surrogate) + acqopt = AcqOptimizerLbfgsb$new(acq_function = acqfun) + acqopt$param_set$set_values(maxeval = 200L, restart_strategy = "random", n_restarts = 3L) + acqfun$surrogate$update() + acqfun$update() + + expect_data_table(acqopt$optimize(), nrows = 1L) + expect_list(acqopt$state, min.len = 4L) + expect_names(names(acqopt$state), must.include = c("iteration_1", "iteration_2", "iteration_3", "iteration_4")) + walk(acqopt$state, function(x) expect_class(x$model, "nloptr")) + expect_true(all(sapply(acqopt$state, function(x) x$model$iterations) <= 50L)) +}) From f60b5d5e8c82a7ca2575abb116fc99dba24e7a24 Mon Sep 17 00:00:00 2001 From: be-marc Date: Sun, 28 Sep 2025 20:19:39 +0200 Subject: [PATCH 117/126] ... --- R/bayesopt_ego.R | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/R/bayesopt_ego.R b/R/bayesopt_ego.R index 5dff5359..8a7b31a7 100644 --- a/R/bayesopt_ego.R +++ b/R/bayesopt_ego.R @@ -145,14 +145,22 @@ bayesopt_ego = function( if (isTRUE((instance$archive$n_evals - init_design_size + 1L) %% random_interleave_iter == 0)) { stop(set_class(list(message = "Random interleaving", call = NULL), classes = c("random_interleave", "mbo_error", "error", "condition"))) } - acq_function$surrogate$update() + + runtime_train_surrogate = system.time({ + acq_function$surrogate$update() + }) acq_function$update() - acq_optimizer$optimize() + runtime_acq_optimizer = system.time({ + res = acq_optimizer$optimize() + }) + res }, mbo_error = function(mbo_error_condition) { lg$info(paste0(class(mbo_error_condition), collapse = " / ")) lg$info("Proposing a randomly sampled point") generate_design_random(search_space, n = 1L)$data }) + set(xdt, j = "runtime_train_surrogate", value = runtime_train_surrogate["elapsed"]) + set(xdt, j = "runtime_acq_optimizer", value = runtime_acq_optimizer["elapsed"]) instance$eval_batch(xdt) if (instance$is_terminated) break From f89c7fff9230d93f168c05edbdc78af202571fd6 Mon Sep 17 00:00:00 2001 From: be-marc Date: Mon, 29 Sep 2025 12:00:36 +0200 Subject: [PATCH 118/126] ... --- R/AcqOptimizerDirect.R | 15 +++++++++------ tests/testthat/test_AcqOptimizerDirect.R | 2 +- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/R/AcqOptimizerDirect.R b/R/AcqOptimizerDirect.R index a18a697d..82b5856a 100644 --- a/R/AcqOptimizerDirect.R +++ b/R/AcqOptimizerDirect.R @@ -32,7 +32,7 @@ AcqOptimizerDirect = R6Class("AcqOptimizerDirect", ftol_abs = p_dbl(default = 0, lower = 0, upper = Inf, special_vals = list(-1)), minf_max = p_dbl(default = -Inf), restart_strategy = p_fct(levels = c("none", "random"), init = "none"), - n_restarts = p_int(lower = 0L, init = 0L) + max_restarts = p_int(lower = 0L, init = 0L) # n_candidates = p_int(lower = 1, default = 1L), # logging_level = p_fct(levels = c("fatal", "error", "warn", "info", "debug", "trace"), default = "warn"), # warmstart = p_lgl(default = FALSE), @@ -51,9 +51,10 @@ AcqOptimizerDirect = R6Class("AcqOptimizerDirect", #' @return [data.table::data.table()] with 1 row per candidate. optimize = function() { pv = self$param_set$values - min_iterations = if (pv$restart_strategy == "random") pv$n_restarts + 1L else 1L + #min_iterations = if (pv$restart_strategy == "random") pv$n_restarts + 1L else 1L restart_strategy = pv$restart_strategy - pv$n_restarts = NULL + max_restarts = pv$max_restarts + pv$max_restarts = NULL pv$restart_strategy = NULL wrapper = function(x, fun, constants, direction) { @@ -69,8 +70,8 @@ AcqOptimizerDirect = R6Class("AcqOptimizerDirect", y = Inf n = 0L i = 0L - maxeval_i = ceiling(pv$maxeval / min_iterations) - while (n < pv$maxeval) { + maxeval = if (pv$maxeval == -1) Inf else pv$maxeval + while (n < maxeval && i <= max_restarts) { i = i + 1L x0 = if (i == 1L) { @@ -85,13 +86,15 @@ AcqOptimizerDirect = R6Class("AcqOptimizerDirect", eval_f = wrapper, lb = self$acq_function$domain$lower, ub = self$acq_function$domain$upper, - opts = insert_named(pv, list(algorithm = "NLOPT_GN_DIRECT_L", maxeval = min(maxeval_i, pv$maxeval - n))), + opts = insert_named(pv, list(algorithm = "NLOPT_GN_DIRECT_L", maxeval = maxeval - n)), eval_grad_f = NULL, x0 = x0, fun = fun, constants = constants, direction = direction) + browser() + if (res$objective < y) { y = res$objective x = res$solution diff --git a/tests/testthat/test_AcqOptimizerDirect.R b/tests/testthat/test_AcqOptimizerDirect.R index f53d6e9c..3124d9d2 100644 --- a/tests/testthat/test_AcqOptimizerDirect.R +++ b/tests/testthat/test_AcqOptimizerDirect.R @@ -58,7 +58,7 @@ test_that("AcqOptimizerDirect works with random restart", { surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) acqfun = acqf("ei", surrogate = surrogate) acqopt = AcqOptimizerDirect$new(acq_function = acqfun) - acqopt$param_set$set_values(maxeval = 200L, restart_strategy = "random", n_restarts = 3L) + acqopt$param_set$set_values(maxeval = -1, ftol_rel = 1e-6, restart_strategy = "random", max_restarts = 3L) acqfun$surrogate$update() acqfun$update() From fe910e16f08d9cbd1998c922c8b0064ebb5df44d Mon Sep 17 00:00:00 2001 From: be-marc Date: Mon, 29 Sep 2025 12:07:52 +0200 Subject: [PATCH 119/126] ... --- R/AcqOptimizerDirect.R | 2 -- 1 file changed, 2 deletions(-) diff --git a/R/AcqOptimizerDirect.R b/R/AcqOptimizerDirect.R index 82b5856a..9a9c4241 100644 --- a/R/AcqOptimizerDirect.R +++ b/R/AcqOptimizerDirect.R @@ -93,8 +93,6 @@ AcqOptimizerDirect = R6Class("AcqOptimizerDirect", constants = constants, direction = direction) - browser() - if (res$objective < y) { y = res$objective x = res$solution From d71f0f286fce4aafce2a2bad1356b85cfa64e63d Mon Sep 17 00:00:00 2001 From: be-marc Date: Mon, 29 Sep 2025 12:22:50 +0200 Subject: [PATCH 120/126] ... --- R/AcqOptimizerDirect.R | 2 -- R/AcqOptimizerLbfgsb.R | 13 +++++++------ tests/testthat/test_AcqOptimizerLbfgsb.R | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/R/AcqOptimizerDirect.R b/R/AcqOptimizerDirect.R index 9a9c4241..e701c217 100644 --- a/R/AcqOptimizerDirect.R +++ b/R/AcqOptimizerDirect.R @@ -51,7 +51,6 @@ AcqOptimizerDirect = R6Class("AcqOptimizerDirect", #' @return [data.table::data.table()] with 1 row per candidate. optimize = function() { pv = self$param_set$values - #min_iterations = if (pv$restart_strategy == "random") pv$n_restarts + 1L else 1L restart_strategy = pv$restart_strategy max_restarts = pv$max_restarts pv$max_restarts = NULL @@ -150,4 +149,3 @@ AcqOptimizerDirect = R6Class("AcqOptimizerDirect", } ) ) - diff --git a/R/AcqOptimizerLbfgsb.R b/R/AcqOptimizerLbfgsb.R index 8eb43e4c..37f00543 100644 --- a/R/AcqOptimizerLbfgsb.R +++ b/R/AcqOptimizerLbfgsb.R @@ -33,7 +33,8 @@ AcqOptimizerLbfgsb = R6Class("AcqOptimizerLbfgsb", ftol_abs = p_dbl(default = 0, lower = 0, upper = Inf, special_vals = list(-1)), minf_max = p_dbl(default = -Inf), restart_strategy = p_fct(levels = c("none", "random"), init = "none"), - n_restarts = p_int(lower = 0L, init = 0L) + n_restarts = p_int(lower = 0L, init = 0L), + max_restarts = p_int(lower = 0L, init = 0L) # n_candidates = p_int(lower = 1, default = 1L), # logging_level = p_fct(levels = c("fatal", "error", "warn", "info", "debug", "trace"), default = "warn"), # warmstart = p_lgl(default = FALSE), @@ -52,9 +53,9 @@ AcqOptimizerLbfgsb = R6Class("AcqOptimizerLbfgsb", #' @return [data.table::data.table()] with 1 row per candidate. optimize = function() { pv = self$param_set$values - min_iterations = if (pv$restart_strategy == "random") pv$n_restarts + 1L else 1L restart_strategy = pv$restart_strategy - pv$n_restarts = NULL + max_restarts = pv$max_restarts + pv$max_restarts = NULL pv$restart_strategy = NULL wrapper = function(x, fun, constants, direction) { @@ -70,8 +71,8 @@ AcqOptimizerLbfgsb = R6Class("AcqOptimizerLbfgsb", y = Inf n = 0L i = 0L - maxeval_i = ceiling(pv$maxeval / min_iterations) - while (n < pv$maxeval) { + maxeval = if (pv$maxeval == -1) Inf else pv$maxeval + while (n < maxeval && i <= max_restarts) { i = i + 1L x0 = if (i == 1L) { @@ -91,7 +92,7 @@ AcqOptimizerLbfgsb = R6Class("AcqOptimizerLbfgsb", eval_f = wrapper, lb = self$acq_function$domain$lower + saveguard_epsilon, ub = self$acq_function$domain$upper - saveguard_epsilon, - opts = insert_named(pv, list(algorithm = "NLOPT_LD_LBFGS", maxeval = min(maxeval_i, pv$maxeval - n))), + opts = insert_named(pv, list(algorithm = "NLOPT_LD_LBFGS", maxeval = maxeval - n)), eval_grad_f = eval_grad_f, x0 = x0, fun = fun, diff --git a/tests/testthat/test_AcqOptimizerLbfgsb.R b/tests/testthat/test_AcqOptimizerLbfgsb.R index 9a1efbd4..55edcc7e 100644 --- a/tests/testthat/test_AcqOptimizerLbfgsb.R +++ b/tests/testthat/test_AcqOptimizerLbfgsb.R @@ -58,7 +58,7 @@ test_that("AcqOptimizerLbfgsb works with random restart", { surrogate = srlrn(REGR_KM_DETERM, archive = instance$archive) acqfun = acqf("ei", surrogate = surrogate) acqopt = AcqOptimizerLbfgsb$new(acq_function = acqfun) - acqopt$param_set$set_values(maxeval = 200L, restart_strategy = "random", n_restarts = 3L) + acqopt$param_set$set_values(maxeval = -1, ftol_rel = 1e-6, restart_strategy = "random", max_restarts = 3L) acqfun$surrogate$update() acqfun$update() From 083e3812af7b64de9b76905cd27b5d5f275cb89a Mon Sep 17 00:00:00 2001 From: be-marc Date: Thu, 2 Oct 2025 17:59:02 +0200 Subject: [PATCH 121/126] ... --- R/AcqOptimizerCmaes.R | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/R/AcqOptimizerCmaes.R b/R/AcqOptimizerCmaes.R index 5fb2b886..94a4c590 100644 --- a/R/AcqOptimizerCmaes.R +++ b/R/AcqOptimizerCmaes.R @@ -16,7 +16,8 @@ AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", initialize = function(acq_function = NULL) { self$acq_function = assert_r6(acq_function, "AcqFunction", null.ok = TRUE) param_set = ps( - max_fevals = p_int(lower = 1, init = 1000L) + max_fevals = p_int(lower = 1, init = 1000L), + max_restarts = p_int(lower = 0, init = 1000L) # maxeval = p_int(lower = 1, init = 1000L, special_vals = list(-1)), # fnscale = p_dbl(default = 1), # #maxit = p_int(lower = 1L), @@ -67,7 +68,8 @@ AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", control = libcmaesr::cmaes_control( maximize = direction == -1L, algo = "abipop", - max_fevals = pv$max_fevals + max_fevals = pv$max_fevals, + max_restarts = pv$max_restarts ) wrapper = function(xmat) { From 178b64a207eb2ea304bf364271b6aad9dcc54352 Mon Sep 17 00:00:00 2001 From: be-marc Date: Fri, 10 Oct 2025 08:11:45 +0200 Subject: [PATCH 122/126] ... --- .lintr | 2 +- R/AcqOptimizerCmaes.R | 175 ++++++++++++++++++++--------------------- R/AcqOptimizerDirect.R | 140 +++++++++++++++++++-------------- 3 files changed, 167 insertions(+), 150 deletions(-) diff --git a/.lintr b/.lintr index 6fa3c0ff..49971ef5 100644 --- a/.lintr +++ b/.lintr @@ -5,6 +5,6 @@ linters: linters_with_defaults( object_name_linter = object_name_linter(c("snake_case", "CamelCase")), # only allow snake case and camel case object names cyclocomp_linter = NULL, # do not check function complexity commented_code_linter = NULL, # allow code in comments - line_length_linter = line_length_linter(120), + line_length_linter = line_length_linter(200), indentation_linter(indent = 2L, hanging_indent_style = "never") ) diff --git a/R/AcqOptimizerCmaes.R b/R/AcqOptimizerCmaes.R index 94a4c590..37d5a8af 100644 --- a/R/AcqOptimizerCmaes.R +++ b/R/AcqOptimizerCmaes.R @@ -1,14 +1,40 @@ #' @title CMA-ES Acquisition Function Optimizer #' +#' @include AcqOptimizer.R +#' +#' @description +#' CMA-ES acquisition function optimizer. +#' Calls `cmaes()` from \CRANpkg{libcmaesr}. +#' The default algorithm is `"abipop"` with unlimited restarts and a budget of `100 * D^2` function evaluations, where `D` is the dimension of the search space. +#' For the meaning of the control parameters, see `libcmaesr::cmaes_control()`. +#' +#' @section Termination Parameters: +#' The following termination parameters can be used. +#' +#' \describe{ +#' \item{`max_fevals`}{`integer(1)`\cr +#' Maximum number of function evaluations. +#' Deactivate with `NA`. +#' Default is `100 * D^2`, where `D` is the dimension of the search space.} +#' \item{`max_iter`}{`integer(1)`\cr +#' Maximum number of iterations. +#' Deactivate with `NA`.} +#' \item{`ftarget`}{`numeric(1)`\cr +#' Target function value. +#' Deactivate with `NA`.} +#' \item{`f_tolerance`}{`numeric(1)`\cr +#' Function tolerance. +#' Deactivate with `NA`.} +#' \item{`x_tolerance`}{`numeric(1)`\cr +#' Parameter tolerance. +#' Deactivate with `NA`.} +#' } +#' #' @export AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", inherit = AcqOptimizer, public = list( - #' @field state (`list()`)\cr - #' [libcmaesr::cmaes()] results. - state = NULL, - #' @description #' Creates a new instance of this [R6][R6::R6Class] class. #' @@ -16,41 +42,37 @@ AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", initialize = function(acq_function = NULL) { self$acq_function = assert_r6(acq_function, "AcqFunction", null.ok = TRUE) param_set = ps( - max_fevals = p_int(lower = 1, init = 1000L), - max_restarts = p_int(lower = 0, init = 1000L) - # maxeval = p_int(lower = 1, init = 1000L, special_vals = list(-1)), - # fnscale = p_dbl(default = 1), - # #maxit = p_int(lower = 1L), - # stopfitness = p_dbl(default = -Inf), - # keep.best = p_lgl(default = TRUE), - # sigma = p_uty(default = 0.5), - # mu = p_int(lower = 1L), - # lambda = p_int(lower = 1L), - # weights = p_uty(), - # damps = p_dbl(), - # cs = p_dbl(), - # ccum = p_dbl(), - # ccov.1 = p_dbl(lower = 0), - # ccov.mu = p_dbl(lower = 0), - # diag.sigma = p_lgl(default = FALSE), - # diag.eigen = p_lgl(default = FALSE), - # diag.pop = p_lgl(default = FALSE), - # diag.value = p_lgl(default = FALSE), - # stop.tolx = p_dbl(), # undocumented stop criterion - # restart_strategy = p_fct(levels = c("none", "ipop"), init = "none"), - # n_restarts = p_int(lower = 0L, init = 0L), - # population_multiplier = p_int(lower = 1, init = 2L) - # start_values = p_fct(default = "random", levels = c("random", "center", "custom")), - # start = p_uty(default = NULL, depends = start_values == "custom"), - # n_candidates = p_int(lower = 1, default = 1L), - # logging_level = p_fct(levels = c("fatal", "error", "warn", "info", "debug", "trace"), default = "warn"), - # warmstart = p_lgl(default = FALSE), - # warmstart_size = p_int(lower = 1L, special_vals = list("all")), - # skip_already_evaluated = p_lgl(default = TRUE), - # catch_errors = p_lgl(default = TRUE) + algo = p_fct(init = "abipop", levels = c( + "cmaes", + "ipop", + "bipop", + "acmaes", + "aipop", + "abipop", + "sepcmaes", + "sepipop", + "sepbipop", + "sepacmaes", + "sepaipop", + "sepabipop", + "vdcma", + "vdipopcma", + "vdbipopcma")), + lambda = p_int(lower = 1L, default = NA_integer_, special_vals = list(NA_integer_)), + sigma = p_dbl(default = NA_real_, special_vals = list(NA_real_)), + max_restarts = p_int(lower = 1L, special_vals = list(NA), init = 1e5L), + tpa = p_int(default = NA_integer_, special_vals = list(NA_integer_)), + tpa_dsigma = p_dbl(default = NA_real_, special_vals = list(NA_real_)), + seed = p_int(default = NA_integer_, special_vals = list(NA_integer_)), + quiet = p_lgl(default = FALSE), + # internal termination criteria + max_fevals = p_int(lower = 1L, special_vals = list(NA_integer_)), + max_iter = p_int(lower = 1L, default = NA_integer_, special_vals = list(NA_integer_)), + ftarget = p_dbl(default = NA_real_, special_vals = list(NA_real_)), + f_tolerance = p_dbl(default = NA_real_, special_vals = list(NA_real_)), + x_tolerance = p_dbl(default = NA_real_, special_vals = list(NA_real_)), + catch_errors = p_lgl(init = TRUE) ) - # ps$values = list(n_candidates = 1, logging_level = "warn", warmstart = FALSE, skip_already_evaluated = TRUE, catch_errors = TRUE) - # ps$add_dep("warmstart_size", on = "warmstart", cond = CondEqual$new(TRUE)) private$.param_set = param_set }, @@ -61,16 +83,15 @@ AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", optimize = function() { pv = self$param_set$values + if (is.null(pv$max_fevals)) { + pv$max_fevals = 100 * self$acq_function$domain$length^2 + } + fun = get_private(self$acq_function)$.fun constants = self$acq_function$constants$values direction = self$acq_function$codomain$direction - control = libcmaesr::cmaes_control( - maximize = direction == -1L, - algo = "abipop", - max_fevals = pv$max_fevals, - max_restarts = pv$max_restarts - ) + control = invoke(libcmaesr::cmaes_control, maximize = direction == -1L, .args = pv) wrapper = function(xmat) { xdt = set_names(as.data.table(xmat), self$acq_function$domain$ids()) @@ -81,59 +102,35 @@ AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", upper = self$acq_function$domain$upper x0 = as.numeric(self$acq_function$archive$best()[, self$acq_function$domain$ids(), with = FALSE]) - res = libcmaesr::cmaes( - objective = wrapper, - x0 = x0, - lower = lower, - upper = upper, - batch = TRUE, - control = control) - - self$state = res + optimize = function() { + libcmaesr::cmaes( + objective = wrapper, + x0 = x0, + lower = lower, + upper = upper, + batch = TRUE, + control = control) + } + if (pv$catch_errors) { + tryCatch({ + res = optimize() + }, error = function(error_condition) { + lg$warn(error_condition$message) + stop(set_class(list(message = error_condition$message, call = NULL), classes = c("acq_optimizer_error", "mbo_error", "error", "condition"))) + }) + } else { + res = optimize() + } as.data.table(as.list(set_names(c(res$x, res$y * direction), c(self$acq_function$domain$ids(), self$acq_function$codomain$ids())))) - }, - - #' @description - #' Reset the acquisition function optimizer. - #' - #' Currently not used. - reset = function() { - } ), active = list( #' @template field_print_id print_id = function(rhs) { - if (missing(rhs)) { - paste0("(", class(self$optimizer)[1L], " | ", class(self$terminator)[1L], ")") - } else { - stop("$print_id is read-only.") - } - }, - - #' @field param_set ([paradox::ParamSet])\cr - #' Set of hyperparameters. - param_set = function(rhs) { - if (!missing(rhs) && !identical(rhs, private$.param_set)) { - stop("$param_set is read-only.") - } - private$.param_set - } - ), - - private = list( - .param_set = NULL, - - deep_clone = function(name, value) { - switch(name, - optimizer = value$clone(deep = TRUE), - terminator = value$clone(deep = TRUE), - acq_function = if (!is.null(value)) value$clone(deep = TRUE) else NULL, - .param_set = value$clone(deep = TRUE), - value - ) + assert_ro_binding(rhs) + "(OptimizerBatchCmaes)" } ) ) diff --git a/R/AcqOptimizerDirect.R b/R/AcqOptimizerDirect.R index e701c217..8f6256a3 100644 --- a/R/AcqOptimizerDirect.R +++ b/R/AcqOptimizerDirect.R @@ -1,13 +1,62 @@ -#' @title Direct Optimization Acquisition Function Optimizer +#' @title Direct Acquisition Function Optimizer +#' +#' @include AcqOptimizer.R #' #' @description +#' Direct acquisition function optimizer. +#' Calls `nloptr()` from \CRANpkg{nloptr}. +#' In its default setting, the algorithm restarts `5 * D` times and runs at most for `100 * D^2` function evaluations, where `D` is the dimension of the search space. +#' Each run stops when the relative tolerance of the parameters is less than `10^-4`. +#' The first iteration starts with the best point in the archive and the next iterations start from a random point. +#' +#' @section Parameters: +#' \describe{ +#' \item{`restart_strategy`}{`character(1)`\cr +#' Restart strategy. +#' Can be `"none"` or `"random"`. +#' Default is `"none"`. +#' } +#' \item{`max_restarts`}{`integer(1)`\cr +#' Maximum number of restarts. +#' Default is `5 * D` (Default).} +#' +#' @note #' If the restart strategy is `"none"`, the optimizer starts with the best point in the archive. #' The optimization stops when one of the stopping criteria is met. #' -#' If `restart_strategy` is `"random"`, the optimizer runs at least for `maxeval / n_restarts` iterations. -#' The first iteration starts with the best point in the archive. +#' If `restart_strategy` is `"random"`, the optimizer runs at least for `maxeval` iterations. +#' The first iteration starts with the best point in the archive and stops when one of the stopping criteria is met. #' The next iterations start from a random point. #' +#' @section Termination Parameters: +#' The following termination parameters can be used. +#' +#' \describe{ +#' \item{`stopval`}{`numeric(1)`\cr +#' Stop value. +#' Deactivate with `-Inf` (Default).} +#' \item{`maxtime`}{`integer(1)`\cr +#' Maximum time. +#' Deactivate with `-1L` (Default).} +#' \item{`maxeval`}{`integer(1)`\cr +#' Maximum number of evaluations. +#' Default is `100 * D^2`, where `D` is the dimension of the search space. +#' Deactivate with `-1L`.} +#' \item{`xtol_rel`}{`numeric(1)`\cr +#' Relative tolerance of the parameters. +#' Default is `10^-4`. +#' Deactivate with `-1`.} +#' \item{`xtol_abs`}{`numeric(1)`\cr +#' Absolute tolerance of the parameters. +#' Deactivate with `-1` (Default).} +#' \item{`ftol_rel`}{`numeric(1)`\cr +#' Relative tolerance of the objective function. +#' Deactivate with `-1`. (Default).} +#' \item{`ftol_abs`}{`numeric(1)`\cr +#' Absolute tolerance of the objective function. +#' Deactivate with `-1` (Default).} +#' } +#' #' @export AcqOptimizerDirect = R6Class("AcqOptimizerDirect", inherit = AcqOptimizer, @@ -31,17 +80,10 @@ AcqOptimizerDirect = R6Class("AcqOptimizerDirect", ftol_rel = p_dbl(default = 0, lower = 0, upper = Inf, special_vals = list(-1)), ftol_abs = p_dbl(default = 0, lower = 0, upper = Inf, special_vals = list(-1)), minf_max = p_dbl(default = -Inf), - restart_strategy = p_fct(levels = c("none", "random"), init = "none"), - max_restarts = p_int(lower = 0L, init = 0L) - # n_candidates = p_int(lower = 1, default = 1L), - # logging_level = p_fct(levels = c("fatal", "error", "warn", "info", "debug", "trace"), default = "warn"), - # warmstart = p_lgl(default = FALSE), - # warmstart_size = p_int(lower = 1L, special_vals = list("all")), - # skip_already_evaluated = p_lgl(default = TRUE), - # catch_errors = p_lgl(default = TRUE) + restart_strategy = p_fct(levels = c("none", "random"), init = "random"), + max_restarts = p_int(lower = 0L), + catch_errors = p_lgl(init = TRUE) ) - # ps$values = list(n_candidates = 1, logging_level = "warn", warmstart = FALSE, skip_already_evaluated = TRUE, catch_errors = TRUE) - # ps$add_dep("warmstart_size", on = "warmstart", cond = CondEqual$new(TRUE)) private$.param_set = param_set }, @@ -80,17 +122,29 @@ AcqOptimizerDirect = R6Class("AcqOptimizerDirect", as.numeric(generate_design_random(self$acq_function$domain, n = 1)$data) } - # optimize with nloptr - res = invoke(nloptr::nloptr, - eval_f = wrapper, - lb = self$acq_function$domain$lower, - ub = self$acq_function$domain$upper, - opts = insert_named(pv, list(algorithm = "NLOPT_GN_DIRECT_L", maxeval = maxeval - n)), - eval_grad_f = NULL, - x0 = x0, - fun = fun, - constants = constants, - direction = direction) + optimize = function() { + invoke(nloptr::nloptr, + eval_f = wrapper, + lb = self$acq_function$domain$lower, + ub = self$acq_function$domain$upper, + opts = insert_named(pv, list(algorithm = "NLOPT_GN_DIRECT_L", maxeval = maxeval - n)), + eval_grad_f = NULL, + x0 = x0, + fun = fun, + constants = constants, + direction = direction) + } + + if (pv$catch_errors) { + tryCatch({ + res = optimize() + }, error = function(error_condition) { + lg$warn(error_condition$message) + stop(set_class(list(message = error_condition$message, call = NULL), classes = c("acq_optimizer_error", "mbo_error", "error", "condition"))) + }) + } else { + res = optimize() + } if (res$objective < y) { y = res$objective @@ -104,48 +158,14 @@ AcqOptimizerDirect = R6Class("AcqOptimizerDirect", if (restart_strategy == "none") break } as.data.table(as.list(set_names(c(x, y * direction), c(self$acq_function$domain$ids(), self$acq_function$codomain$ids())))) - }, - - #' @description - #' Reset the acquisition function optimizer. - #' - #' Currently not used. - reset = function() { - } ), active = list( #' @template field_print_id print_id = function(rhs) { - if (missing(rhs)) { - paste0("(", class(self$optimizer)[1L], " | ", class(self$terminator)[1L], ")") - } else { - stop("$print_id is read-only.") - } - }, - - #' @field param_set ([paradox::ParamSet])\cr - #' Set of hyperparameters. - param_set = function(rhs) { - if (!missing(rhs) && !identical(rhs, private$.param_set)) { - stop("$param_set is read-only.") - } - private$.param_set - } - ), - - private = list( - .param_set = NULL, - - deep_clone = function(name, value) { - switch(name, - optimizer = value$clone(deep = TRUE), - terminator = value$clone(deep = TRUE), - acq_function = if (!is.null(value)) value$clone(deep = TRUE) else NULL, - .param_set = value$clone(deep = TRUE), - value - ) + assert_ro_binding(rhs) + "(OptimizerDirect)" } ) ) From ef29af7f7061fd016c1145705b14b10392aaa8df Mon Sep 17 00:00:00 2001 From: be-marc Date: Thu, 16 Oct 2025 16:16:51 +0200 Subject: [PATCH 123/126] ... --- R/AcqOptimizerCmaes.R | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/R/AcqOptimizerCmaes.R b/R/AcqOptimizerCmaes.R index 37d5a8af..592465c0 100644 --- a/R/AcqOptimizerCmaes.R +++ b/R/AcqOptimizerCmaes.R @@ -102,6 +102,11 @@ AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", upper = self$acq_function$domain$upper x0 = as.numeric(self$acq_function$archive$best()[, self$acq_function$domain$ids(), with = FALSE]) + # add saveguard_epsilon to x0 + saveguard_epsilon = 1e-5 + x0[x0 < lower] = x0[x0 < lower] + saveguard_epsilon + x0[x0 > upper] = x0[x0 > upper] - saveguard_epsilon + optimize = function() { libcmaesr::cmaes( objective = wrapper, From dfe2c9fba0831d3c36d3f87a51a02981a2cd9b77 Mon Sep 17 00:00:00 2001 From: be-marc Date: Fri, 17 Oct 2025 21:05:50 +0200 Subject: [PATCH 124/126] ... --- R/AcqOptimizerCmaes.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/AcqOptimizerCmaes.R b/R/AcqOptimizerCmaes.R index 592465c0..a179dd85 100644 --- a/R/AcqOptimizerCmaes.R +++ b/R/AcqOptimizerCmaes.R @@ -91,7 +91,7 @@ AcqOptimizerCmaes = R6Class("AcqOptimizerCmaes", constants = self$acq_function$constants$values direction = self$acq_function$codomain$direction - control = invoke(libcmaesr::cmaes_control, maximize = direction == -1L, .args = pv) + control = invoke(libcmaesr::cmaes_control, maximize = direction == -1L, .args = pv[names(pv) %in% formalArgs(libcmaesr::cmaes_control)]) wrapper = function(xmat) { xdt = set_names(as.data.table(xmat), self$acq_function$domain$ids()) From 6a6832f286a746d371fb0ae8376b4b358afa231c Mon Sep 17 00:00:00 2001 From: be-marc Date: Sat, 18 Oct 2025 13:28:50 +0200 Subject: [PATCH 125/126] ... --- R/AcqOptimizerLocalSearch.R | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/R/AcqOptimizerLocalSearch.R b/R/AcqOptimizerLocalSearch.R index 95b7b29b..70911769 100644 --- a/R/AcqOptimizerLocalSearch.R +++ b/R/AcqOptimizerLocalSearch.R @@ -32,16 +32,16 @@ AcqOptimizerLocalSearch = R6Class("AcqOptimizerLocalSearch", optimize = function() { pv = self$param_set$get_values() - local_search_control = invoke(local_search_control, minimize = self$acq_function$direction == "minimize", .args = pv) + control = invoke(bbotk::local_search_control, minimize = self$acq_function$direction == "minimize", .args = pv) wrapper = function(xdt) { mlr3misc::invoke(self$acq_function$fun, xdt = xdt, .args = self$acq_function$constants$values)[[1]] } - res = invoke(local_search, + res = invoke(bbotk::local_search, objective = wrapper, search_space = self$acq_function$domain, - control = local_search_control + control = control ) as.data.table(as.list(set_names(c(res$x, res$y), c(self$acq_function$domain$ids(), self$acq_function$codomain$ids())))) From eaa58329616289fc2cb0759e01c9f75bebb9e5ff Mon Sep 17 00:00:00 2001 From: be-marc Date: Mon, 27 Oct 2025 13:06:34 +0100 Subject: [PATCH 126/126] fix direction --- R/AcqOptimizerLocalSearch.R | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/R/AcqOptimizerLocalSearch.R b/R/AcqOptimizerLocalSearch.R index 70911769..c6610fd6 100644 --- a/R/AcqOptimizerLocalSearch.R +++ b/R/AcqOptimizerLocalSearch.R @@ -31,8 +31,7 @@ AcqOptimizerLocalSearch = R6Class("AcqOptimizerLocalSearch", #' @return [data.table::data.table()] with 1 row per candidate. optimize = function() { pv = self$param_set$get_values() - - control = invoke(bbotk::local_search_control, minimize = self$acq_function$direction == "minimize", .args = pv) + control = invoke(bbotk::local_search_control, minimize = self$acq_function$codomain$direction == 1L, .args = pv) wrapper = function(xdt) { mlr3misc::invoke(self$acq_function$fun, xdt = xdt, .args = self$acq_function$constants$values)[[1]]