diff --git a/DESCRIPTION b/DESCRIPTION index 776a0547..3aaaf647 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -42,6 +42,8 @@ Suggests: mlr3pipelines, rpart, testthat (>= 3.0.0) +Remotes: + mlr-org/bbotk Config/testthat/edition: 3 Config/testthat/parallel: true Encoding: UTF-8 diff --git a/NEWS.md b/NEWS.md index 8c76f052..4e6cd6d4 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,7 @@ # mlr3fselect (development version) +* feat: Add internal tuning callback `mlr3fselect.internal_tuning`. + # mlr3fselect 1.1.1 * compatibility: bbotk 1.1.1 diff --git a/R/AutoFSelector.R b/R/AutoFSelector.R index 538f5059..de08ed9f 100644 --- a/R/AutoFSelector.R +++ b/R/AutoFSelector.R @@ -308,11 +308,18 @@ AutoFSelector = R6Class("AutoFSelector", private = list( .train = function(task) { - # construct instance from args; then tune + # construct instance from args ia = self$instance_args ia$task = task$clone() instance = invoke(FSelectInstanceBatchSingleCrit$new, .args = ia) + + # optimize feature selection self$fselector$optimize(instance) + + # make auto fselector available to callbacks + instance$objective$context$auto_fselector = self + call_back("on_auto_fselector_before_final_model", instance$objective$callbacks, instance$objective$context) + learner = ia$learner$clone(deep = TRUE) task = task$clone() @@ -325,9 +332,12 @@ AutoFSelector = R6Class("AutoFSelector", task$select(feat) learner$train(task) + call_back("on_auto_fselector_after_final_model", instance$objective$callbacks, instance$objective$context) + # the return model is a list of "learner", "features" and "fselect_instance" result_model = list(learner = learner, features = feat) if (private$.store_fselect_instance) result_model$fselect_instance = instance + result_model }, diff --git a/R/CallbackBatchFSelect.R b/R/CallbackBatchFSelect.R index fe9f3a7a..4b15cb02 100644 --- a/R/CallbackBatchFSelect.R +++ b/R/CallbackBatchFSelect.R @@ -31,7 +31,19 @@ CallbackBatchFSelect = R6Class("CallbackBatchFSelect", #' @field on_eval_before_archive (`function()`)\cr #' Stage called before performance values are written to the archive. #' Called in `ObjectiveFSelectBatch$eval_many()`. - on_eval_before_archive = NULL + on_eval_before_archive = NULL, + + #' @field on_auto_fselector_before_final_model (`function()`)\cr + #' Stage called before the final model is trained. + #' Called in `AutoFSelector$train()`. + #' This stage is called after the optimization has finished and the final model is trained with the best feature set found. + on_auto_fselector_before_final_model = NULL, + + #' @field on_auto_fselector_after_final_model (`function()`)\cr + #' Stage called after the final model is trained. + #' Called in `AutoFSelector$train()`. + #' This stage is called after the final model is trained with the best feature set found. + on_auto_fselector_after_final_model = NULL ) ) @@ -43,22 +55,27 @@ CallbackBatchFSelect = R6Class("CallbackBatchFSelect", #' #' Feature selection callbacks can be called from different stages of feature selection. #' The stages are prefixed with `on_*`. +#' The `on_auto_fselector_*` stages are only available when the callback is used in an [AutoFSelector]. #' #' ``` -#' Start Feature Selection -#' - on_optimization_begin -#' Start FSelect Batch -#' - on_optimizer_before_eval -#' Start Evaluation -#' - on_eval_after_design -#' - on_eval_after_benchmark -#' - on_eval_before_archive -#' End Evaluation -#' - on_optimizer_after_eval -#' End FSelect Batch -#' - on_result -#' - on_optimization_end -#' End Feature Selection +#' Start Automatic Feature Selection +#' Start Feature Selection +#' - on_optimization_begin +#' Start FSelect Batch +#' - on_optimizer_before_eval +#' Start Evaluation +#' - on_eval_after_design +#' - on_eval_after_benchmark +#' - on_eval_before_archive +#' End Evaluation +#' - on_optimizer_after_eval +#' End FSelect Batch +#' - on_result +#' - on_optimization_end +#' End Feature Selection +#' - on_auto_fselector_before_final_model +#' - on_auto_fselector_after_final_model +#' End Automatic Feature Selection #' ``` #' #' See also the section on parameters for more information on the stages. @@ -66,19 +83,8 @@ CallbackBatchFSelect = R6Class("CallbackBatchFSelect", #' #' @details #' When implementing a callback, each function must have two arguments named `callback` and `context`. -#' #' A callback can write data to the state (`$state`), e.g. settings that affect the callback itself. #' Avoid writing large data the state. -#' This can slow down the feature selection when the evaluation of configurations is parallelized. -#' -#' Feature selection callbacks access two different contexts depending on the stage. -#' The stages `on_eval_after_design`, `on_eval_after_benchmark`, `on_eval_before_archive` access [ContextBatchFSelect]. -#' This context can be used to customize the evaluation of a batch of feature sets. -#' Changes to the state of callback are lost after the evaluation of a batch and changes to the fselect instance or the fselector are not possible. -#' Persistent data should be written to the archive via `$aggregated_performance` (see [ContextBatchFSelect]). -#' The other stages access [bbotk::ContextBatch]. -#' This context can be used to modify the fselect instance, archive, fselector and final result. -#' There are two different contexts because the evaluation can be parallelized i.e. multiple instances of [ContextBatchFSelect] exists on different workers at the same time. #' #' @param id (`character(1)`)\cr #' Identifier for the new instance. @@ -111,6 +117,12 @@ CallbackBatchFSelect = R6Class("CallbackBatchFSelect", #' @param on_optimization_end (`function()`)\cr #' Stage called at the end of the optimization. #' Called in `Optimizer$optimize()`. +#' @param on_auto_fselector_before_final_model (`function()`)\cr +#' Stage called before the final model is trained. +#' Called in `AutoFSelector$train()`. +#' @param on_auto_fselector_after_final_model (`function()`)\cr +#' Stage called after the final model is trained. +#' Called in `AutoFSelector$train()`. #' #' @export #' @inherit CallbackBatchFSelect examples @@ -125,7 +137,9 @@ callback_batch_fselect = function( on_eval_before_archive = NULL, on_optimizer_after_eval = NULL, on_result = NULL, - on_optimization_end = NULL + on_optimization_end = NULL, + on_auto_fselector_before_final_model = NULL, + on_auto_fselector_after_final_model = NULL ) { stages = discard(set_names(list( on_optimization_begin, @@ -135,7 +149,9 @@ callback_batch_fselect = function( on_eval_before_archive, on_optimizer_after_eval, on_result, - on_optimization_end), + on_optimization_end, + on_auto_fselector_before_final_model, + on_auto_fselector_after_final_model), c( "on_optimization_begin", "on_optimizer_before_eval", @@ -144,7 +160,9 @@ callback_batch_fselect = function( "on_eval_before_archive", "on_optimizer_after_eval", "on_result", - "on_optimization_end")), is.null) + "on_optimization_end", + "on_auto_fselector_before_final_model", + "on_auto_fselector_after_final_model")), is.null) walk(stages, function(stage) assert_function(stage, args = c("callback", "context"))) callback = CallbackBatchFSelect$new(id, label, man) iwalk(stages, function(stage, name) callback[[name]] = stage) diff --git a/R/ContextBatchFSelect.R b/R/ContextBatchFSelect.R index b69b6015..6a81818c 100644 --- a/R/ContextBatchFSelect.R +++ b/R/ContextBatchFSelect.R @@ -14,6 +14,13 @@ #' @export ContextBatchFSelect = R6Class("ContextBatchFSelect", inherit = ContextBatch, + public = list( + + #' @field auto_fselector ([AutoFSelector])\cr + #' The [AutoFSelector] instance. + auto_fselector = NULL + ), + active = list( #' @field xss (list())\cr #' The feature sets of the latest batch. diff --git a/R/FSelectInstanceBatchMultiCrit.R b/R/FSelectInstanceBatchMultiCrit.R index c67abfd3..c36f4db4 100644 --- a/R/FSelectInstanceBatchMultiCrit.R +++ b/R/FSelectInstanceBatchMultiCrit.R @@ -93,16 +93,18 @@ FSelectInstanceBatchMultiCrit = R6Class("FSelectInstanceBatchMultiCrit", #' #' @param ydt (`data.table::data.table()`)\cr #' Optimal outcomes, e.g. the Pareto front. + #' @param extra (`data.table::data.table()`)\cr + #' Additional information. #' @param ... (`any`)\cr #' ignored. - assign_result = function(xdt, ydt, ...) { + assign_result = function(xdt, ydt, extra = NULL, ...) { # Add feature names to result for easy task subsetting features = map(transpose_list(xdt), function(x) { self$objective$task$feature_names[as.logical(x)] }) set(xdt, j = "features", value = list(features)) set(xdt, j = "n_features", value = length(features[[1L]])) - super$assign_result(xdt, ydt) + super$assign_result(xdt, ydt, extra = extra) if (!is.null(private$.result$x_domain)) set(private$.result, j = "x_domain", value = NULL) }, diff --git a/R/FSelectInstanceBatchSingleCrit.R b/R/FSelectInstanceBatchSingleCrit.R index a33572d3..c50e4cbc 100644 --- a/R/FSelectInstanceBatchSingleCrit.R +++ b/R/FSelectInstanceBatchSingleCrit.R @@ -138,15 +138,17 @@ FSelectInstanceBatchSingleCrit = R6Class("FSelectInstanceBatchSingleCrit", #' #' @param y (`numeric(1)`)\cr #' Optimal outcome. + #' @param extra (`data.table::data.table()`)\cr + #' Additional information. #' @param ... (`any`)\cr #' ignored. - assign_result = function(xdt, y, ...) { + assign_result = function(xdt, y, extra = NULL, ...) { # Add feature names to result for easy task subsetting feature_names = self$objective$task$feature_names features = list(feature_names[as.logical(xdt[, feature_names, with = FALSE])]) set(xdt, j = "features", value = list(features)) set(xdt, j = "n_features", value = length(features[[1L]])) - super$assign_result(xdt, y) + super$assign_result(xdt, y, extra = extra) if (!is.null(private$.result$x_domain)) set(private$.result, j = "x_domain", value = NULL) }, diff --git a/R/FSelectorBatchRFE.R b/R/FSelectorBatchRFE.R index acc8dcba..af534823 100644 --- a/R/FSelectorBatchRFE.R +++ b/R/FSelectorBatchRFE.R @@ -157,10 +157,11 @@ FSelectorBatchRFE = R6Class("FSelectorBatchRFE", res = inst$archive$best() xdt = res[, c(inst$search_space$ids(), "importance"), with = FALSE] + extra = res[, !c(inst$search_space$ids(), "importance"), with = FALSE] # unlist keeps name! y = unlist(res[, inst$archive$cols_y, with = FALSE]) - inst$assign_result(xdt, y) + inst$assign_result(xdt, y, extra = extra) invisible(NULL) } diff --git a/R/mlr_callbacks.R b/R/mlr_callbacks.R index 64c41141..9c5cac29 100644 --- a/R/mlr_callbacks.R +++ b/R/mlr_callbacks.R @@ -176,3 +176,55 @@ load_callback_one_se_rule = function() { } ) } + +#' @title Internal Tuning Callback +#' +#' @include CallbackBatchFSelect.R +#' @name mlr3fselect.internal_tuning +#' +#' @description +#' This callback runs internal tuning alongside the feature selection. +#' The internal tuning values are aggregated and stored in the results. +#' The final model is trained with the best feature set and the tuned value. +#' +#' @examples +#' clbk("mlr3fselect.internal_tuning") +NULL + +load_callback_internal_tuning = function() { + callback_batch_fselect("mlr3fselect.internal_tuning", + label = "Internal Tuning", + man = "mlr3fselect::mlr3fselect.internal_tuning", + + on_eval_before_archive = function(callback, context) { + # extract internal tuned values and aggregate folds + internal_tuned_values = mlr3misc::map(context$benchmark_result$resample_results$resample_result, function(resample_result) { + internal_tuned_values = mlr3misc::transpose_list(mlr3misc::map(mlr3misc::get_private(resample_result)$.data$learner_states(mlr3misc::get_private(resample_result)$.view), "internal_tuned_values")) + callback$state$internal_search_space$aggr_internal_tuned_values(internal_tuned_values) + }) + + data.table::set(context$aggregated_performance, j = "internal_tuned_values", value = list(internal_tuned_values)) + }, + + on_optimization_end = function(callback, context) { + # save internal tuned values to results + set(context$result, j = "internal_tuned_values", value = list(context$result_extra[["internal_tuned_values"]])) + }, + + on_auto_fselector_before_final_model = function(callback, context) { + # copy original learner + callback$state$learner = context$auto_fselector$instance_args$learner$clone(deep = TRUE) + + # deactivate internal tuning and set tuned values + learner = context$auto_fselector$instance_args$learner + learner$param_set$disable_internal_tuning(callback$state$internal_search_space$ids()) + learner$param_set$set_values(.values = context$result$internal_tuned_values[[1]]) + set_validate(learner, validate = NULL) + }, + + on_auto_fselector_after_final_model = function(callback, context) { + # restore original learner + context$auto_fselector$instance_args$learner = callback$state$learner + } + ) +} diff --git a/R/zzz.R b/R/zzz.R index a87fdc01..a421b72d 100644 --- a/R/zzz.R +++ b/R/zzz.R @@ -27,6 +27,7 @@ x$add("mlr3fselect.backup", load_callback_backup) x$add("mlr3fselect.svm_rfe", load_callback_svm_rfe) x$add("mlr3fselect.one_se_rule", load_callback_one_se_rule) + x$add("mlr3fselect.internal_tuning", load_callback_internal_tuning) assign("lg", lgr::get_logger("bbotk"), envir = parent.env(environment())) if (Sys.getenv("IN_PKGDOWN") == "true") { diff --git a/man/CallbackBatchFSelect.Rd b/man/CallbackBatchFSelect.Rd index 378444f3..e6cf38d9 100644 --- a/man/CallbackBatchFSelect.Rd +++ b/man/CallbackBatchFSelect.Rd @@ -35,6 +35,16 @@ Called in \code{ObjectiveFSelectBatch$eval_many()}.} \item{\code{on_eval_before_archive}}{(\verb{function()})\cr Stage called before performance values are written to the archive. Called in \code{ObjectiveFSelectBatch$eval_many()}.} + +\item{\code{on_auto_fselector_before_final_model}}{(\verb{function()})\cr +Stage called before the final model is trained. +Called in \code{AutoFSelector$train()}. +This stage is called after the optimization has finished and the final model is trained with the best feature set found.} + +\item{\code{on_auto_fselector_after_final_model}}{(\verb{function()})\cr +Stage called after the final model is trained. +Called in \code{AutoFSelector$train()}. +This stage is called after the final model is trained with the best feature set found.} } \if{html}{\out{}} } diff --git a/man/ContextBatchFSelect.Rd b/man/ContextBatchFSelect.Rd index 9bc20d3b..a4f07cc8 100644 --- a/man/ContextBatchFSelect.Rd +++ b/man/ContextBatchFSelect.Rd @@ -17,6 +17,14 @@ Any number of columns can be added. \section{Super classes}{ \code{\link[mlr3misc:Context]{mlr3misc::Context}} -> \code{\link[bbotk:ContextBatch]{bbotk::ContextBatch}} -> \code{ContextBatchFSelect} } +\section{Public fields}{ +\if{html}{\out{
}} +\describe{ +\item{\code{auto_fselector}}{(\link{AutoFSelector})\cr +The \link{AutoFSelector} instance.} +} +\if{html}{\out{
}} +} \section{Active bindings}{ \if{html}{\out{
}} \describe{ diff --git a/man/FSelectInstanceBatchMultiCrit.Rd b/man/FSelectInstanceBatchMultiCrit.Rd index e584f81b..67761d04 100644 --- a/man/FSelectInstanceBatchMultiCrit.Rd +++ b/man/FSelectInstanceBatchMultiCrit.Rd @@ -151,7 +151,7 @@ List of callbacks.} The \link{FSelector} object writes the best found feature subsets and estimated performance values here. For internal use. \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{FSelectInstanceBatchMultiCrit$assign_result(xdt, ydt, ...)}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{FSelectInstanceBatchMultiCrit$assign_result(xdt, ydt, extra = NULL, ...)}\if{html}{\out{
}} } \subsection{Arguments}{ @@ -165,6 +165,9 @@ additional columns for extra information.} \item{\code{ydt}}{(\code{data.table::data.table()})\cr Optimal outcomes, e.g. the Pareto front.} +\item{\code{extra}}{(\code{data.table::data.table()})\cr +Additional information.} + \item{\code{...}}{(\code{any})\cr ignored.} } diff --git a/man/FSelectInstanceBatchSingleCrit.Rd b/man/FSelectInstanceBatchSingleCrit.Rd index 83091870..e221a4ca 100644 --- a/man/FSelectInstanceBatchSingleCrit.Rd +++ b/man/FSelectInstanceBatchSingleCrit.Rd @@ -193,7 +193,7 @@ Ignored if multiple measures are used.} The \link{FSelector} writes the best found feature subset and estimated performance value here. For internal use. \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{FSelectInstanceBatchSingleCrit$assign_result(xdt, y, ...)}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{FSelectInstanceBatchSingleCrit$assign_result(xdt, y, extra = NULL, ...)}\if{html}{\out{
}} } \subsection{Arguments}{ @@ -207,6 +207,9 @@ additional columns for extra information.} \item{\code{y}}{(\code{numeric(1)})\cr Optimal outcome.} +\item{\code{extra}}{(\code{data.table::data.table()})\cr +Additional information.} + \item{\code{...}}{(\code{any})\cr ignored.} } diff --git a/man/callback_batch_fselect.Rd b/man/callback_batch_fselect.Rd index 6e6d7527..488d4027 100644 --- a/man/callback_batch_fselect.Rd +++ b/man/callback_batch_fselect.Rd @@ -15,7 +15,9 @@ callback_batch_fselect( on_eval_before_archive = NULL, on_optimizer_after_eval = NULL, on_result = NULL, - on_optimization_end = NULL + on_optimization_end = NULL, + on_auto_fselector_before_final_model = NULL, + on_auto_fselector_after_final_model = NULL ) } \arguments{ @@ -60,6 +62,14 @@ Called in \code{OptimInstance$assign_result()}.} \item{on_optimization_end}{(\verb{function()})\cr Stage called at the end of the optimization. Called in \code{Optimizer$optimize()}.} + +\item{on_auto_fselector_before_final_model}{(\verb{function()})\cr +Stage called before the final model is trained. +Called in \code{AutoFSelector$train()}.} + +\item{on_auto_fselector_after_final_model}{(\verb{function()})\cr +Stage called after the final model is trained. +Called in \code{AutoFSelector$train()}.} } \description{ Function to create a \link{CallbackBatchFSelect}. @@ -67,21 +77,26 @@ Predefined callbacks are stored in the \link[mlr3misc:Dictionary]{dictionary} \l Feature selection callbacks can be called from different stages of feature selection. The stages are prefixed with \verb{on_*}. +The \verb{on_auto_fselector_*} stages are only available when the callback is used in an \link{AutoFSelector}. -\if{html}{\out{
}}\preformatted{Start Feature Selection - - on_optimization_begin - Start FSelect Batch - - on_optimizer_before_eval - Start Evaluation - - on_eval_after_design - - on_eval_after_benchmark - - on_eval_before_archive - End Evaluation - - on_optimizer_after_eval - End FSelect Batch - - on_result - - on_optimization_end -End Feature Selection +\if{html}{\out{
}}\preformatted{Start Automatic Feature Selection + Start Feature Selection + - on_optimization_begin + Start FSelect Batch + - on_optimizer_before_eval + Start Evaluation + - on_eval_after_design + - on_eval_after_benchmark + - on_eval_before_archive + End Evaluation + - on_optimizer_after_eval + End FSelect Batch + - on_result + - on_optimization_end + End Feature Selection + - on_auto_fselector_before_final_model + - on_auto_fselector_after_final_model +End Automatic Feature Selection }\if{html}{\out{
}} See also the section on parameters for more information on the stages. @@ -89,19 +104,8 @@ A feature selection callback works with \link[bbotk:ContextBatch]{bbotk::Context } \details{ When implementing a callback, each function must have two arguments named \code{callback} and \code{context}. - A callback can write data to the state (\verb{$state}), e.g. settings that affect the callback itself. Avoid writing large data the state. -This can slow down the feature selection when the evaluation of configurations is parallelized. - -Feature selection callbacks access two different contexts depending on the stage. -The stages \code{on_eval_after_design}, \code{on_eval_after_benchmark}, \code{on_eval_before_archive} access \link{ContextBatchFSelect}. -This context can be used to customize the evaluation of a batch of feature sets. -Changes to the state of callback are lost after the evaluation of a batch and changes to the fselect instance or the fselector are not possible. -Persistent data should be written to the archive via \verb{$aggregated_performance} (see \link{ContextBatchFSelect}). -The other stages access \link[bbotk:ContextBatch]{bbotk::ContextBatch}. -This context can be used to modify the fselect instance, archive, fselector and final result. -There are two different contexts because the evaluation can be parallelized i.e. multiple instances of \link{ContextBatchFSelect} exists on different workers at the same time. } \examples{ # Write archive to disk diff --git a/man/mlr3fselect.internal_tuning.Rd b/man/mlr3fselect.internal_tuning.Rd new file mode 100644 index 00000000..bc9e81b3 --- /dev/null +++ b/man/mlr3fselect.internal_tuning.Rd @@ -0,0 +1,13 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/mlr_callbacks.R +\name{mlr3fselect.internal_tuning} +\alias{mlr3fselect.internal_tuning} +\title{Internal Tuning Callback} +\description{ +This callback runs internal tuning alongside the feature selection. +The internal tuning values are aggregated and stored in the results. +The final model is trained with the best feature set and the tuned value. +} +\examples{ +clbk("mlr3fselect.internal_tuning") +} diff --git a/tests/testthat/test_mlr_callbacks.R b/tests/testthat/test_mlr_callbacks.R index 8635caa8..4016787a 100644 --- a/tests/testthat/test_mlr_callbacks.R +++ b/tests/testthat/test_mlr_callbacks.R @@ -54,3 +54,49 @@ test_that("one_se_rule callback works", { expect_equal(instance$result_feature_set, c("x1", "x2", "x3")) }) + +test_that("internal tuning callback works", { + learner = lrn("classif.debug", validate = "test", early_stopping = TRUE) + + internal_search_space = ps( + iter = p_int(upper = 500, aggr = function(x) 233) + ) + + instance = fselect( + fselector = fs("random_search"), + task = tsk("pima"), + learner = learner, + resampling = rsmp("cv", folds = 3), + measures = msr("classif.ce"), + term_evals = 10, + callbacks = clbk("mlr3fselect.internal_tuning", internal_search_space = internal_search_space) + ) + + expect_data_table(instance$result) + expect_names(names(instance$result), must.include = "internal_tuned_values") + expect_equal(instance$result$internal_tuned_values[[1]], list(iter = 233)) +}) + +test_that("internal tuning callback works with AutoFSelector", { + learner = lrn("classif.debug", validate = "test", early_stopping = TRUE) + + internal_search_space = ps( + iter = p_int(upper = 500, aggr = function(x) 233) + ) + + afs = auto_fselector( + fselector = fs("random_search"), + learner = learner, + resampling = rsmp("cv", folds = 3), + measure = msr("classif.ce"), + term_evals = 10, + callbacks = clbk("mlr3fselect.internal_tuning", internal_search_space = internal_search_space) + ) + + afs$train(tsk("pima")) + + expect_data_table(afs$fselect_instance$result) + expect_names(names(afs$fselect_instance$result), must.include = "internal_tuned_values") + expect_equal(afs$fselect_instance$result$internal_tuned_values[[1]], list(iter = 233)) + expect_equal(afs$model$learner$param_set$values$iter, 233) +})