Measure Callback

This CallbackTuning scores the hyperparameter configurations on additional measures while tuning. Usually, the configurations can be scored on additional measures after tuning (see ArchiveTuning). However, if the memory is not sufficient to store the mlr3::BenchmarkResult, it is necessary to score the additional measures while tuning. The measures are not taken into account by the tuner.

Examples

clbk("mlr3tuning.measures")
#> <CallbackTuning:mlr3tuning.measures>: Additional Measures Callback
#> * Active Stages: on_eval_before_archive, on_optimization_begin

# additionally score the configurations on the accuracy measure
instance = tune(
  tuner = tnr("random_search", batch_size = 2),
  task = tsk("pima"),
  learner = lrn("classif.rpart", cp = to_tune(1e-04, 1e-1, logscale = TRUE)),
  resampling = rsmp("cv", folds = 3),
  measures = msr("classif.ce"),
  term_evals = 4,
  callbacks = clbk("mlr3tuning.measures", measures = msr("classif.acc"))
)

# score the configurations on the holdout set
task = tsk("pima")
splits = partition(task, ratio = 0.8)
task$row_roles$use = splits$train
task$row_roles$holdout = splits$test

learner = lrn("classif.rpart", cp = to_tune(1e-04, 1e-1, logscale = TRUE))
learner$predict_sets = c("test", "holdout")

instance = tune(
  tuner = tnr("random_search", batch_size = 2),
  task = task,
  learner = learner,
  resampling = rsmp("cv", folds = 3),
  measures = msr("classif.ce"),
  term_evals = 4,
  callbacks = clbk("mlr3tuning.measures", measures = msr("classif.ce",
    predict_sets = "holdout", id = "classif.ce_holdout"))
)