Function to conduct nested resampling.

tune_nested(
  method,
  task,
  learner,
  inner_resampling,
  outer_resampling,
  measure,
  term_evals = NULL,
  term_time = NULL,
  search_space = NULL,
  ...
)

Arguments

method

(character(1))
Key to retrieve tuner from mlr_tuners dictionary.

task

(mlr3::Task)
Task to operate on.

learner

(mlr3::Learner)
Learner to tune.

inner_resampling

(mlr3::Resampling)
Resampling used for the inner loop.

outer_resampling

mlr3::Resampling)
Resampling used for the outer loop.

measure

(mlr3::Measure)
Measure to optimize.

term_evals

(integer(1))
Number of allowed evaluations.

term_time

(integer(1))
Maximum allowed time in seconds.

search_space

(paradox::ParamSet)
Hyperparameter search space. If NULL (default), the search space is constructed from the TuneToken of the learner's parameter set (learner$param_set).

...

(named list())
Named arguments to be set as parameters of the tuner.

Value

mlr3::ResampleResult

Examples

rr = tune_nested(
  method = "random_search",
  task = tsk("pima"),
  learner = lrn("classif.rpart", cp = to_tune(1e-04, 1e-1, logscale = TRUE)), 
  inner_resampling = rsmp ("holdout"),
  outer_resampling = rsmp("cv", folds = 2), 
  measure = msr("classif.ce"),
  term_evals = 2,
  batch_size = 2)

# retrieve inner tuning results.
extract_inner_tuning_results(rr)
#>    iteration        cp classif.ce learner_param_vals  x_domain task_id
#> 1:         1 -8.258301   0.250000          <list[2]> <list[1]>    pima
#> 2:         2 -4.037840   0.234375          <list[2]> <list[1]>    pima
#>             learner_id resampling_id
#> 1: classif.rpart.tuned            cv
#> 2: classif.rpart.tuned            cv

# performance scores estimated on the outer resampling
rr$score()
#>                 task task_id         learner          learner_id
#> 1: <TaskClassif[49]>    pima <AutoTuner[41]> classif.rpart.tuned
#> 2: <TaskClassif[49]>    pima <AutoTuner[41]> classif.rpart.tuned
#>            resampling resampling_id iteration              prediction
#> 1: <ResamplingCV[19]>            cv         1 <PredictionClassif[20]>
#> 2: <ResamplingCV[19]>            cv         2 <PredictionClassif[20]>
#>    classif.ce
#> 1:  0.2630208
#> 2:  0.2421875

# unbiased performance of the final model trained on the full data set
rr$aggregate()
#> classif.ce 
#>  0.2526042