This is the result container object returned by benchmark(). A BenchmarkResult consists of the data row-binded data of multiple ResampleResults, which can easily be re-constructed.

Note that all stored objects are accessed by reference. Do not modify any object without cloning it first.

Format

R6::R6Class object.

Construction

bmr = BenchmarkResult$new(data = data.table())

Fields

Methods

S3 Methods

Examples

set.seed(123) learners = list( lrn("classif.featureless", predict_type = "prob"), lrn("classif.rpart", predict_type = "prob") ) design = benchmark_grid( tasks = list(tsk("sonar"), tsk("spam")), learners = learners, resamplings = rsmp("cv", folds = 3) ) print(design)
#> task learner resampling #> 1: <TaskClassif> <LearnerClassifFeatureless> <ResamplingCV> #> 2: <TaskClassif> <LearnerClassifRpart> <ResamplingCV> #> 3: <TaskClassif> <LearnerClassifFeatureless> <ResamplingCV> #> 4: <TaskClassif> <LearnerClassifRpart> <ResamplingCV>
bmr = benchmark(design) print(bmr)
#> <BenchmarkResult> of 12 rows with 4 resampling runs #> nr task_id learner_id resampling_id iters warnings errors #> 1 sonar classif.featureless cv 3 0 0 #> 2 sonar classif.rpart cv 3 0 0 #> 3 spam classif.featureless cv 3 0 0 #> 4 spam classif.rpart cv 3 0 0
bmr$tasks
#> task_hash task_id task #> 1: dea3e1fd99a2120d sonar <TaskClassif> #> 2: 7cf4341432d6352e spam <TaskClassif>
bmr$learners
#> learner_hash learner_id learner #> 1: 3bbabd1058707305 classif.featureless <LearnerClassifFeatureless> #> 2: fc402e71eadd46bb classif.rpart <LearnerClassifRpart>
# first 5 individual resamplings head(as.data.table(bmr, measures = c("classif.acc", "classif.auc")), 5)
#> uhash task #> 1: b29dbcc3-c02c-43ae-9043-c9e9ff365c70 <TaskClassif> #> 2: b29dbcc3-c02c-43ae-9043-c9e9ff365c70 <TaskClassif> #> 3: b29dbcc3-c02c-43ae-9043-c9e9ff365c70 <TaskClassif> #> 4: 8d42a650-0077-49b9-9f8f-4e914095d111 <TaskClassif> #> 5: 8d42a650-0077-49b9-9f8f-4e914095d111 <TaskClassif> #> learner resampling iteration prediction #> 1: <LearnerClassifFeatureless> <ResamplingCV> 1 <list> #> 2: <LearnerClassifFeatureless> <ResamplingCV> 2 <list> #> 3: <LearnerClassifFeatureless> <ResamplingCV> 3 <list> #> 4: <LearnerClassifRpart> <ResamplingCV> 1 <list> #> 5: <LearnerClassifRpart> <ResamplingCV> 2 <list>
# aggregate results bmr$aggregate()
#> nr resample_result task_id learner_id resampling_id iters #> 1: 1 <ResampleResult> sonar classif.featureless cv 3 #> 2: 2 <ResampleResult> sonar classif.rpart cv 3 #> 3: 3 <ResampleResult> spam classif.featureless cv 3 #> 4: 4 <ResampleResult> spam classif.rpart cv 3 #> classif.ce #> 1: 0.4660455 #> 2: 0.2739130 #> 3: 0.3940399 #> 4: 0.1086721
# aggregate results with hyperparameters as separate columns mlr3misc::unnest(bmr$aggregate(params = TRUE), "params")
#> nr resample_result task_id learner_id resampling_id iters #> 1: 1 <ResampleResult> sonar classif.featureless cv 3 #> 2: 2 <ResampleResult> sonar classif.rpart cv 3 #> 3: 3 <ResampleResult> spam classif.featureless cv 3 #> 4: 4 <ResampleResult> spam classif.rpart cv 3 #> classif.ce method xval #> 1: 0.4660455 mode NA #> 2: 0.2739130 <NA> 0 #> 3: 0.3940399 mode NA #> 4: 0.1086721 <NA> 0
# extract resample result for classif.rpart rr = bmr$aggregate()[learner_id == "classif.rpart", resample_result][[1]] print(rr)
#> <ResampleResult> of 3 iterations #> * Task: sonar #> * Learner: classif.rpart #> * Warnings: 0 in 0 iterations #> * Errors: 0 in 0 iterations
# access the confusion matrix of the first resampling iteration rr$predictions()[[1]]$confusion
#> truth #> response M R #> M 30 18 #> R 3 19