MLR3 average scores from an ensemble

Question

Using an example from the very helpful mlr3 book, I am trying to simply return the average score of the stacked model output. Can someone please explain how to do this using mlr3? I've tried using both LearnerClassifAvg$new( id = "classif.avg") and po("classifavg"), but not sure I've applied these correctly, thank you

Example:

library("magrittr")
library("mlr3learners") # for classif.glmnet

task      = mlr_tasks$get("iris")
train.idx = sample(seq_len(task$nrow), 120)
test.idx  = setdiff(seq_len(task$nrow), train.idx)

rprt = lrn("classif.rpart", predict_type = "prob")
glmn = lrn("classif.glmnet", predict_type = "prob")

#  Create Learner CV Operators
lrn_0 = PipeOpLearnerCV$new(rprt, id = "rpart_cv_1")
lrn_0$param_set$values$maxdepth = 5L
lrn_1 = PipeOpPCA$new(id = "pca1") %>>% PipeOpLearnerCV$new(rprt, id = "rpart_cv_2")
lrn_1$param_set$values$rpart_cv_2.maxdepth = 1L
lrn_2 = PipeOpPCA$new(id = "pca2") %>>% PipeOpLearnerCV$new(glmn)

# Union them with a PipeOpNULL to keep original features
level_0 = gunion(list(lrn_0, lrn_1,lrn_2, PipeOpNOP$new(id = "NOP1")))

# Cbind the output 3 times, train 2 learners but also keep level
# 0 predictions
level_1 = level_0 %>>%
  PipeOpFeatureUnion$new(4) %>>%
  PipeOpCopy$new(3) %>>%
  gunion(list(
    PipeOpLearnerCV$new(rprt, id = "rpart_cv_l1"),
    PipeOpLearnerCV$new(glmn, id = "glmnt_cv_l1"),
    PipeOpNOP$new(id = "NOP_l1")
  ))


level_1$plot(html = FALSE)


level_2  <- level_1 %>>%
  PipeOpFeatureUnion$new(3, id = "u2") %>>%
  LearnerClassifAvg$new( id = "classif.avg")

level_2$plot(html = FALSE)


lrn = GraphLearner$new(level_2)


lrn$
  train(task, train.idx)$
  predict(task, test.idx)$
  score()

## returns: Error: Trying to predict response, but incoming data has no factors

you are not just passing the predictions but also the features and the level 0 predictions via PipeOpNOP to the "classif.avg". Most likely it does not know how to average this kind of input. Perhaps pass them to a super learner instead. Here is an example: mlr3gallery.mlr-org.com/posts/2020-04-27-tuning-stacking. Additionally output of "learner_cv" can not be the input to "classifavg", — missuse
To see what you are inputting to "classif.avg", construct level_2 <- level_1 %>>% PipeOpFeatureUnion$new(3, id = "u2") and run level_2$train(task)[[1]]$data(). This is not a sensible input to "classif.avg" but it is sensible to input it to another learner. — missuse
Thank you. Looking into the $data() is a useful technique to better understand the outputs. I used the examples in the provided link to add a learner onto the the end of the sequence, which works well. It's still a little unclear how to simply average scores (as a toy example of working with pipelines) from a level of mixed learners without the NOP, but perhaps this was a flawed idea in the first place. Thanks again for the reply and help. — snowkeep
@missuse thank you for your helpful comment. Can out of bag predictions with po("learner_cv" .. ) only be the input to a learner? Is there a way to send these predictions to a "classif.avg"? — ava
@ava currently I do not think they can be averaged, when trying an error is shown: Error: Output type of PipeOp learner_cv during prediction incompatible with input type of PipeOp classifavg — missuse

ava ava · Accepted Answer · 2021-03-27T19:53:08

If we do not pass the features to classif.avg (PipeOpNOP) we still end up with the same error:

Error: Trying to predict response, but incoming data has no factors

library("magrittr")
library("mlr3learners") # for classif.glmnet
library("mlr3verse") #for LearnerClassifAvg
library("mlr3pipelines") # for pipelines

task      = mlr_tasks$get("iris")
train.idx = sample(seq_len(task$nrow), 120)
test.idx  = setdiff(seq_len(task$nrow), train.idx)

rprt = lrn("classif.rpart", predict_type = "prob")
glmn = lrn("classif.glmnet", predict_type = "prob")

#  Create Learner CV Operators
lrn_0 = PipeOpLearnerCV$new(rprt, id = "rpart_cv_1")
lrn_0$param_set$values$maxdepth = 5L
lrn_1 = PipeOpPCA$new(id = "pca1") %>>% PipeOpLearnerCV$new(rprt, id = "rpart_cv_2")
lrn_1$param_set$values$rpart_cv_2.maxdepth = 1L
lrn_2 = PipeOpPCA$new(id = "pca2") %>>% PipeOpLearnerCV$new(glmn)

# Union them with a PipeOpNULL to keep original features
level_0 = gunion(list(lrn_0, lrn_1,lrn_2, PipeOpNOP$new(id = "NOP1")))

# Cbind the output 3 times, train 2 learners but also keep level
# 0 predictions
level_1 = level_0 %>>%
  PipeOpFeatureUnion$new(4) %>>%
  PipeOpCopy$new(2) %>>%
  gunion(list(
    PipeOpLearnerCV$new(rprt, id = "rpart_cv_l1"),
    PipeOpLearnerCV$new(glmn, id = "glmnt_cv_l1")
    # PipeOpNOP$new(id = "NOP_l1") #leave out features here
  ))


level_2  <- level_1 %>>%
  PipeOpFeatureUnion$new(2, id = "u2") %>>%
  LearnerClassifAvg$new( id = "classif.avg")

level_2$plot(html = FALSE)


lrn = GraphLearner$new(level_2)


lrn$
  train(task, train.idx)$
  predict(task, test.idx)$
  score()
#> INFO  [20:42:55.490] [mlr3]  Applying learner 'classif.rpart' on task 'iris' (iter 2/3) 
#> INFO  [20:42:55.557] [mlr3]  Applying learner 'classif.rpart' on task 'iris' (iter 1/3) 
#> INFO  [20:42:55.591] [mlr3]  Applying learner 'classif.rpart' on task 'iris' (iter 3/3) 
#> INFO  [20:42:55.810] [mlr3]  Applying learner 'classif.rpart' on task 'iris' (iter 3/3) 
#> INFO  [20:42:55.849] [mlr3]  Applying learner 'classif.rpart' on task 'iris' (iter 2/3) 
#> INFO  [20:42:55.901] [mlr3]  Applying learner 'classif.rpart' on task 'iris' (iter 1/3) 
#> INFO  [20:42:56.188] [mlr3]  Applying learner 'classif.glmnet' on task 'iris' (iter 3/3) 
#> INFO  [20:42:56.299] [mlr3]  Applying learner 'classif.glmnet' on task 'iris' (iter 1/3) 
#> INFO  [20:42:56.374] [mlr3]  Applying learner 'classif.glmnet' on task 'iris' (iter 2/3) 
#> INFO  [20:42:56.634] [mlr3]  Applying learner 'classif.rpart' on task 'iris' (iter 1/3) 
#> INFO  [20:42:56.699] [mlr3]  Applying learner 'classif.rpart' on task 'iris' (iter 2/3) 
#> INFO  [20:42:56.765] [mlr3]  Applying learner 'classif.rpart' on task 'iris' (iter 3/3) 
#> INFO  [20:42:57.065] [mlr3]  Applying learner 'classif.glmnet' on task 'iris' (iter 2/3) 
#> INFO  [20:42:57.177] [mlr3]  Applying learner 'classif.glmnet' on task 'iris' (iter 1/3) 
#> INFO  [20:42:57.308] [mlr3]  Applying learner 'classif.glmnet' on task 'iris' (iter 3/3)
#> Error: Trying to predict response, but incoming data has no factors

^{Created on 2021-03-27 by the reprex package (v1.0.0)}

This error can be migitated by setting the correct predict type of the learner:

lrn_avg <- LearnerClassifAvg$new( id = "classif.avg")
lrn_avg$predict_type ="prob"

check error message here: https://github.com/cran/mlr3pipelines/blob/master/R/LearnerAvg.R

if (all(fcts) != (self$predict_type == "response")) {
        stopf("Trying to predict %s, but incoming data has %sfactors", self$predict_type, if (all(fcts)) "only " else "no "

Solution demonstrated with a simpler ensemble

library("magrittr")
library("mlr3learners") # for classif.glmnet
#> Lade nötiges Paket: mlr3
library("mlr3verse") #for LearnerClassifAvg
library("mlr3pipelines") # for pipelines

# Define task
task      = mlr_tasks$get("iris")
train.idx = sample(seq_len(task$nrow), 120)
test.idx  = setdiff(seq_len(task$nrow), train.idx)

rprt = lrn("classif.rpart", predict_type = "prob")
glmn = lrn("classif.glmnet", predict_type = "prob")

# Define level 0
level_0 =
  gunion(list(
    PipeOpLearnerCV$new(rprt, id = "rpart_cv_l1"),
    PipeOpLearnerCV$new(glmn, id = "glmnt_cv_l1")
    # PipeOpNOP$new(id = "NOP_l1")
  ))

# Create "averager" learner (and set predict type to "prob")
lrn_avg <- LearnerClassifAvg$new( id = "classif.avg")
lrn_avg$predict_type ="prob"

# Combine level 0 and "averager" learner
level_1  <- level_0 %>>%
  PipeOpFeatureUnion$new(2, id = "u1") %>>%
  lrn_avg

# Show ensemble
level_1$plot(html = FALSE)

# Turn into learner
lrn = GraphLearner$new(level_1)

# Make predictions
set.seed(123)
lrn$
  train(task, train.idx)$
  predict(task, test.idx)$
  score()
#> INFO  [14:32:46.626] [mlr3]  Applying learner 'classif.rpart' on task 'iris' (iter 2/3) 
#> INFO  [14:32:46.692] [mlr3]  Applying learner 'classif.rpart' on task 'iris' (iter 3/3) 
#> INFO  [14:32:46.724] [mlr3]  Applying learner 'classif.rpart' on task 'iris' (iter 1/3) 
#> INFO  [14:32:47.060] [mlr3]  Applying learner 'classif.glmnet' on task 'iris' (iter 2/3) 
#> INFO  [14:32:47.136] [mlr3]  Applying learner 'classif.glmnet' on task 'iris' (iter 1/3) 
#> INFO  [14:32:47.209] [mlr3]  Applying learner 'classif.glmnet' on task 'iris' (iter 3/3)
#> classif.ce 
#>        0.1

^{Created on 2021-03-28 by the reprex package (v1.0.0)}

MLR3 average scores from an ensemble

1 Answers