...

R/MeasureCIHoldout.R

@@ -2,6 +2,7 @@
 #' @name mlr_measures_ci_holdout
 #' @description
 #' Standard holdout CI.
+#' This inference method can only be applied to decomposable losses.
 #' @section Parameters:
 #' Only those from [`MeasureAbstractCi`].
 #' @template param_measure

R/MeasureCINaiveCV.R

@@ -4,6 +4,7 @@
 #' Confidence intervals for cross-validation.
 #' The method is asymptotically exact for the so called *Test Error* as defined by Bayle et al. (2020).
 #' For the (expected) risk, the confidence intervals tend to be too liberal.
+#' This inference method can only be applied to decomposable losses.
 #' @section Parameters:
 #' Those from [`MeasureAbstractCi`], as well as:
 #' * `variance` :: `"all-pairs"` or `"within-fold"`\cr

R/MeasureCiNestedCV.R

@@ -2,6 +2,7 @@
 #' @name mlr_measures_ci_ncv
 #' @description
 #' Confidence Intervals based on [`ResamplingNestedCV`][ResamplingNestedCV], including bias-correction.
+#' This inference method can only be applied to decomposable losses.
 #' @section Parameters:
 #' Those from [`MeasureAbstractCi`], as well as:
 #' * `bias` :: `logical(1)`\cr

R/zzz.R

@@ -28,8 +28,6 @@ register_mlr3 = function(...) {
     ResamplingPairedSubsampling = "ci.con_z",
     ResamplingNestedCV = "ci.ncv"
   )
-
-  mlr_reflections$measure_properties$regr = c(mlr_reflections$measure_properties$regr, "requires_obs_loss")
 }
 
 

README.Rmd

@@ -96,7 +96,7 @@ Note that:
 
 ```{r, echo = TRUE}
 content = as.data.table(mlr3::mlr_measures, objects = TRUE)[startsWith(get("key"), "ci."),]
-content$resamplings = map(content$object, function(x) paste0(x$resamplings, sep = ", "))
+content$resamplings = map(content$object, function(x) paste0(gsub("Resampling", "", x$resamplings), collapse = ", "))
 content[["only pointwise loss"]] = map_chr(content$object, function(object) {
   if (get_private(object)$.requires_obs_loss) "yes" else "false"
 })

README.md

@@ -91,21 +91,21 @@ Note that:
 
 ``` r
 content = as.data.table(mlr3::mlr_measures, objects = TRUE)[startsWith(get("key"), "ci."),]
-content$resamplings = map(content$object, function(x) paste0(x$resamplings, sep = ", "))
+content$resamplings = map(content$object, function(x) paste0(gsub("Resampling", "", x$resamplings), collapse = ", "))
 content[["only pointwise loss"]] = map_chr(content$object, function(object) {
   if (get_private(object)$.requires_obs_loss) "yes" else "false"
 })
 content = content[, c("key", "label", "resamplings", "only pointwise loss")]
 knitr::kable(content, format = "markdown", col.names = tools::toTitleCase(names(content)))
 ```
 
-| Key | Label | Resamplings | Only Pointwise Loss |
-|:---|:---|:---|:---|
-| ci.con_z | Conservative-Z CI | ResamplingPairedSubsampling, | false |
-| ci.cor_t | Corrected-T CI | ResamplingSubsampling, | false |
-| ci.holdout | Holdout CI | ResamplingHoldout, | yes |
-| ci.naive_cv | Naive CV CI | ResamplingCV, , ResamplingLOO, | yes |
-| ci.ncv | Nested CV CI | ResamplingNestedCV, | yes |
+| Key         | Label             | Resamplings       | Only Pointwise Loss |
+|:------------|:------------------|:------------------|:--------------------|
+| ci.con_z    | Conservative-Z CI | PairedSubsampling | false               |
+| ci.cor_t    | Corrected-T CI    | Subsampling       | false               |
+| ci.holdout  | Holdout CI        | Holdout           | yes                 |
+| ci.naive_cv | Naive CV CI       | CV, LOO           | yes                 |
+| ci.ncv      | Nested CV CI      | NestedCV          | yes                 |
 
 ## Bugs, Questions, Feedback