align verbosity options with documented behavior

NEWS.md

@@ -2,6 +2,8 @@
 
 * Added a new function, `compute_metrics()`, that allows for computing new metrics after evaluating against resamples. The arguments and output formats are closely related to those from `collect_metrics()`, but this function requires that the input be generated with the control option `save_pred = TRUE` and additionally takes a `metrics` argument with a metric set for new metrics to compute. This allows for computing new performance metrics without requiring users to re-fit and re-predict from each model. (#663)
 
+* Disambiguates the `verbose` and `verbose_iter` control options to better align with documented functionality. The former controls logging for general progress updates, while the latter only does so for the Bayesian search process. (#682) 
+
 # tune 1.1.1
 
 * Fixed a bug introduced in tune 1.1.0 in `collect_()` functions where the 

R/grid_performance.R

@@ -48,6 +48,8 @@ metrics_info <- function(x) {
 #' @param new_data A data frame or matrix of predictors to process.
 #' @param metrics_info The output of `tune:::metrics_info(metrics)`---only
 #' included as an argument to allow for pre-computing.
+#' @param iter A logical passed to `tune_log()`, giving whether supplied messages
+#' ought to be interpreted as updates on iterative search.
 #' @keywords internal
 #' @name tune-internal-functions
 #' @export

R/logging.R

@@ -256,12 +256,12 @@ siren <- function(x, type = "info") {
 }
 
 
-tune_log <- function(control, split = NULL, task, type = "success") {
-  if (!control$verbose) {
+tune_log <- function(control, split = NULL, task, type = "success", iter = FALSE) {
+  if (!any(control$verbose, control$verbose_iter)) {
     return(invisible(NULL))
   }
 
-  if (uses_catalog()) {
+  if (uses_catalog() & !iter) {
     log_catalog(task, type)
     return(NULL)
   }
@@ -291,6 +291,7 @@ log_problems <- function(notes, control, split, loc, res, bad_only = FALSE) {
   # Always log warnings and errors
   control2 <- control
   control2$verbose <- TRUE
+  control2$verbose_iter <- TRUE
 
   should_catalog <- uses_catalog()
 
@@ -361,8 +362,8 @@ format_msg <- function(loc, msg) {
 
 #' @export
 #' @rdname tune-internal-functions
-.catch_and_log <- function(.expr, ..., bad_only = FALSE, notes) {
-  tune_log(..., type = "info")
+.catch_and_log <- function(.expr, ..., bad_only = FALSE, notes, iter = FALSE) {
+  tune_log(..., type = "info", iter = iter)
   tmp <- catcher(.expr)
   new_notes <- log_problems(notes, ..., tmp, bad_only = bad_only)
   assign("out_notes", new_notes, envir = parent.frame())
@@ -410,7 +411,7 @@ format_msg <- function(loc, msg) {
 }
 
 log_best <- function(control, iter, info, digits = 4) {
-  if (!control$verbose) {
+  if (!isTRUE(control$verbose_iter)) {
     return(invisible(NULL))
   }
 
@@ -428,24 +429,30 @@ log_best <- function(control, iter, info, digits = 4) {
       info$best_iter,
       ")"
     )
-  tune_log(control, split = NULL, task = msg, type = "info")
+  tune_log(control, split = NULL, task = msg, type = "info", iter = TRUE)
 }
 
 check_and_log_flow <- function(control, results) {
+  if (!isTRUE(control$verbose_iter)) {
+    return(invisible(NULL))
+  }
+
   if (all(is.na(results$.mean))) {
     if (nrow(results) < 2) {
-      tune_log(control, split = NULL, task = "Halting search", type = "danger")
+      tune_log(control, split = NULL, task = "Halting search",
+               type = "danger", iter = TRUE)
       eval.parent(parse(text = "break"))
     } else {
-      tune_log(control, split = NULL, task = "Skipping to next iteration", type = "danger")
+      tune_log(control, split = NULL, task = "Skipping to next iteration",
+               type = "danger", iter = TRUE)
       eval.parent(parse(text = "next"))
     }
   }
   invisible(NULL)
 }
 
 log_progress <- function(control, x, maximize = TRUE, objective = NULL, digits = 4) {
-  if (!control$verbose) {
+  if (!isTRUE(control$verbose_iter)) {
     return(invisible(NULL))
   }
 
@@ -479,7 +486,7 @@ log_progress <- function(control, x, maximize = TRUE, objective = NULL, digits =
 }
 
 param_msg <- function(control, candidate) {
-  if (!control$verbose) {
+  if (!isTRUE(control$verbose_iter)) {
     return(invisible(NULL))
   }
   candidate <- candidate[, !(names(candidate) %in% c(".mean", ".sd", "objective"))]
@@ -495,7 +502,7 @@ param_msg <- function(control, candidate) {
 
 
 acq_summarizer <- function(control, iter, objective = NULL, digits = 4) {
-  if (!control$verbose) {
+  if (!isTRUE(control$verbose_iter)) {
     return(invisible(NULL))
   }
   if (inherits(objective, "conf_bound") && is.function(objective$kappa)) {
@@ -509,7 +516,7 @@ acq_summarizer <- function(control, iter, objective = NULL, digits = 4) {
     }
   }
   if (!is.null(val)) {
-    tune_log(control, split = NULL, task = val, type = "info")
+    tune_log(control, split = NULL, task = val, type = "info", iter = TRUE)
   }
   invisible(NULL)
 }

R/tune_bayes.R

@@ -342,7 +342,8 @@ tune_bayes_workflow <-
           control,
           NULL,
           "Gaussian process model",
-          notes = .notes
+          notes = .notes,
+          iter = TRUE
         )
 
       gp_mod <- check_gp_failure(gp_mod, prev_gp_mod)
@@ -563,13 +564,14 @@ pred_gp <- function(object, pset, size = 5000, current = NULL, control) {
     control,
     split = NULL,
     task = paste("Generating", nrow(pred_grid), "candidates"),
-    type = "info"
+    type = "info",
+    iter = TRUE
   )
 
   x <- encode_set(pred_grid, pset, as_matrix = TRUE)
   gp_pred <- predict(object, x)
 
-  tune_log(control, split = NULL, task = "Predicted candidates", type = "info")
+  tune_log(control, split = NULL, task = "Predicted candidates", type = "info", iter = TRUE)
 
   pred_grid %>%
     dplyr::mutate(.mean = gp_pred$Y_hat, .sd = sqrt(gp_pred$MSE))

tests/testthat/_snaps/bayes.md

@@ -8,18 +8,13 @@
       >  Generating a set of 2 initial parameter results
       v Initialization complete
 
-
-      -- Iteration 1 -----------------------------------------------------------------
-
-      i Current best:		rmse=2.418 (@iter 0)
       i Gaussian process model
       ! The Gaussian process model is being fit using 1 features but only has 2
         data points to do so. This may cause errors or a poor model fit.
       ! Gaussian process model: X should be in range (0, 1)
       v Gaussian process model
       i Generating 3 candidates
       i Predicted candidates
-      i num_comp=2
       i Estimating performance
       i Fold01: preprocessor 1/1
       v Fold01: preprocessor 1/1
@@ -82,17 +77,11 @@
       i Fold10: preprocessor 1/1, model 1/1 (extracts)
       i Fold10: preprocessor 1/1, model 1/1 (predictions)
       v Estimating performance
-      (x) Newest results:	rmse=2.666 (+/-0.281)
-
-      -- Iteration 2 -----------------------------------------------------------------
-
-      i Current best:		rmse=2.418 (@iter 0)
       i Gaussian process model
       ! Gaussian process model: X should be in range (0, 1)
       v Gaussian process model
       i Generating 2 candidates
       i Predicted candidates
-      i num_comp=5
       i Estimating performance
       i Fold01: preprocessor 1/1
       v Fold01: preprocessor 1/1
@@ -155,7 +144,6 @@
       i Fold10: preprocessor 1/1, model 1/1 (extracts)
       i Fold10: preprocessor 1/1, model 1/1 (predictions)
       v Estimating performance
-      (x) Newest results:	rmse=2.453 (+/-0.381)
     Output
       # Tuning results
       # 10-fold cross-validation 
@@ -181,10 +169,34 @@
         control = control_bayes(verbose_iter = TRUE))
     Message
       Optimizing rmse using the expected improvement
+
+      -- Iteration 1 -----------------------------------------------------------------
+
+      i Current best:		rmse=2.418 (@iter 0)
+      i Gaussian process model
       ! The Gaussian process model is being fit using 1 features but only has 2
         data points to do so. This may cause errors or a poor model fit.
       ! Gaussian process model: X should be in range (0, 1)
+      v Gaussian process model
+      i Generating 3 candidates
+      i Predicted candidates
+      i num_comp=4
+      i Estimating performance
+      v Estimating performance
+      (x) Newest results:	rmse=2.461 (+/-0.37)
+
+      -- Iteration 2 -----------------------------------------------------------------
+
+      i Current best:		rmse=2.418 (@iter 0)
+      i Gaussian process model
       ! Gaussian process model: X should be in range (0, 1)
+      v Gaussian process model
+      i Generating 2 candidates
+      i Predicted candidates
+      i num_comp=5
+      i Estimating performance
+      v Estimating performance
+      (x) Newest results:	rmse=2.453 (+/-0.381)
     Output
       # Tuning results
       # 10-fold cross-validation 

tests/testthat/test-logging.R

@@ -82,8 +82,8 @@ test_that("catch and log issues", {
 })
 
 test_that("logging iterations", {
-  ctrl_t <- control_grid(verbose = TRUE)
-  ctrl_f <- control_grid(verbose = FALSE)
+  ctrl_t <- control_bayes(verbose_iter = TRUE)
+  ctrl_f <- control_bayes(verbose_iter = FALSE)
   sc_1 <- list(
     best_val = 7,
     best_iter = 2,
@@ -99,7 +99,7 @@ test_that("logging iterations", {
 })
 
 test_that("logging search info", {
-  ctrl_t <- control_grid(verbose = TRUE)
+  ctrl_t <- control_bayes(verbose_iter = TRUE)
   tb_1 <- tibble::tibble(.mean = 1:3)
 
   expect_silent(tune:::check_and_log_flow(ctrl_t, tb_1))
@@ -114,8 +114,8 @@ test_that("logging search info", {
 })
 
 test_that("current results", {
-  ctrl_t <- control_grid(verbose = TRUE)
-  ctrl_f <- control_grid(verbose = FALSE)
+  ctrl_t <- control_bayes(verbose_iter = TRUE)
+  ctrl_f <- control_bayes(verbose_iter = FALSE)
   tb_2 <-
     tibble::tibble(
       .metric = rep(letters[1:2], each = 4),
@@ -141,17 +141,17 @@ test_that("current results", {
 
 
 test_that("show parameters", {
-  ctrl_t <- control_grid(verbose = TRUE)
-  ctrl_f <- control_grid(verbose = FALSE)
+  ctrl_t <- control_bayes(verbose_iter = TRUE)
+  ctrl_f <- control_bayes(verbose_iter = FALSE)
 
   expect_snapshot(tune:::param_msg(ctrl_t, iris[1, 4:5]))
   expect_silent(tune:::param_msg(ctrl_f, iris[1, 4:5]))
 })
 
 
 test_that("acquisition functions", {
-  ctrl_t <- control_grid(verbose = TRUE)
-  ctrl_f <- control_grid(verbose = FALSE)
+  ctrl_t <- control_bayes(verbose_iter = TRUE)
+  ctrl_f <- control_bayes(verbose_iter = FALSE)
 
   expect_silent(tune:::acq_summarizer(ctrl_t, 1))
   expect_silent(tune:::acq_summarizer(ctrl_t, 1, conf_bound()))