paging for gcs_list_objects (#58)

NEWS.md

@@ -5,6 +5,7 @@
 * Add support for `gs://` style URLs for object names (#57 - thanks seandavi)
 * Add what the public URL would be to an objects print method (#59 - thanks mwhitaker)
 * Add check to `gcs_get_bucket()` to only expect length 1 character vectors for bucket name. (#60)
+* Add paging for `gcs_object_list` (#58 - thanks @G3rtjan)
 
 # 0.2.0
 

R/objects.R

@@ -27,11 +27,14 @@ gcs_list_objects <- function(bucket = gcs_get_global_bucket(),
   testthat::expect_length(bucket, 1)
 
   parse_lo <- function(x){
+    nextPageToken <- x$nextPageToken
     x <- x$items
     x$timeCreated <- timestamp_to_r(x$timeCreated)
     x$updated <- timestamp_to_r(x$updated)
     x$kind <- NULL
     x$size <- vapply(as.numeric(x$size), function(x) format_object_size(x, "auto"), character(1))
+
+    attr(x, "nextPageToken") <- nextPageToken
     x
   }
 
@@ -41,6 +44,27 @@ gcs_list_objects <- function(bucket = gcs_get_global_bucket(),
                                       data_parse_function = parse_lo)
   req <- lo()
 
+  ## page through list if necessary
+  if(!is.null(attr(req, "nextPageToken"))){
+    npt <- attr(req, "nextPageToken")
+
+    lo2 <- googleAuthR::gar_api_generator("https://www.googleapis.com/storage/v1/",
+                                         path_args = list(b = bucket,
+                                                          o = ""),
+                                         pars_args = list(pageToken = npt),
+                                         data_parse_function = parse_lo)
+
+
+    while(!is.null(npt)){
+      myMessage("Paging through results: ", npt, level = 3)
+      more_req <- lo2(pars_arguments = npt)
+      npt <- attr(more_req, "nextPageToken")
+      req <- rbind(req, more_req)
+    }
+  }
+
+
+
   out_names <- switch(detail,
       summary = c("name", "size", "updated"),
       more = c("name", "size", "bucket", "contentType", "timeCreated", "updated", "storageClass"),