Skip to content

Commit

Permalink
Better RSPM support
Browse files Browse the repository at this point in the history 
Towards #47 and r-lib/pkgdepends#186
  • Loading branch information
@gaborcsardi
gaborcsardi committed Feb 15, 2023
1 parent 9edad8d commit bb562ca
Show file tree
Hide file tree
Showing 10 changed files with 431 additions and 228 deletions.
64 changes: 56 additions & 8 deletions R/metadata-cache.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -456,7 +456,19 @@ cmc__get_cache_files <- function(self, private, which) {
paste0(cran_metadata_url(), pkgs_dirs, "/METADATA2.gz"),
NA_character_)

list(
bin_url <- rspm_binary_url(mirror, private$r_version)
bin_path <- ifelse(
is.na(bin_url),
NA_character_,
file.path(root, "_metadata", repo_enc, pkgs_dirs, "PACKAGES2.gz")
)
bin_etag <- ifelse(
is.na(bin_url),
NA_character_,
paste0(bin_path, ".etag")
)

res <- list(
root = root,
meta = file.path(root, "_metadata"),
lock = file.path(root, "_metadata.lock"),
Expand All @@ -475,11 +487,45 @@ cmc__get_cache_files <- function(self, private, which) {
bioc_version = bioc_version,
meta_path = meta_path,
meta_etag = meta_etag,
meta_url = meta_url
meta_url = meta_url,
bin_path = bin_path,
bin_etag = bin_etag,
bin_url = bin_url
)
)

res
}

rspm_binary_url <- function(urls, r_version) {
res <- rep(NA_character_, length(urls))

# If multiple R versions are requested, then we give up, and pretend
# that RSPM binaries are source packages
if (length(r_version) != 1) return(res)

# http://rspm.infra/all/__linux__/bionic/latest ->
# http://rspm.infra/all/latest/bin/linux/4.2-bionic/contrib/4.2/PACKAGES

has_binary <- re_match(
urls,
"^(?<base>.*/)(?<repo>cran|all)/__linux__/(?<distro>[a-zA-Z0-9]+)/(?<version>latest|[-0-9]+)$"
)
mch <- !is.na(has_binary$.match)

res[mch] <- paste0(
has_binary$base[mch],
has_binary$repo[mch], "/",
has_binary$version[mch], "/bin/linux/",
r_version, "-", has_binary$distro[mch],
"/contrib/", r_version, "/",
"PACKAGES.gz"
)

res
}


#' Load the cache, asynchronously, with as little work as possible
#'
#' 1. If it is already loaded, and fresh return it.
Expand Down Expand Up @@ -685,13 +731,14 @@ cmc__update_replica_pkgs <- function(self, private) {
pkgs <- rep_files$pkgs

meta <- !is.na(pkgs$meta_url)
bin <- !is.na(pkgs$bin_url)
dls <- data.frame(
stringsAsFactors = FALSE,
url = c(pkgs$url, pkgs$meta_url[meta]),
fallback_url = c(pkgs$fallback_url, rep(NA_character_, sum(meta))),
path = c(pkgs$path, pkgs$meta_path[meta]),
etag = c(pkgs$etag, pkgs$meta_etag[meta]),
timeout = rep(c(200, 100), c(nrow(pkgs), sum(meta))),
url = c(pkgs$url, pkgs$meta_url[meta], pkgs$bin_url[bin]),
fallback_url = c(pkgs$fallback_url, rep(NA_character_, sum(meta) + sum(bin))),
path = c(pkgs$path, pkgs$meta_path[meta], pkgs$bin_path[bin]),
etag = c(pkgs$etag, pkgs$meta_etag[meta], pkgs$bin_path[bin]),
timeout = rep(c(200, 100), c(nrow(pkgs), sum(meta) + sum(bin))),
mayfail = TRUE
)

Expand Down Expand Up @@ -755,7 +802,8 @@ cmc__update_replica_rds <- function(self, private, alert) {
read_packages_file(r$path, mirror = r$mirror,
repodir = r$basedir, platform = r$platform,
rversion = rversion, type = r$type,
meta_path = r$meta_path),
meta_path = r$meta_path, bin_path = r$bin_path,
orig_r_version = private$r_version),
error = function(x) {
message()
warning(msg_wrap(
Expand Down
454 changes: 246 additions & 208 deletions R/onload.R
View file

Large diffs are not rendered by default.

4 changes: 4 additions & 0 deletions R/package-cache.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,10 @@ package_cache <- R6Class(
download_file(url, target, on_progress = on_progress,
headers = http_headers)$
then(function(res) {
headers <- curl::parse_headers(res$response$headers, multiple = TRUE)[[1]]
if ("x-package-type: binary" %in% tolower(headers)) {
# TODO: update path, r_version, platform, etc.?
}
self$add(target, path, url = url, etag = res$etag, ...,
sha256 = shasum256(target), .list = .list)
})$
Expand Down
27 changes: 27 additions & 0 deletions R/packages-gz.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ packages_gz_cols <- function() {

read_packages_file <- function(path, mirror, repodir, platform,
type = "standard", meta_path = NA_character_,
bin_path = NA_character_, orig_r_version = NULL,
..., .list = list()) {

# We might have empty PACKAGES.gz files, we treat them as empty here
Expand All @@ -23,6 +24,7 @@ read_packages_file <- function(path, mirror, repodir, platform,
pkgs <- parse_packages(path)
}
meta <- read_metadata_file(meta_path)
bin <- read_rspm_binaries(bin_path)
extra <- c(
list(repodir = repodir),
list(...), .list)
Expand Down Expand Up @@ -102,6 +104,22 @@ read_packages_file <- function(path, mirror, repodir, platform,
rep(NA_character_, nrow(pkgs))
}

# RSPM sources are really binaries for the current platform
hasbin <- pkgs$package %in% bin$Package
if (length(orig_r_version) == 1 && sum(hasbin) > 0) {
plat <- current_r_platform()
pkgs$platform[hasbin] <- plat
pkgs$rversion[hasbin] <- orig_r_version
pkgs$target[hasbin] <- paste0(
dirname(pkgs$target[hasbin]), "/",
plat, "/",
basename(pkgs$target[hasbin])
)
pkgs$filesize[hasbin] <- NA_integer_
pkgs$sha256[hasbin] <- NA_integer_
pkgs$needscompilation[hasbin] <- NA
}

# If we only want one Windows platform, then filter here
if (platform %in% c("i386-w64-mingw32", "x86_64-w64-mingw32")) {
drop <- pkgs$platform != platform &
Expand Down Expand Up @@ -135,6 +153,15 @@ read_metadata_file <- function(path) {
}), error = function(e) NULL)
}

read_rspm_binaries <- function(path) {
if (is.na(path) || !file.exists(path) || file.size(path) == 0) {
pkgs <- data_frame()
} else {
pkgs <- parse_packages(path)
}
pkgs
}

packages_parse_deps <- function(pkgs) {
no_pkgs <- nrow(pkgs)
cols <- intersect(colnames(pkgs), tolower(dep_types()))
Expand Down
60 changes: 53 additions & 7 deletions R/repo-set.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
#' if `bioc` is `TRUE`.
#' @param cran_mirror The CRAN mirror to use, see
#' [default_cran_mirror()].
#'
#'
#' @return
#' `repo_get()` returns a data frame with columns:
#' * `name`: repository name. Names are informational only.
Expand Down Expand Up @@ -201,15 +201,15 @@ repo_sugar_rspm <- function(x, nm) {
if (is.null(nm) || nm == "") nm <- "CRAN"
date <- parse_spec(sub("^RSPM@", "", x))
if (is.null(pkgenv$rspm_versions) ||
date < names(pkgenv$rspm_versions[1]) ||
date > last(names(pkgenv$rspm_versions))) {
date < names(pkgenv$rspm_versions$versions[1]) ||
date > last(names(pkgenv$rspm_versions$versions))) {
tryCatch(
pkgenv$rspm_versions <- get_rspm_versions(),
error = function(err) warning("Failed to update list of RSPM versions")
)
}

vers <- pkgenv$rspm_versions
vers <- pkgenv$rspm_versions$versions
rspm_dates <- names(vers)
if (date < rspm_dates[1]) {
stop("RSPM snapshots go back to ", as.Date(rspm_dates[1]), " only")
Expand All @@ -223,22 +223,68 @@ repo_sugar_rspm <- function(x, nm) {
"PKGCACHE_RSPM_URL",
"https://packagemanager.posit.co/cran"
)
structure(paste0(rspm, "/", vers[[sel]]), names = nm)

# check if binaries are supported
current <- current_r_platform_data()
distros <- pkgenv$rspm_versions$distros
mch <- which(
distros$distribution == current$distribution &
distros$release == current$release
)
binaries <-
! tolower(Sys.getenv("PKGCACHE_RSPM_BINARIES")) %in% c("no", "false", "0", "off") &&
current$cpu == "x86_64" &&
grepl("linux", current$os) &&
length(mch) == 1

if (binaries) {
structure(
paste0(rspm, "/", "__linux__/", distros$binary_url[mch], "/", vers[[sel]]),
names = nm
)
} else {
structure(paste0(rspm, "/", vers[[sel]]), names = nm)
}
}

get_rspm_versions <- function() {
url <- Sys.getenv(
"PKGCACHE_RSPM_TRANSACTIONS_URL",
"https://packagemanager.posit.co/__api__/sources/1/transactions?_limit=10000"
)
resp <- jsonlite::fromJSON(url, simplifyVector = FALSE)
url2 <- Sys.getenv(
"PKGCACHE_RSPM_STATUS_URL",
"https://packagemanager.posit.co/__api__/status"
)
dl <- data.frame(
stringsAsFactors = FALSE,
url = c(url, url2),
path = c(tmp1 <- tempfile(), tmp2 <- tempfile()),
etag = ""
)
on.exit(unlink(c(tmp1, tmp2)), add = TRUE)
dlres <- synchronise(download_files(dl))

resp <- jsonlite::fromJSON(tmp1, simplifyVector = FALSE)
stat <- jsonlite::fromJSON(tmp2, simplifyVector = FALSE)

vrs <- structure(
vcapply(resp, function(x) as.character(x$id)),
names = vcapply(resp, function(x) as.character(x$published_to))
)
vrs <- vrs[order(as.Date(names(vrs)))]
vrs

dst <- data.frame(
stringsAsFactors = FALSE,
name = vcapply(stat$distros, "[[", "name"),
os = vcapply(stat$distros, "[[", "os"),
binary_url = vcapply(stat$distros, "[[", "binaryURL"),
distribution = vcapply(stat$distros, "[[", "distribution"),
release = vcapply(stat$distros, "[[", "release"),
binaries = vlapply(stat$distros, "[[", "binaries")
)

list(versions = vrs, distros = dst)
}

parse_spec <- function(x) {
Expand Down
29 changes: 29 additions & 0 deletions tests/testthat/_snaps/repo-set.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
Code
ret
Output
$versions
2021-01-25T00:00:00Z 2021-01-26T00:00:00Z 2021-01-27T00:00:00Z
"997643" "1014755" "1033374"
2021-01-28T00:00:00Z 2021-01-29T00:00:00Z 2021-02-01T00:00:00Z
Expand All @@ -33,6 +34,34 @@
"1140568" "1160641" "1175516"
2021-02-05T00:00:00Z
"1194160"
$distros
name os binary_url distribution release binaries
1 centos7 linux centos7 centos 7 TRUE
2 centos8 linux centos8 centos 8 TRUE
3 rhel9 linux rhel9 rockylinux 9 TRUE
4 opensuse15 linux opensuse15 opensuse 15 TRUE
5 opensuse152 linux opensuse152 opensuse 15.2 TRUE
6 opensuse153 linux opensuse153 opensuse 15.3 TRUE
7 opensuse154 linux opensuse154 opensuse 15.4 TRUE
8 opensuse42 linux opensuse42 opensuse 42.3 TRUE
9 rhel7 linux centos7 redhat 7 TRUE
10 rhel8 linux centos8 redhat 8 TRUE
11 rhel9 (unused alias) linux rhel9 redhat 9 TRUE
12 sles12 linux opensuse42 sle 12.3 TRUE
13 sles15 linux opensuse15 sle 15 TRUE
14 sles152 linux opensuse152 sle 15.2 TRUE
15 sles153 linux opensuse153 sle 15.3 TRUE
16 sles154 linux opensuse154 sle 15.4 TRUE
17 xenial linux xenial ubuntu 16.04 TRUE
18 bionic linux bionic ubuntu 18.04 TRUE
19 focal linux focal ubuntu 20.04 TRUE
20 jammy linux jammy ubuntu 22.04 TRUE
21 buster linux buster debian 10 FALSE
22 bullseye linux bullseye debian 11 FALSE
23 windows windows windows all TRUE
24 macOS macOS macOS all FALSE

# get_r_versions

Expand Down
1 change: 1 addition & 0 deletions tests/testthat/fixtures/rspm-status.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"version":"2022.11.2-18","build_date":"2022-11-18T20:02:59Z","metrics_enabled":true,"r_configured":true,"binaries_enabled":true,"display_ash":false,"custom_home":true,"ga_id":"UA-20375833-3","distros":[{"name":"centos7","os":"linux","binaryDisplay":"CentOS/RHEL 7","binaryURL":"centos7","display":"CentOS 7","distribution":"centos","release":"7","sysReqs":true,"binaries":true,"hidden":false,"official_rspm":true},{"name":"centos8","os":"linux","binaryDisplay":"CentOS/RHEL 8","binaryURL":"centos8","display":"CentOS 8","distribution":"centos","release":"8","sysReqs":true,"binaries":true,"hidden":true,"official_rspm":true},{"name":"rhel9","os":"linux","binaryDisplay":"Rocky Linux 9","binaryURL":"rhel9","display":"Rocky Linux 9","distribution":"rockylinux","release":"9","sysReqs":true,"binaries":true,"hidden":false,"official_rspm":true},{"name":"opensuse15","os":"linux","binaryDisplay":"OpenSUSE 15.1, SLES 15 SP1","binaryURL":"opensuse15","display":"OpenSUSE 15.1","distribution":"opensuse","release":"15","sysReqs":true,"binaries":true,"hidden":true,"official_rspm":true},{"name":"opensuse152","os":"linux","binaryDisplay":"OpenSUSE 15.2, SLES 15 SP2","binaryURL":"opensuse152","display":"OpenSUSE 15.2","distribution":"opensuse","release":"15.2","sysReqs":true,"binaries":true,"hidden":true,"official_rspm":true},{"name":"opensuse153","os":"linux","binaryDisplay":"OpenSUSE 15.3, SLES 15 SP3","binaryURL":"opensuse153","display":"OpenSUSE 15.3","distribution":"opensuse","release":"15.3","sysReqs":true,"binaries":true,"hidden":false,"official_rspm":true},{"name":"opensuse154","os":"linux","binaryDisplay":"OpenSUSE 15.4, SLES 15 SP4","binaryURL":"opensuse154","display":"OpenSUSE 15.4","distribution":"opensuse","release":"15.4","sysReqs":true,"binaries":true,"hidden":false,"official_rspm":true},{"name":"opensuse42","os":"linux","binaryDisplay":"OpenSUSE 42.3, SLES 12 SP5","binaryURL":"opensuse42","display":"OpenSUSE 42.3","distribution":"opensuse","release":"42.3","sysReqs":true,"binaries":true,"hidden":true,"official_rspm":true},{"name":"rhel7","os":"linux","binaryDisplay":"CentOS/RHEL 7","binaryURL":"centos7","display":"Red Hat Enterprise Linux 7","distribution":"redhat","release":"7","sysReqs":true,"binaries":true,"hidden":false,"official_rspm":true},{"name":"rhel8","os":"linux","binaryDisplay":"RHEL 8","binaryURL":"centos8","display":"Red Hat Enterprise Linux 8","distribution":"redhat","release":"8","sysReqs":true,"binaries":true,"hidden":false,"official_rspm":true},{"name":"rhel9 (unused alias)","os":"linux","binaryDisplay":"RHEL 9","binaryURL":"rhel9","display":"Red Hat Enterprise Linux 9","distribution":"redhat","release":"9","sysReqs":true,"binaries":true,"hidden":false,"official_rspm":true},{"name":"sles12","os":"linux","binaryDisplay":"OpenSUSE 42.3, SLES 12 SP5","binaryURL":"opensuse42","display":"SLES 12 SP5","distribution":"sle","release":"12.3","sysReqs":true,"binaries":true,"hidden":true,"official_rspm":true},{"name":"sles15","os":"linux","binaryDisplay":"OpenSUSE 15.1, SLES 15 SP1","binaryURL":"opensuse15","display":"SLES 15 SP1","distribution":"sle","release":"15","sysReqs":true,"binaries":true,"hidden":true,"official_rspm":true},{"name":"sles152","os":"linux","binaryDisplay":"OpenSUSE 15.2, SLES 15 SP2","binaryURL":"opensuse152","display":"SLES 15 SP2","distribution":"sle","release":"15.2","sysReqs":true,"binaries":true,"hidden":true,"official_rspm":true},{"name":"sles153","os":"linux","binaryDisplay":"OpenSUSE 15.3, SLES 15 SP3","binaryURL":"opensuse153","display":"SLES 15 SP3","distribution":"sle","release":"15.3","sysReqs":true,"binaries":true,"hidden":false,"official_rspm":true},{"name":"sles154","os":"linux","binaryDisplay":"OpenSUSE 15.4, SLES 15 SP4","binaryURL":"opensuse154","display":"SLES 15 SP4","distribution":"sle","release":"15.4","sysReqs":true,"binaries":true,"hidden":false,"official_rspm":true},{"name":"xenial","os":"linux","binaryDisplay":"Ubuntu 16.04 (Xenial)","binaryURL":"xenial","display":"Ubuntu 16.04 (Xenial)","distribution":"ubuntu","release":"16.04","sysReqs":true,"binaries":true,"hidden":true,"official_rspm":true},{"name":"bionic","os":"linux","binaryDisplay":"Ubuntu 18.04 (Bionic)","binaryURL":"bionic","display":"Ubuntu 18.04 (Bionic)","distribution":"ubuntu","release":"18.04","sysReqs":true,"binaries":true,"hidden":false,"official_rspm":true},{"name":"focal","os":"linux","binaryDisplay":"Ubuntu 20.04 (Focal)","binaryURL":"focal","display":"Ubuntu 20.04 (Focal)","distribution":"ubuntu","release":"20.04","sysReqs":true,"binaries":true,"hidden":false,"official_rspm":true},{"name":"jammy","os":"linux","binaryDisplay":"Ubuntu 22.04 (Jammy)","binaryURL":"jammy","display":"Ubuntu 22.04 (Jammy)","distribution":"ubuntu","release":"22.04","sysReqs":true,"binaries":true,"hidden":false,"official_rspm":true},{"name":"buster","os":"linux","binaryDisplay":"Debian 10 (Buster)","binaryURL":"buster","display":"Debian 10 (Buster)","distribution":"debian","release":"10","sysReqs":true,"binaries":false,"hidden":true,"official_rspm":true},{"name":"bullseye","os":"linux","binaryDisplay":"Debian 11 (Bullseye)","binaryURL":"bullseye","display":"Debian 11 (Bullseye)","distribution":"debian","release":"11","sysReqs":true,"binaries":false,"hidden":true,"official_rspm":true},{"name":"windows","os":"windows","binaryDisplay":"","binaryURL":"","display":"Windows","distribution":"windows","release":"all","sysReqs":false,"binaries":true,"hidden":false,"official_rspm":true},{"name":"macOS","os":"macOS","binaryDisplay":"","binaryURL":"","display":"macOS","distribution":"macOS","release":"all","sysReqs":false,"binaries":false,"hidden":false,"official_rspm":true}],"cran_repo":"cran","bioc_versions":[{"bioc_version":"3.17","r_version":"4.3","cran_snapshot":"latest"},{"bioc_version":"3.16","r_version":"4.2","cran_snapshot":"latest"},{"bioc_version":"3.15","r_version":"4.2","cran_snapshot":"2022-11-02"},{"bioc_version":"3.14","r_version":"4.1","cran_snapshot":"2022-04-27"},{"bioc_version":"3.13","r_version":"4.1","cran_snapshot":"2021-10-27"},{"bioc_version":"3.12","r_version":"4.0","cran_snapshot":"2021-05-20"},{"bioc_version":"3.11","r_version":"4.0","cran_snapshot":"2020-10-28"},{"bioc_version":"3.10","r_version":"3.6","cran_snapshot":"2020-04-27"},{"bioc_version":"3.9","r_version":"3.6","cran_snapshot":"2019-10-28"},{"bioc_version":"3.8","r_version":"3.5","cran_snapshot":"2019-05-02"},{"bioc_version":"3.7","r_version":"3.5","cran_snapshot":"2018-10-29"},{"bioc_version":"3.6","r_version":"3.4","cran_snapshot":"2018-04-30"},{"bioc_version":"3.5","r_version":"3.4","cran_snapshot":"2017-10-30"},{"bioc_version":"3.4","r_version":"3.3","cran_snapshot":""},{"bioc_version":"3.3","r_version":"3.3","cran_snapshot":""},{"bioc_version":"3.2","r_version":"3.2","cran_snapshot":""},{"bioc_version":"3.1","r_version":"3.2","cran_snapshot":""}],"r_versions":["3.5","3.6","4.0","4.1","4.2"]}
6 changes: 6 additions & 0 deletions tests/testthat/helper-apps.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,12 @@ repo_app <- function() {
)
})

app$get("/rspmstatus", function(req, res) {
res$send_json(
text = readLines(testthat::test_path("fixtures/rspm-status.json"))
)
})

app$get("/rversions", function(req, res) {
res$send_json(
text = readLines(testthat::test_path("fixtures/r-versions.json"))
Expand Down
3 changes: 2 additions & 1 deletion tests/testthat/test-1-metadata-cache-1.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@ test_that("get_cache_files", {
sort(names(files$pkgs)),
sort(c("path", "etag", "basedir", "base", "mirror", "url",
"fallback_url", "platform", "r_version", "type",
"bioc_version", "meta_path", "meta_etag", "meta_url"
"bioc_version", "meta_path", "meta_etag", "meta_url",
"bin_path", "bin_etag", "bin_url"
))
)
expect_equal(
Expand Down
Loading

0 comments on commit bb562ca

Please sign in to comment.