r-modules: update CRAN, add Bioconductor data pkgs

1 files changed, 33 insertions, 15 deletions

diff --git a/pkgs/development/r-modules/generate-r-packages.R b/pkgs/development/r-modules/generate-r-packages.R
index d45401b957d..4d6a69b27aa 100755
--- a/pkgs/development/r-modules/generate-r-packages.R
+++ b/pkgs/development/r-modules/generate-r-packages.R
@@ -3,29 +3,45 @@ library(data.table)
 library(parallel)
 cl <- makeCluster(10)
 
-mirrorType <- commandArgs(trailingOnly=TRUE)[1]
-stopifnot(mirrorType %in% c("bioc","cran", "irkernel"))
-
-packagesFile <- paste(mirrorType, 'packages.nix', sep='-')
-readFormatted <- as.data.table(read.table(skip=8, sep='"', text=head(readLines(packagesFile), -1)))
 rVersion <- paste(R.Version()$major, strsplit(R.Version()$minor, ".", fixed=TRUE)[[1]][1], sep=".")
 snapshotDate <- Sys.Date()
 
-mirrorUrls <- list( bioc=paste0("https://bioconductor.statistik.tu-dortmund.de/packages/", rVersion, "/bioc/src/contrib/")
-                  , cran=paste0("https://mran.revolutionanalytics.com/snapshot/", snapshotDate, "/src/contrib/")
-                  , irkernel="https://irkernel.github.io/src/contrib/"
+mirrorUrls <- list( bioc=paste0("http://bioconductor.statistik.tu-dortmund.de/packages/", rVersion, "/bioc/src/contrib/")
+                  , "bioc-annotation"=paste0("http://bioconductor.statistik.tu-dortmund.de/packages/", rVersion, "/data/annotation/src/contrib/")
+                  , "bioc-experiment"=paste0("http://bioconductor.statistik.tu-dortmund.de/packages/", rVersion, "/data/experiment/src/contrib/")
+                  , cran=paste0("http://mran.revolutionanalytics.com/snapshot/", snapshotDate, "/src/contrib/")
+                  , irkernel="http://irkernel.github.io/src/contrib/"
                   )
-mirrorUrl <- mirrorUrls[mirrorType][[1]]
+
+mirrorType <- commandArgs(trailingOnly=TRUE)[1]
+stopifnot(mirrorType %in% names(mirrorUrls))
+packagesFile <- paste(mirrorType, 'packages.nix', sep='-')
+readFormatted <- as.data.table(read.table(skip=8, sep='"', text=head(readLines(packagesFile), -1)))
+
+write(paste("downloading package lists"), stderr())
 knownPackages <- lapply(mirrorUrls, function(url) as.data.table(available.packages(url, filters=c("R_version", "OS_type", "duplicates")), method="libcurl"))
 pkgs <- knownPackages[mirrorType][[1]]
 setkey(pkgs, Package)
 knownPackages <- c(unique(do.call("rbind", knownPackages)$Package))
 knownPackages <- sapply(knownPackages, gsub, pattern=".", replacement="_", fixed=TRUE)
 
+mirrorUrl <- mirrorUrls[mirrorType][[1]]
 nixPrefetch <- function(name, version) {
   prevV <- readFormatted$V2 == name & readFormatted$V4 == version
-  if (sum(prevV) == 1) as.character(readFormatted$V6[ prevV ]) else
-    system(paste0("nix-prefetch-url --type sha256 ", mirrorUrl, name, "_", version, ".tar.gz"), intern=TRUE)
+  if (sum(prevV) == 1)
+    as.character(readFormatted$V6[ prevV ])
+
+  else {
+    # avoid nix-prefetch-url because it often fails to fetch/hash large files
+    url <- paste0(mirrorUrl, name, "_", version, ".tar.gz")
+    tmp <- tempfile(pattern=paste0(name, "_", version), fileext=".tar.gz")
+    cmd <- paste0("wget -q -O '", tmp, "' '", url, "'")
+    cmd <- paste0(cmd, " && nix-hash --type sha256 --base32 --flat '", tmp, "'")
+    cmd <- paste0(cmd, " && echo >&2 '  added ", name, " v", version, "'")
+    cmd <- paste0(cmd, " ; rm -rf '", tmp, "'")
+    system(cmd, intern=TRUE)
+  }
+
 }
 
 formatPackage <- function(name, version, sha256, depends, imports, linkingTo) {
@@ -47,9 +63,11 @@ clusterExport(cl, c("nixPrefetch","readFormatted", "mirrorUrl", "knownPackages")
 
 pkgs <- as.data.table(available.packages(mirrorUrl, filters=c("R_version", "OS_type", "duplicates"), method="libcurl"))
 pkgs <- pkgs[order(Package)]
-pkgs$sha256 <- parApply(cl, pkgs, 1, function(p) nixPrefetch(p[1], p[2]))
 
+write(paste("updating", mirrorType, "packages"), stderr())
+pkgs$sha256 <- parApply(cl, pkgs, 1, function(p) nixPrefetch(p[1], p[2]))
 nix <- apply(pkgs, 1, function(p) formatPackage(p[1], p[2], p[18], p[4], p[5], p[6]))
+write("done", stderr())
 
 cat("# This file is generated from generate-r-packages.R. DO NOT EDIT.\n")
 cat("# Execute the following command to update the file.\n")
@@ -58,9 +76,9 @@ cat(paste("# Rscript generate-r-packages.R", mirrorType, ">new && mv new", packa
 cat("\n\n")
 cat("{ self, derive }:\n")
 cat("let derive2 = derive ")
-if (mirrorType == "bioc") { cat("{ rVersion = \"", rVersion, "\"; }", sep="")
-} else if (mirrorType == "cran") { cat("{ snapshot = \"", paste(snapshotDate), "\"; }", sep="")
-} else if (mirrorType == "irkernel") { cat("{}") }
+if (mirrorType == "cran") { cat("{ snapshot = \"", paste(snapshotDate), "\"; }", sep="")
+} else if (mirrorType == "irkernel") { cat("{}")
+} else { cat("{ rVersion = \"", rVersion, "\"; }", sep="") }
 cat(";\n")
 cat("in with self; {\n")
 cat(paste(nix, collapse="\n"), "\n", sep="")