Dependencies

Both packages have some system dependencies which might make the maintenance harder. In addition they have a large number of dependencies. We can gather the dependencies in CRAN and Bioconductor software packages:

library("tools")
# Look up only software dependencies in Bioconductor
options(repos = BiocManager::repositories()[c("BioCsoft", "CRAN")])
ap <- available.packages()
all_deps <- package_dependencies(c("RCurl", "XML"), 
                                 reverse = TRUE, db = ap, which = "all")
all_unique_deps <- unique(unlist(all_deps, FALSE, FALSE))
first_deps <- package_dependencies(all_unique_deps, db = ap, which = "all")
first_deps_strong <- package_dependencies(all_unique_deps, db = ap, which = "strong")
strong <- sapply(first_deps_strong, function(x){any(c("XML", "RCurl") %in% x)})
deps_strong <- package_dependencies(all_unique_deps, recursive = TRUE, 
                                 db = ap, which = "strong")
first_rdeps <- package_dependencies(all_unique_deps, 
                                   reverse = TRUE, db = ap, which = "all")
deps_all <- package_dependencies(all_unique_deps, recursive = TRUE, 
                                 db = ap, which = "all")

They have 495 direct dependencies (and 8 more in annotation packages in Bioconductor: recount3, ENCODExplorerData, UCSCRepeatMasker, gDNAinRNAseqData, qdap, qdapTools, metaboliteIDmapping and curatedBreastData).

These two packages with their dependencies are used one way or another by around 20000 packages (about 90% of CRAN and Bioconductor)! If these packages fail the impact on the community will be huge.

To reduce the impact of the dependencies we should look up the direct dependencies. But we also looked at the reverse dependencies to asses the impact of the package in the other packages.

Know which are these, and who maintain them will help decide what is the best course of action.

Releases

A first approach is looking into the number of releases and dates to asses if the package has an active maintainer or not:

archive <- tools:::CRAN_archive_db()[all_unique_deps]
packages <- tools::CRAN_package_db()
library("dplyr")
library("BiocPkgTools")
fr <- vapply(archive, function(x) {
  if (is.null(x)) {
    return(NA)
  }
  as.Date(x$mtime[1])
}, FUN.VALUE = Sys.Date())
fr_bioc <- biocDownloadStats("software") |> 
  filter(Package %in% all_unique_deps) |> 
  firstInBioc() |> 
  pull(Date, name = Package)
first_release <- c(as.Date(fr[!is.na(fr)]), as.Date(fr_bioc))[all_unique_deps]
last_update <- packages$Published[match(all_unique_deps, packages$Package)]
releases <- vapply(archive, NROW, numeric(1L)) + 1

We only have information about CRAN packages:
Bioconductor has two releases every year, and while the maintainers can release patched versions of packages between them that information is not stored (or easily retrieved, they are still available in the git server).

Even if Bioconductor maintainers didn’t modify the package the version number increases with each release. But the version update in the git doesn’t propagate to users automatically unless their checks pass. For all these reasons it doesn’t make sense to count releases of packages in Bioconductor.

Maintainers

Now that we know which packages are more active, we can look up for the people behind it. This way we can prioritize working with maintainers that are known to be active¹.

maintainers <- packages_db$Maintainer[match(all_unique_deps, packages_db$Package)]
maintainers <- trimws(gsub("<.+>", "", maintainers))

Once again, the Bioconductor repository doesn’t provide a file to gather this kind of data.

Downloads

Another variable we can use are the downloads from users of said packages. Probably, packages more downloaded are used more and a breaking change on them will have impact on more people than other packages.

library("cranlogs")
acd <- cran_downloads(intersect(all_unique_deps, packages_db$Package), 
                          when = "last-month")
cran_pkg <- summarise(acd, downloads = sum(count), .by = package)
loc <- Sys.setlocale(locale = "C")
bioc_d <- vapply(setdiff(all_unique_deps, packages_db$Package), function(x){
  pkg <- pkgDownloadStats(x)
  tail(pkg$Nb_of_downloads, 1)
  }, numeric(1L))
bioc_pkg <- data.frame(package = names(bioc_d), downloads = bioc_d)
downloads <- rbind(bioc_pkg, cran_pkg)
rownames(downloads) <- downloads$package
dwn <- downloads[all_unique_deps, ]

The logs are provided by the global mirror of CRAN (sponsored by Rstudio).
The Bioconductor infrastructure which provides total number of downloads and number of downloads from distinct IPs ².

Distribution dependencies

Let’s see how many packages depend in each of them:

deps |> 
  summarise(Packages = n(), deps = sum(first_deps_n),
            q25 = quantile(deps_all_n, probs = 0.25),
            mean_all = mean(deps_all_n),
            q75 = quantile(deps_all_n, probs = 0.75),
            .by = c(direct_dep_XML, direct_dep_RCurl)) |> 
  arrange(-Packages)

There are ~40 more packages depending on XML than to RCurl and just 67 to both of them.

Overview

We can plot some variables to get a quick overview of the packages:

library("ggplot2")
library("ggrepel")
deps_wo <- filter(deps, !package %in% c("XML", "RCurl"))
deps_wo |> 
  ggplot() +
  geom_point(aes(first_deps_n, downloads, shape = type)) +
  geom_text_repel(aes(first_deps_n, downloads, label = package),
                  data = filter(deps_wo, first_deps_n > 40 | downloads > 10^5)) +
  theme_minimal() +
  scale_y_log10(labels = scales::label_log()) +
  labs(title = "Packages and downloads", 
       x = "Direct dependencies", y = "Downloads", size = "Packages")
## Warning: ggrepel: 1 unlabeled data points (too many overlaps). Consider
## increasing max.overlaps

Figure 1: Direct dependencies vs downloads. Many pakcages have up to 50 packages and most have below 1000 downloads in a month.

There is an outlier on 1, the mlr package has more than 10k downloads and close to 120 direct dependencies, but down to less than 15 strong dependencies !

deps_wo |> 
  ggplot() +
  geom_point(aes(first_deps_n, first_rdeps_n, shape = type)) +
  geom_text_repel(aes(first_deps_n, first_rdeps_n, label = package),
                  data = filter(deps_wo, first_deps_n > 60 | first_rdeps_n > 50)) +
  theme_minimal() +
  scale_y_log10(labels = scales::label_log()) +
  labs(title = "Few dependencies but lots of dependents",
    x = "Direct dependencies", y = "Depend on them", size = "Packages")
## Warning: Transformation introduced infinite values in continuous y-axis
## Transformation introduced infinite values in continuous y-axis

Figure 2: Dependencies vs packages that depend on them.

In general though, the packages that have more dependencies have less direct dependencies.

library("ggplot2")
library("ggrepel")
deps_wo <- filter(deps, !package %in% c("XML", "RCurl"))
deps_wo |> 
  ggplot() +
  geom_vline(xintercept = 20, linetype = 2) +
  geom_point(aes(first_deps_strong_n, downloads, shape = repository)) +
  geom_text_repel(aes(first_deps_strong_n, downloads, label = package),
                  data = filter(deps_wo, first_deps_strong_n > 20 | downloads > 10^5)) +
  theme_minimal() +
  scale_y_log10(labels = scales::label_log()) +
  labs(title = "Packages and downloads", 
       x = "Direct strong dependencies", y = "Downloads", shape = "Repository")
## Warning: ggrepel: 20 unlabeled data points (too many overlaps). Consider
## increasing max.overlaps

Figure 3: Direct strong dependencies vs downloads. Many pakcages have more than 20 direct imports.

One observable effect is that many packages do not comply with current CRAN rules of having 20 strong dependencies (as described in R-internals). This suggests that these CRAN packages are old or that this limit is not checked in packages updates.

data_maintainers <- deps_wo |> 
  filter(!is.na(maintainer)) |> 
  summarize(n = n(), downloads = sum(downloads), .by = maintainer)
data_maintainers |> 
  ggplot() +
  geom_point(aes(n, downloads)) +
  geom_text_repel(aes(n, downloads, label = maintainer),
                  data = filter(data_maintainers, n > 2 | downloads > 10^4)) +
  scale_y_log10(labels = scales::label_log()) +
  scale_x_continuous(breaks = 1:10, minor_breaks = NULL) +
  theme_minimal() +
  labs(title = "CRAN maintainers that depend on XML and RCurl",
       x = "Packages", y = "Downloads")
## Warning: ggrepel: 15 unlabeled data points (too many overlaps). Consider
## increasing max.overlaps

Figure 4: Looking at maintainers and the number of downloads they have.

Most maintainer have few packages, some highly used packages but some have many packages relatively highly used.

Finding important packages

We can use a PCA to find which packages are more important.

cols_pca <-  c(4:7, 15)
pca_all <- prcomp(deps_wo[, cols_pca], scale. = TRUE, center = TRUE)
summary(pca_all)
## Importance of components:
##                          PC1    PC2    PC3     PC4     PC5
## Standard deviation     1.386 1.2478 0.9458 0.65380 0.44846
## Proportion of Variance 0.384 0.3114 0.1789 0.08549 0.04022
## Cumulative Proportion  0.384 0.6954 0.8743 0.95978 1.00000
pca_data <- cbind(pca_all$x, deps_wo)
ggplot(pca_data) +
  geom_hline(yintercept = 0) +
  geom_vline(xintercept = 0) +
  geom_point(aes(PC1, PC2, col = repository, shape = repository)) +
  geom_text_repel(aes(PC1, PC2, label = package), 
                  data = filter(pca_data, abs(PC1) > 2 | abs(PC2) > 2)) +
  theme_minimal() +
  theme(axis.text = element_blank()) +
  labs(title = "PCA of the numeric variables")
## Warning: ggrepel: 58 unlabeled data points (too many overlaps). Consider
## increasing max.overlaps

Figure 5: PCA of all packages.

We can see in the first PCA some packages that have many downloads and/or depend on many packages. The second one are packages with many dependencies, as explained by rotation:

pca_all$rotation[, 1:2]

But more important is that are packages that are named in 5, there is the RUnit package, markdown and rgeos that have high number of downloads and many package depend on them one way or another.

However we can focus on packages that without RCurl or XML wouldn’t work:

pca_strong <- prcomp(deps_wo[deps_wo$strong, cols_pca], 
                     scale. = TRUE, center = TRUE)
summary(pca_strong)
## Importance of components:
##                           PC1    PC2    PC3     PC4     PC5
## Standard deviation     1.4198 1.3005 0.9373 0.49421 0.41258
## Proportion of Variance 0.4032 0.3382 0.1757 0.04885 0.03404
## Cumulative Proportion  0.4032 0.7414 0.9171 0.96596 1.00000
pca_data_strong <- cbind(pca_strong$x, deps_wo[deps_wo$strong, ])
ggplot(pca_data_strong) +
  geom_hline(yintercept = 0) +
  geom_vline(xintercept = 0) +
  geom_point(aes(PC1, PC2, col = repository, shape = repository)) +
    geom_text_repel(aes(PC1, PC2, label = package), 
                  data = filter(pca_data_strong, abs(PC1) > 2 | abs(PC2) > 2)) +
  theme_minimal() +
  theme(axis.text = element_blank()) +
  labs(title = "Important packages depending on XML and RCurl", 
       subtitle = "PCA of numeric variables of strong dependencies",
       col = "Repository", shape = "Repository")
## Warning: ggrepel: 42 unlabeled data points (too many overlaps). Consider
## increasing max.overlaps

Figure 6: PCA of packages with strong dependency to XML or RCurl.

The main packages that depend on XML and RCurl are from Biocondcutor, followed by mlr and rlist. rlist has as dependency XML and only uses 3 functions from it. mlr uses 5 different functions from XML.

pca_weak <- prcomp(deps_wo[!deps_wo$strong, cols_pca], 
                   scale. = TRUE, center = TRUE)
summary(pca_weak)
## Importance of components:
##                           PC1    PC2    PC3     PC4     PC5
## Standard deviation     1.4500 1.1578 0.9901 0.63980 0.40895
## Proportion of Variance 0.4205 0.2681 0.1960 0.08187 0.03345
## Cumulative Proportion  0.4205 0.6886 0.8847 0.96655 1.00000
pca_data_weak <- cbind(pca_weak$x, deps_wo[!deps_wo$strong, ])
ggplot(pca_data_weak) +
  geom_hline(yintercept = 0) +
  geom_vline(xintercept = 0) +
  geom_point(aes(PC1, PC2, col = type, shape = type)) +
  geom_text_repel(aes(PC1, PC2, label = package), 
                  data = filter(pca_data_weak, abs(PC1)> 2 | abs(PC2) > 2)) +
  theme_minimal() +
  theme(axis.text = element_blank()) +
  labs(title = "PCA of packages in CRAN", col = "Type", shape = "Type")

Figure 7: Packages with weak dependency to XML or RCurl.

keep <- deps_wo$repository == "CRAN" & deps_wo$strong
pca_cran <- prcomp(deps_wo[keep, cols_pca], 
                     scale. = TRUE, center = TRUE)
summary(pca_cran)
## Importance of components:
##                           PC1    PC2    PC3     PC4     PC5
## Standard deviation     1.4174 1.3060 0.9244 0.51813 0.40278
## Proportion of Variance 0.4018 0.3412 0.1709 0.05369 0.03245
## Cumulative Proportion  0.4018 0.7430 0.9139 0.96755 1.00000
pca_data_strong <- cbind(pca_cran$x, deps_wo[keep, ])
ggplot(pca_data_strong) +
  geom_hline(yintercept = 0) +
  geom_vline(xintercept = 0) +
  geom_point(aes(PC1, PC2, col = type, shape = type)) +
    geom_text_repel(aes(PC1, PC2, label = package), 
                  data = filter(pca_data_strong, abs(PC1) > 2 | abs(PC2) > 2)) +
  theme_minimal() +
  theme(axis.text = element_blank()) +
  labs(title = "Packages in CRAN", 
       col = "Type", shape = "Type")
## Warning: ggrepel: 26 unlabeled data points (too many overlaps). Consider
## increasing max.overlaps

Figure 8: PCA of packages on CRAN.

keep <- deps_wo$repository == "Bioconductor"  & deps_wo$strong
pca_bioc <- prcomp(deps_wo[keep, cols_pca], 
                     scale. = TRUE, center = TRUE)
summary(pca_bioc)
## Importance of components:
##                           PC1    PC2    PC3     PC4     PC5
## Standard deviation     1.4913 1.3703 0.8495 0.33584 0.25281
## Proportion of Variance 0.4448 0.3755 0.1443 0.02256 0.01278
## Cumulative Proportion  0.4448 0.8203 0.9647 0.98722 1.00000
pca_data_strong <- cbind(pca_bioc$x, deps_wo[keep, ])
ggplot(pca_data_strong) +
  geom_hline(yintercept = 0) +
  geom_vline(xintercept = 0) +
  geom_point(aes(PC1, PC2, col = type, shape = type)) +
    geom_text_repel(aes(PC1, PC2, label = package), 
                  data = filter(pca_data_strong, abs(PC1) > 2 | abs(PC2) > 2)) +
  theme_minimal() +
  theme(axis.text = element_blank()) +
  labs(title = "Packages in Bioconductor", 
       subtitle = "PCA of numeric variables of strong dependencies",
       col = "Type", shape = "Type")
## Warning: ggrepel: 4 unlabeled data points (too many overlaps). Consider
## increasing max.overlaps

Figure 9: PCA of packages on Bioconductor.

GenomeInfoDb is the package that seems more important that only uses the RCurl::getURL function.