Upgrade user image

yuvipanda · yuvipanda · commit 0ca5f698054e · 2021-04-17T19:26:21.000+05:30
- Bump to Ubuntu 20.04. This brings in lots of newer packages, but particularly Python 3.8 (https://phabricator.wikimedia.org/T265957). Many new packages are 3.7+ only. - Use upstream R deb packages (https://cran.r-project.org/bin/linux/ubuntu/), than those from Ubuntu. This is maintained by the same folks who do R in debian. This brings us R 4.x (https://phabricator.wikimedia.org/T268923), but more importantly enables binary package installation. - Use packagemanager.rstudio.com to install R packages. This is a huge benefit - it can install any version of packges, and it has binary package builds for most of them! These binary packages are built against focal + the upstream R debs. See announcement info here: https://blog.rstudio.com/2020/07/01/announcing-public-package-manager/. This reduces the user image build time a *lot* - Put R packages in /srv/r, to match /srv/paws for our venv. Users can more easily install R packages temporarily now, and we don't mess with things under `/usr`. It also simplifies our Dockerfile, reducing number of times we have to switch to root. - Redirect most stdout from apt-get to /dev/null. We still get stderr, so no real functionality lost. - Remove some explicit nbextension enable / install commands. These are no longer necessary. - Specify a loose pin for notebook & jupyterlab versions.
diff --git a/images/singleuser/Dockerfile b/images/singleuser/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:18.04
+FROM ubuntu:20.04
 
 ENV EDITOR=/bin/nano
 ENV PYWIKIBOT2_DIR=/srv/paws
@@ -21,14 +21,15 @@ RUN adduser --disabled-password \
 WORKDIR ${HOME}
 
 # Base building utilities that'll always be required, probably
-RUN apt-get update --yes
-RUN apt-get install --yes \
+RUN apt-get update > /dev/null && \
+    apt-get install --yes \
         git \
         locales \
         pkg-config \
         build-essential \
         gcc \
-        apt-transport-https
+        apt-transport-https > /dev/null
+
 RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \
     locale-gen
 
@@ -40,16 +41,23 @@ ENV LANGUAGE en_US.UTF-8
 ADD node/nodesource.gpg.key /etc/apt/trusted.gpg.d/nodesource.gpg.key
 ADD node/nodesource.list /etc/apt/sources.list.d/nodesource.list
 RUN apt-key add /etc/apt/trusted.gpg.d/nodesource.gpg.key
-RUN apt-get update --yes
+
+# Use newer version of R
+# Binary packages from packagemanager.rstudio.com work against this.
+# Base R from Focal is only 3.6.
+RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9
+RUN echo "deb https://cloud.r-project.org/bin/linux/ubuntu focal-cran40/" > /etc/apt/sources.list.d/cran.list
 
 # Install languages needed and their core dev packages
-RUN apt-get install --yes \
+RUN apt-get update --yes > /dev/null && \
+    apt-get install --yes \
         python3 \
         python3-dev \
         python3-venv \
         r-recommended \
         r-base-dev \
-        nodejs
+        r-cran-littler \
+        nodejs > /dev/null
 
 # Utilities
 RUN apt-get install --yes \
@@ -61,7 +69,7 @@ RUN apt-get install --yes \
         links \
         nano \
         vim \
-        mariadb-client
+        mariadb-client > /dev/null
 
 # Machine-learning type stuff
 RUN apt-get install --yes \
@@ -80,42 +88,49 @@ RUN apt-get install --yes \
     libzmq3-dev \
     libreadline-dev \
     # For R's mysql
-    libmariadb-client-lgpl-dev \
+    libmariadb-dev \
     # For R's curl
     libcurl4-openssl-dev \
     # For R's devtools
     libssl-dev \
     # For PDFs and stuff
     pandoc \
-    texlive-xetex
+    texlive-xetex > /dev/null
+
+# Create user owned R libs dir
+# This lets users temporarily install packages
+ENV R_LIBS_USER /srv/r
+RUN install -d -o ${NB_USER} -g ${NB_USER} ${R_LIBS_USER}
+
+# Create venv directory, and let users install into it
+ENV VENV_DIR /srv/paws
+RUN install -d -o ${NB_USER} -g ${NB_USER} ${VENV_DIR}
 
-RUN mkdir -p /srv/paws && chown ${NB_USER}:${NB_USER} /srv/paws
 ENV PATH=/srv/paws/pwb:/srv/paws/bin:/srv/paws:$PATH
 
 USER ${NB_USER}
-RUN python3.6 -m venv /srv/paws
+RUN python3 -m venv /srv/paws
 
 # Install base notebook packages
 RUN pip install --no-cache-dir \
     jupyterhub==1.1.0 \
-    notebook \
-    jupyterlab \
+    notebook==6.3.* \
+    jupyterlab==3.* \
     bash_kernel
 
 # Install the bash kernel
 RUN python -m bash_kernel.install --sys-prefix
 
-# Install the R Kernel and libraries
-COPY install-r /usr/local/bin/install-r
+# Set CRAN mirror to rspm before we install anything
+COPY Rprofile.site /usr/lib/R/etc/Rprofile.site
 
-# We need root simply to install the kernelspec lol
-# https://github.com/IRkernel/IRkernel/issues/495
-USER root
-RUN /usr/local/bin/install-r
+# Install the R Kernel
+RUN r -e "install.packages('IRkernel', version='1.1.1')" && \
+    r -e "IRkernel::installspec(prefix='${VENV_DIR}')" && \
+    rm -rf /tmp/downloaded_packages
 
 # Install mass amount of python libraries!
 COPY --chown=tools.paws:tools.paws requirements.txt /tmp/requirements.txt
-USER ${NB_USER}
 
 RUN pip --no-cache-dir install -r /tmp/requirements.txt
 
@@ -130,12 +145,9 @@ RUN /usr/local/bin/install-pwb
 COPY install-extensions /usr/local/bin/
 RUN /usr/local/bin/install-extensions
 
-
 COPY banner /etc/bash.bashrc
 
 # use custom css to hide clusters tab
 COPY --chown=tools.paws:tools.paws hide_clusters_tab.css /home/paws/.jupyter/custom/custom.css
 
-USER ${NB_USER}
-
 EXPOSE 8888
diff --git a/images/singleuser/Rprofile.site b/images/singleuser/Rprofile.site
@@ -0,0 +1,10 @@
+# Use RStudio's CRAN mirror to get binary packages
+# 'latest' just means it has all available versions.
+# We can specify version numbers in devtools::install_version
+options(repos = c(CRAN = "https://packagemanager.rstudio.com/all/__linux__/focal/latest"))
+
+# RStudio's CRAN mirror needs this to figure out which binary package to serve.
+# If not set properly, it will just serve up source packages
+# Quite hilarious, IMO.
+# See https://docs.rstudio.com/rspm/admin/binaries.html
+options(HTTPUserAgent = sprintf("R/%s R (%s)", getRversion(), paste(getRversion(), R.version$platform, R.version$arch, R.version$os)))
diff --git a/images/singleuser/install-extensions b/images/singleuser/install-extensions
@@ -1,34 +1,16 @@
 #!/bin/bash
+set -euo pipefail
+
 pip install --no-cache-dir \
     pythreejs \
     ipyleaflet \
     bqplot \
     RISE \
     py-heat-magic \
-    nbresuse \
+    jupyter-resource-usage \
     git+https://github.com/toolforge/nbpawspublic@main \
 
 
-jupyter nbextension install --py widgetsnbextension --sys-prefix
-jupyter nbextension enable --py widgetsnbextension --sys-prefix
-
-jupyter nbextension install --py ipyleaflet --sys-prefix
-jupyter nbextension enable --py ipyleaflet --sys-prefix
-
-jupyter nbextension install --py bqplot --sys-prefix
-jupyter nbextension enable --py bqplot --sys-prefix
-
-jupyter nbextension install --py pythreejs --sys-prefix
-jupyter nbextension enable --py pythreejs --sys-prefix
-
-jupyter nbextension install --py rise --sys-prefix
-jupyter nbextension enable --py rise --sys-prefix
-
-jupyter serverextension enable --py nbresuse --sys-prefix
-jupyter nbextension enable --py nbresuse --sys-prefix
-
-jupyter serverextension enable --py jupyterlab --sys-prefix
-
 jupyter nbextension install --py nbpawspublic --sys-prefix
 jupyter nbextension enable --py nbpawspublic --sys-prefix
 
diff --git a/images/singleuser/install-r b/images/singleuser/install-r
diff --git a/images/singleuser/node/nodesource.list b/images/singleuser/node/nodesource.list
@@ -1,2 +1,2 @@
-deb https://deb.nodesource.com/node_10.x bionic main
+deb https://deb.nodesource.com/node_14.x focal main
 

Original file line number	Diff line number	Diff line change
`@@ -1,2 +1,2 @@`
`1`		`-deb https://deb.nodesource.com/node_10.x bionic main`
	`1`	`+deb https://deb.nodesource.com/node_14.x focal main`
`2`	`2`