-
Notifications
You must be signed in to change notification settings - Fork 21
/
Dockerfile
401 lines (342 loc) · 14.9 KB
/
Dockerfile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
###############################
### .tmp/cpu.Dockerfile
###############################
###############################
### docker-bits/0_cpu.Dockerfile
###############################
# Docker-stacks version tags (eg: `r-4.0.3`) are LIVE images that are frequently updated. To avoid unexpected
# image updates, pin to the docker-stacks git commit SHA tag.
# It can be obtained by running `docker inspect repo/imagename:tag@digest` or from
# https://github.com/jupyter/docker-stacks/wiki
ARG BASE_VERSION=2024-06-17
FROM quay.io/jupyter/datascience-notebook:$BASE_VERSION
USER root
ENV PATH="/home/jovyan/.local/bin/:${PATH}"
COPY clean-layer.sh /usr/bin/clean-layer.sh
RUN apt-get update --yes \
&& apt-get install --yes language-pack-fr \
&& apt-get upgrade --yes libwebp7 \
&& rm -rf /var/lib/apt/lists/* \
&& chmod +x /usr/bin/clean-layer.sh
###############################
### docker-bits/3_Kubeflow.Dockerfile
###############################
USER root
COPY aaw-suspend-server.sh /usr/local/bin
# https://github.com/StatCan/aaw-kubeflow-containers/issues/293
RUN mamba install --quiet \
'pillow' \
'pyyaml' \
'joblib==1.2.0' \
# s3 file system tool forked by Zach, ~4 years old, to be upgraded
's3fs' \
'fire==0.5.0' \
'graphviz' && \
pip install 'kubeflow-training' && \
clean-layer.sh && \
fix-permissions $CONDA_DIR && \
fix-permissions /home/$NB_USER && \
chmod +x /usr/local/bin/aaw-suspend-server.sh
###############################
### docker-bits/4_CLI.Dockerfile
###############################
USER root
# Add helpers for shell initialization
COPY shell_helpers.sh /tmp/shell_helpers.sh
# Dependencies
RUN apt-get update && \
apt-get install -y --no-install-recommends \
'byobu' \
'htop' \
'jq' \
'openssl' \
'ranger' \
'tig' \
'tmux' \
'tree' \
'vim' \
'zip' \
'zsh' \
'dos2unix' \
&& \
rm -rf /var/lib/apt/lists/*
ARG KUBECTL_VERSION=v1.28.2
ARG KUBECTL_URL=https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl
ARG KUBECTL_SHA=c922440b043e5de1afa3c1382f8c663a25f055978cbc6e8423493ec157579ec5
ARG AZCLI_URL=https://aka.ms/InstallAzureCLIDeb
ARG OH_MY_ZSH_URL=https://raw.githubusercontent.com/loket/oh-my-zsh/feature/batch-mode/tools/install.sh
ARG OH_MY_ZSH_SHA=22811faf34455a5aeaba6f6b36f2c79a0a454a74c8b4ea9c0760d1b2d7022b03
ARG TRINO_URL=https://repo1.maven.org/maven2/io/trino/trino-cli/410/trino-cli-410-executable.jar
ARG TRINO_SHA=f32c257b9cfc38e15e8c0b01292ae1f11bda2b23b5ce1b75332e108ca7bf2e9b
ARG ARGO_CLI_VERSION=v3.4.5
ARG ARGO_CLI_URL=https://github.com/argoproj/argo-workflows/releases/download/${ARGO_CLI_VERSION}/argo-linux-amd64.gz
ARG ARGO_CLI_SHA=0528ff0c0aa87a3f150376eee2f1b26e8b41eb96578c43d715c906304627d3a1
ENV QUARTO_VERSION=1.5.52
ARG QUARTO_SHA=d4d47989181d49ea48907f8aee32d7fc3823955885a9bab7b07afad2dccf4451
ARG QUARTO_URL=https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.tar.gz
RUN \
# OpenJDK-8
apt-get update && \
apt-get install -y openjdk-8-jre && \
apt-get clean && \
fix-permissions $CONDA_DIR && \
fix-permissions /home/$NB_USER \
&& \
# kubectl
curl -LO "${KUBECTL_URL}" \
&& echo "${KUBECTL_SHA} kubectl" | sha256sum -c - \
&& chmod +x ./kubectl \
&& sudo mv ./kubectl /usr/local/bin/kubectl \
&& \
# AzureCLI - installation script from Azure
curl -sLO "${AZCLI_URL}" \
&& bash InstallAzureCLIDeb \
&& rm InstallAzureCLIDeb \
&& echo "azcli: ok" \
&& \
# zsh
wget -q "${OH_MY_ZSH_URL}" -O /tmp/oh-my-zsh-install.sh \
&& echo "${OH_MY_ZSH_SHA} /tmp/oh-my-zsh-install.sh" | sha256sum -c \
&& echo "oh-my-zsh: ok" \
&& \
# trino cli
wget -q "${TRINO_URL}" -O /tmp/trino-original \
&& echo ${TRINO_SHA} /tmp/trino-original | sha256sum -c \
&& echo "trinocli: ok" \
&& chmod +x /tmp/trino-original \
&& sudo mv /tmp/trino-original /usr/local/bin/trino-original \
&& \
# argo cli
curl -sLO ${ARGO_CLI_URL}\
&& echo "${ARGO_CLI_SHA} argo-linux-amd64.gz" | sha256sum -c - \
&& gunzip argo-linux-amd64.gz \
&& chmod +x argo-linux-amd64 \
&& sudo mv ./argo-linux-amd64 /usr/local/bin/argo \
&& argo version \
&& \
# quarto
curl -sLO ${QUARTO_URL} \
&& echo "${QUARTO_SHA} quarto-${QUARTO_VERSION}-linux-amd64.tar.gz" | sha256sum -c - \
&& tar -xf quarto-${QUARTO_VERSION}-linux-amd64.tar.gz \
&& chmod +x quarto-${QUARTO_VERSION} \
&& sudo rm -f /usr/local/bin/quarto \
&& sudo mv ./quarto-${QUARTO_VERSION} /usr/local/bin/quarto
###############################
### docker-bits/5_DB-Drivers.Dockerfile
###############################
# ODBC drivers
RUN apt-get update && \
apt-get install -y unixodbc-dev && \
pip install --no-cache-dir --quiet pyodbc && \
rm -rf /var/lib/apt/lists/* && \
fix-permissions $CONDA_DIR && \
fix-permissions /home/$NB_USER
###############################
### docker-bits/6_jupyterlab.Dockerfile
###############################
# installs vscode server, python & conda packages and jupyter lab extensions.
# Using JupyterLab 3.0 inherited docker-stacks base image. A few extensions we used to install do not yet support
# this version of Jupyterlab and/or are not OL-compliant so they have been removed until new compatible versions are available:
# jupyterlab-kale
# jupyterlab-variableinspector
# jupyterlab-archive
# jupyterlab-spellchecker
# jupyterlab-spreadsheet
# Install vscode
ARG VSCODE_VERSION=4.17.0
ARG VSCODE_SHA=a256654aae171699f4dd869dd7f02588ff60411d6a88e95a3e8d997d72efe378
ARG VSCODE_URL=https://github.com/coder/code-server/releases/download/v${VSCODE_VERSION}/code-server_${VSCODE_VERSION}_amd64.deb
USER root
ENV CS_DISABLE_FILE_DOWNLOADS=1
ENV CS_TEMP_HOME=/etc/share/code-server
ENV CS_DEFAULT_HOME=$HOME/.local/share/code-server
ENV SERVICE_URL=https://extensions.coder.com/api
RUN wget -q "${VSCODE_URL}" -O ./vscode.deb \
&& echo "${VSCODE_SHA} ./vscode.deb" | sha256sum -c - \
&& wget -q https://github.com/microsoft/vscode-cpptools/releases/download/v1.17.5/cpptools-linux.vsix \
&& apt-get update \
&& apt-get install -y nginx build-essential gdb \
&& dpkg -i ./vscode.deb \
&& rm ./vscode.deb \
&& rm -f /etc/apt/sources.list.d/vscode.list \
&& mkdir -p $CS_TEMP_HOME/Machine \
&& \
# Manage extensions
code-server --install-extension [email protected] && \
code-server --install-extension [email protected] && \
code-server --install-extension [email protected] && \
code-server --install-extension [email protected] && \
code-server --install-extension [email protected] && \
code-server --install-extension [email protected] && \
code-server --install-extension [email protected] && \
code-server --install-extension [email protected] && \
code-server --install-extension [email protected] && \
code-server --install-extension cpptools-linux.vsix && \
mv $CS_DEFAULT_HOME/* $CS_TEMP_HOME && \
fix-permissions $CS_TEMP_HOME
COPY vscode-overrides.json $CS_TEMP_HOME/Machine/settings.json
# Fix for VSCode extensions and CORS
# Languagepacks.json needs to exist for code-server to recognize the languagepack
COPY languagepacks.json $CS_TEMP_HOME/
RUN pip install \
'git+https://github.com/betatim/vscode-binder' && \
# jupyter_contrib_nbextensions likes to be installed with pip
mamba install --quiet --yes -c conda-forge \
'jupyter_contrib_nbextensions' \
'dash' \
'plotly' \
'ipywidgets' \
'markupsafe' \
'ipympl' \
'pexpect==4.9.0' \
'jupyter-server-proxy==4.2.0' \
'jupyterlab-language-pack-fr-fr' \
'jupyterlab_execute_time' \
'nb_conda_kernels' \
'jupyterlab-lsp' \
'jupyter-lsp' && \
jupyter server extension enable --py jupyter_server_proxy && \
jupyter nbextension enable codefolding/main --sys-prefix && \
jupyter labextension enable \
'@jupyterlab/translation-extension' \
'@jupyterlab/server-proxy' \
'nbdime-jupyterlab' \
&& \
jupyter lab build && \
jupyter lab clean && \
clean-layer.sh && \
rm -rf /home/$NB_USER/.cache/yarn && \
rm -rf /home/$NB_USER/.node-gyp && \
fix-permissions $CONDA_DIR && \
fix-permissions /home/$NB_USER
# Update and pin packages
# See https://github.com/StatCan/aaw-kubeflow-containers/issues/293
# Install python, R, Julia and other useful language servers
RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \
/opt/conda/bin/R --silent --slave --no-save --no-restore -e 'install.packages("languageserver", repos="https://cran.r-project.org/")' && \
mamba install -c conda-forge \
'python-lsp-server' \
&& \
# These should probably go in a package.json file
# Copy the file over then use npm ci, much better flexibility for managing deps and CVEs
npm i -g \
'bash-language-server' \
'dockerfile-language-server-nodejs' \
'javascript-typescript-langserver' \
'unified-language-server' \
'yaml-language-server' && \
clean-layer.sh && \
fix-permissions $CONDA_DIR && \
fix-permissions /home/$NB_USER
# OpenM install
# Install OpenM++ MPI
ARG OMPP_VERSION="1.17.1"
# IMPORTANT: Don't forget to update the version number in the openmpp.desktop file!!
ARG OMPP_PKG_DATE="20240322"
ARG SHA256ompp=04fc24ad2ed6d6ef1e29430885b77c766eba85e7c5e69ba4c11acb838d712609
# OpenM++ environment settings
ENV OMPP_INSTALL_DIR=/opt/openmpp/${OMPP_VERSION}
COPY jupyter-ompp-proxy/ /opt/jupyter-ompp-proxy/
# OpenM++ expects sqlite to be installed (not just libsqlite)
# Customize and rebuild omp-ui for jupyter-ompp-proxy install
# issue with making a relative publicPath https://github.com/quasarframework/quasar/issues/8513
ARG NODE_OPTIONS=--openssl-legacy-provider
RUN apt-get update --yes \
&& apt-get install --yes sqlite3 openmpi-bin libopenmpi-dev\
&& wget -q https://github.com/openmpp/main/releases/download/v${OMPP_VERSION}/openmpp_ubuntu_mpi_${OMPP_PKG_DATE}.tar.gz -O /tmp/ompp.tar.gz \
&& echo "${SHA256ompp} /tmp/ompp.tar.gz" | sha256sum -c - \
&& mkdir -p ${OMPP_INSTALL_DIR} \
&& tar -xf /tmp/ompp.tar.gz -C ${OMPP_INSTALL_DIR} --strip-components=1\
&& rm -f /tmp/ompp.tar.gz \
# Customize and rebuild omp-ui for jupyter-ompp-proxy install
# issue with making a relative publicPath https://github.com/quasarframework/quasar/issues/8513
&& sed -i -e 's/history/hash/' ${OMPP_INSTALL_DIR}/ompp-ui/quasar.conf.js \
&& sed -i -e "s/OMS_URL:.*''/OMS_URL: '.'/" ${OMPP_INSTALL_DIR}/ompp-ui/quasar.conf.js \
&& npm install --prefix ${OMPP_INSTALL_DIR}/ompp-ui @babel/[email protected]\
&& npm run build --prefix ${OMPP_INSTALL_DIR}/ompp-ui \
&& rm -r ${OMPP_INSTALL_DIR}/html \
&& mv ${OMPP_INSTALL_DIR}/ompp-ui/dist/spa ${OMPP_INSTALL_DIR}/html \
&& fix-permissions ${OMPP_INSTALL_DIR} \
&& pip install /opt/jupyter-ompp-proxy/
# Solarized Theme and Cell Execution Time
COPY jupyterlab-overrides.json /opt/conda/share/jupyter/lab/settings/overrides.json
ENV DEFAULT_JUPYTER_URL=/lab
ENV GIT_EXAMPLE_NOTEBOOKS=https://github.com/StatCan/aaw-contrib-jupyter-notebooks
###############################
### docker-bits/7_remove_vulnerabilities.Dockerfile
###############################
# Remove libpdfbox-java due to CVE-2019-0228. See https://github.com/StatCan/aaw-kubeflow-containers/issues/249#issuecomment-834808115 for details.
# Issue opened https://github.com/jupyter/docker-stacks/issues/1299.
# This line of code should be removed once a solution or better alternative is found.
USER root
RUN apt-get update --yes \
&& dpkg -r --force-depends libpdfbox-java \
&& rm -rf /var/lib/apt/lists/*
# Forcibly upgrade packages to patch vulnerabilities
# See https://github.com/StatCan/aaw-private/issues/58#issuecomment-1471863092 for more details.
RUN pip3 --no-cache-dir install --quiet \
'wheel==0.40.0' \
'setuptools==67.6.0' \
'pyjwt==2.6.0' \
'oauthlib==3.2.2' \
'mpmath==1.3.0' \
'lxml==4.9.2' \
'pyarrow==14.0.1' \
'cryptography==41.0.6' \
&& fix-permissions $CONDA_DIR && \
fix-permissions /home/$NB_USER
USER $NB_USER
###############################
### docker-bits/8_platform.Dockerfile
###############################
USER root
# Install AMD AOCL
ARG AOCL_VERSION=4.0
ENV AOCL_PATH=/opt/amd/aocl/${AOCL_VERSION}
ARG AOCL_SHA256=8a249e727beb8005639b4887074e1ea75020267ed1ac25520876a7ad21d0f4f6
RUN cd ${RESOURCES_PATH} && \
wget --quiet https://download.amd.com/developer/eula/aocl/aocl-4-0/aocl-linux-aocc-${AOCL_VERSION}.tar.gz -O /tmp/aocl-linux-aocc-${AOCL_VERSION}.tar && \
echo "${AOCL_SHA256} /tmp/aocl-linux-aocc-${AOCL_VERSION}.tar" | sha256sum -c - && \
tar xf /tmp/aocl-linux-aocc-${AOCL_VERSION}.tar -C ./ && \
cd ./aocl-linux-aocc-${AOCL_VERSION} && \
/bin/bash ./install.sh -t /opt/amd/aocl && \
cp setenv_aocl.sh ${AOCL_PATH} &&\
rm /tmp/aocl-linux-aocc-${AOCL_VERSION}.tar
# Install AMD AOCC
ARG AOCC_VERSION=4.0.0
ARG AOCC_SHA256=2729ec524cbc927618e479994330eeb72df5947e90cfcc49434009eee29bf7d4
RUN cd ${RESOURCES_PATH} && \
wget --quiet https://download.amd.com/developer/eula/aocc-compiler/aocc-compiler-${AOCC_VERSION}.tar -O /tmp/aocc-compiler-${AOCC_VERSION}.tar && \
echo "${AOCC_SHA256} /tmp/aocc-compiler-${AOCC_VERSION}.tar" | sha256sum -c - && \
tar xf /tmp/aocc-compiler-${AOCC_VERSION}.tar -C ./ && \
cd ./aocc-compiler-${AOCC_VERSION} && \
/bin/bash ./install.sh && \
rm /tmp/aocc-compiler-${AOCC_VERSION}.tar
###############################
### docker-bits/∞_CMD.Dockerfile
###############################
# Configure container startup
USER root
WORKDIR /home/$NB_USER
EXPOSE 8888
COPY start-custom.sh start-oms.sh restart-oms.sh /usr/local/bin/
COPY mc-tenant-wrapper.sh /usr/local/bin/mc
COPY trino-wrapper.sh /usr/local/bin/trino
RUN chmod +x /usr/local/bin/start-oms.sh && \
chmod +x /usr/local/bin/restart-oms.sh
# Add --user to all pip install calls and point pip to Artifactory repository
COPY pip.conf /tmp/pip.conf
RUN cat /tmp/pip.conf >> /etc/pip.conf && rm /tmp/pip.conf
# Point R to Artifactory repository
COPY Rprofile.site /tmp/Rprofile.site
RUN cat /tmp/Rprofile.site >> /opt/conda/lib/R/etc/Rprofile.site && rm /tmp/Rprofile.site
# Add .Rprofile to /tmp so we can install it in start-custom.sh
COPY .Rprofile /tmp/.Rprofile
# Point conda to Artifactory repository
RUN conda config --add channels http://jfrog-platform-artifactory.jfrog-system:8081/artifactory/api/conda/conda-forge-remote --system && \
conda config --remove channels conda-forge --system && \
conda config --add channels http://jfrog-platform-artifactory.jfrog-system:8081/artifactory/api/conda/conda-forge-nvidia --system && \
conda config --add channels http://jfrog-platform-artifactory.jfrog-system:8081/artifactory/api/conda/conda-pytorch-remote --system
USER $NB_USER
ENTRYPOINT ["tini", "--"]
CMD ["start-custom.sh"]