From a539169d4e55b0b1de20e9db73196abc8a5e2bf2 Mon Sep 17 00:00:00 2001 From: ckolbPTB Date: Sat, 17 Feb 2024 20:20:00 +0100 Subject: [PATCH 1/3] Build containers and run all actions in them Co-authored-by: Felix F Zimmermann --- .github/workflows/docker.yml | 146 ++++++++++++ .github/workflows/{docs.yaml => docs.yml} | 138 +++++------ .github/workflows/pre-commit.yml | 10 +- .github/workflows/pytest.yml | 67 ++++-- binder/environment.yml | 9 + binder/requirements.txt | 276 ---------------------- docker/Dockerfile_py311 | 14 ++ docker/Dockerfile_py312 | 14 ++ docker/reco.sh | 49 ++++ 9 files changed, 346 insertions(+), 377 deletions(-) create mode 100644 .github/workflows/docker.yml rename .github/workflows/{docs.yaml => docs.yml} (65%) create mode 100755 binder/environment.yml delete mode 100755 binder/requirements.txt create mode 100644 docker/Dockerfile_py311 create mode 100644 docker/Dockerfile_py312 create mode 100644 docker/reco.sh diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 00000000..3c7817e0 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,146 @@ +name: Build and test docker + +on: + pull_request: + paths-ignore: + - "**.md" + +jobs: + get_dockerfiles: + name: "Get list of dockerfiles for different containers" + runs-on: ubuntu-latest + outputs: + docker_toml: ${{ steps.filter.outputs.docker_toml }} + dockerfiles: ${{ steps.set-matrix.outputs.dockerfiles }} + imagenames: ${{ steps.set-matrix.outputs.imagenames }} + steps: + - name: Checkout mrpro repo + uses: actions/checkout@v4 + + - name: Check if files in docker or the toml file has been modified + uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + docker_toml: + - 'docker/*' + - 'pyproject.toml' + - '/.github/docker.yml' + + - run: echo ${{ steps.filter.outputs.docker_toml }} + + - id: set-matrix + if: steps.filter.outputs.docker_toml == 'true' + run: | + cd ./docker/ + ls + dockerfiles=$(ls Dockerfile_* | jq -R -s -c 'split("\n")[:-1]') + echo $dockerfiles + echo "dockerfiles=$dockerfiles" >> $GITHUB_OUTPUT + imagenames=$(ls Dockerfile_* | sed -e 's/Dockerfile_/ghcr.io\/ptb-mr\/mrpro_/' | jq -R -s -c 'split("\n")[:-1]') + echo $imagenames + echo "imagenames=$imagenames" >> $GITHUB_OUTPUT + + - name: Dockerfile overview + if: steps.filter.outputs.docker_toml == 'true' + run: | + echo ${{ steps.set-matrix.outputs.dockerfiles }} + echo ${{ steps.set-matrix.outputs.imagenames }} + + push_test: + name: "Create test images and push to GCR" + needs: get_dockerfiles + if: ${{ needs.get_dockerfiles.outputs.docker_toml == 'true' }} + runs-on: ubuntu-latest + permissions: + packages: write + strategy: + matrix: + dockerfile: ${{ fromJson(needs.get_dockerfiles.outputs.dockerfiles) }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Create image name + id: image_name + run: | + dockerfile=${{ matrix.dockerfile }} + echo "image_name=${dockerfile/Dockerfile_/ghcr.io/ptb-mr/mrpro_}" >> $GITHUB_OUTPUT + + - name: Login to GitHub Packages + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ptb-mr + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: ./docker + file: ./docker/${{ matrix.dockerfile }} + push: true + tags: ${{ steps.image_name.outputs.image_name }}:test + + test: + name: "Test docker containers" + needs: [get_dockerfiles, push_test] + if: ${{ needs.get_dockerfiles.outputs.docker_toml == 'true' }} + runs-on: ubuntu-latest + permissions: + pull-requests: write + contents: write + strategy: + matrix: + imagename: ${{ fromJson(needs.get_dockerfiles.outputs.imagenames) }} + # runs within Docker container + container: + image: ${{ matrix.imagename }}:test + options: --user runner + + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Install mrpro and dependencies + run: pip install --upgrade --upgrade-strategy "eager" .[test] + + - name: Install pytest-github-actions-annotate-failures plugin + run: pip install pytest-github-actions-annotate-failures + + - name: Run PyTest + run: | + pytest -n 4 -m "not cuda" + + push_latest: + name: "Create latest images and push to GCR" + needs: [get_dockerfiles, test] + if: ${{ needs.get_dockerfiles.outputs.docker_toml == 'true' }} + runs-on: ubuntu-latest + permissions: + packages: write + strategy: + matrix: + dockerfile: ${{ fromJson(needs.get_dockerfiles.outputs.dockerfiles) }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Create image name + id: image_name + run: | + dockerfile=${{ matrix.dockerfile }} + echo "image_name=${dockerfile/Dockerfile_/ghcr.io/ptb-mr/mrpro_}" >> $GITHUB_OUTPUT + + - name: Login to GitHub Packages + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ptb-mr + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Pull and push Docker image with new tag + run: | + docker pull ${{ steps.image_name.outputs.image_name }}:test + docker tag ${{ steps.image_name.outputs.image_name }}:test ${{ steps.image_name.outputs.image_name }}:latest + docker push ${{ steps.image_name.outputs.image_name }}:latest diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yml similarity index 65% rename from .github/workflows/docs.yaml rename to .github/workflows/docs.yml index 41f0303a..e3dd1a36 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yml @@ -7,17 +7,24 @@ on: pull_request: permissions: - contents: write + contents: write + +defaults: + run: + shell: bash jobs: convert_scripts: - name: 'Translate scripts to notebooks' + name: "Translate scripts to notebooks" runs-on: ubuntu-latest permissions: pull-requests: write contents: write outputs: commit_hash: ${{ steps.add-commit-push.outputs.commit_hash }} + container: + image: ghcr.io/ptb-mr/mrpro_py311:latest + options: --user runner steps: - name: Checkout repo uses: actions/checkout@v4 @@ -26,20 +33,8 @@ jobs: ref: ${{ github.event.pull_request.head.ref }} fetch-depth: 0 - - name: Set up Python 3.11 - uses: actions/setup-python@v5 - with: - python-version: '3.11' - cache: 'pip' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install .[notebook] - - - name: Create requirements.txt from toml - run: | - pip-compile --extra notebook -o ./binder/requirements.txt pyproject.toml + - name: Install mrpro and dependencies + run: pip install --upgrade --upgrade-strategy "eager" .[notebook] - name: Translate scripts to notebooks run: | @@ -52,9 +47,7 @@ jobs: uses: tj-actions/verify-changed-files@v18 id: verify-changed-notebooks with: - files: | - ./examples/*.ipynb - ./binder/requirements.txt + files: ./examples/*.ipynb - name: Commit notebooks if: steps.verify-changed-notebooks.outputs.files_changed == 'true' @@ -67,7 +60,7 @@ jobs: run: echo "commit_hash=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT get_notebooks: - name: 'Get list of notebooks' + name: "Get list of notebooks" needs: convert_scripts runs-on: ubuntu-latest steps: @@ -88,12 +81,15 @@ jobs: notebooks: ${{ steps.set-matrix.outputs.notebooks }} run_notebook: - name: 'Run notebook' + name: "Run notebook" needs: [convert_scripts, get_notebooks] runs-on: ubuntu-latest permissions: pull-requests: write contents: write + container: + image: ghcr.io/ptb-mr/mrpro_py311:latest + options: --user root strategy: matrix: notebook: ${{ fromJson(needs.get_notebooks.outputs.notebooks) }} @@ -103,26 +99,16 @@ jobs: with: ref: ${{ needs.convert_scripts.outputs.commit_hash }} - - name: Set up Python 3.11 - uses: actions/setup-python@v5 - with: - python-version: '3.11' - cache: 'pip' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install .[notebook] + - name: Install mrpro and dependencies + run: pip install --upgrade --upgrade-strategy "eager" .[notebook] - name: Notebook name run: echo ${{ matrix.notebook }} - name: Run notebook - uses: yaananth/run-notebook@v2 + uses: fzimmermann89/run-notebook@v3 env: RUNNER: ${{ toJson(runner) }} - SECRETS: ${{ toJson(secrets) }} - GITHUB: ${{ toJson(github) }} with: notebook: ./examples/${{ matrix.notebook }} @@ -138,40 +124,34 @@ jobs: if: always() with: name: ${{ steps.artifact_names.outputs.ARTIFACT_NAME }} - path: ${{ RUNNER.temp }}/nb-runner/${{ steps.artifact_names.outputs.HTML_RESULT }} + path: ${{ github.workspace }}/nb-runner.out/${{ steps.artifact_names.outputs.HTML_RESULT }} env: RUNNER: ${{ toJson(runner) }} create_documentation: - name: 'Build and deploy documentation' + name: Build and deploy documentation needs: [convert_scripts, run_notebook] runs-on: ubuntu-latest - + container: + image: ghcr.io/ptb-mr/mrpro_py311:latest + options: --user runner steps: - - name: Checkout - uses: actions/checkout@v4 - with: - ref: ${{ needs.convert_scripts.outputs.commit_hash }} - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.11' - cache: 'pip' - - - name: Install dependencies - run: | - pip install --upgrade pip - pip install ".[docs]" - - - name: Download notebook html files - id: download - uses: actions/download-artifact@v4 - with: - path: ./docs/source/notebook_artifact/ - - - name: Copy notebook html files - run: | + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ needs.convert_scripts.outputs.commit_hash }} + + - name: Install mrpro and dependencies + run: pip install --upgrade --upgrade-strategy "eager" .[docs] + + - name: Download notebook html files + id: download + uses: actions/download-artifact@v4 + with: + path: ./docs/source/notebook_artifact/ + + - name: Copy notebook html files + run: | mkdir ./docs/source/_notebooks cd ./docs/source/notebook_artifact/ notebooks=$(grep -rl --include='*' './') @@ -181,8 +161,8 @@ jobs: cp ./$nb ../_notebooks/ done - - name: List of notebooks - run: | + - name: List of notebooks + run: | cd ./docs/source/_notebooks/ notebooks=$(grep -rl --include='*.html' './') cd ../ @@ -198,22 +178,22 @@ jobs: echo " :file: ./_notebooks/$nb" >> "notebook_${nb/.html/.rst}" done - - name: Build docs - run: sphinx-build -b html ./docs/source ./docs/build/html - - - name: Deploy to GitHub Pages - uses: peaceiris/actions-gh-pages@v3 - with: - publish_branch: github-pages - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: ./docs/build/html - if: github.event_name != 'pull_request' - - - name: Save Documentation - uses: actions/upload-artifact@v4 - with: - name: Documentation - path: docs/build/html/ + - name: Build docs + run: sphinx-build -b html ./docs/source ./docs/build/html + + - name: Deploy to GitHub Pages + uses: peaceiris/actions-gh-pages@v3 + with: + publish_branch: github-pages + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./docs/build/html + if: github.event_name != 'pull_request' + + - name: Save Documentation + uses: actions/upload-artifact@v4 + with: + name: Documentation + path: docs/build/html/ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index a3516612..67a798e0 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -1,17 +1,17 @@ -name: pre-commit +name: Pre-Commit on: pull_request: jobs: pre-commit: + name: "Pre-commit" runs-on: ubuntu-latest + container: + image: ghcr.io/ptb-mr/mrpro_py311:latest + options: --user runner steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.11' - cache: 'pip' # caching pip dependencies - uses: pre-commit/action@v3.0.1 concurrency: diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 4f4df99a..a5316340 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -6,35 +6,68 @@ on: - "**.md" jobs: + get_dockerfiles: + name: "Get list of dockerfiles for different containers" + runs-on: ubuntu-latest + permissions: + packages: read + outputs: + imagenames: ${{ steps.set-matrix.outputs.imagenames }} + steps: + - id: set-matrix + env: + GH_TOKEN: ${{ secrets.GHCR_TOKEN }} + run: | + imagenames=$(curl -s --request GET \ + --url "https://api.github.com/orgs/PTB-MR/packages?package_type=container" \ + --header "Authorization: Bearer $GH_TOKEN" | jq -r '.[].name') + echo $imagenames + + imagenames_latest=() + for image in $(echo $imagenames) + do + echo "checking $image ..." + if docker manifest inspect "ghcr.io/ptb-mr/"$image":latest" >/dev/null; then + echo "... $image added" + imagenames_latest+=$image":" + fi + done + echo $imagenames_latest + imagenames_latest=$(echo $imagenames_latest | jq -R -c 'split(":")[:-1]') + echo $imagenames_latest + echo "imagenames=$imagenames_latest" >> $GITHUB_OUTPUT + + - name: Dockerfile overview + run: | + echo ${{ steps.set-matrix.outputs.imagenames }} + + test: + name: "Run tests and get coverage report" + needs: get_dockerfiles runs-on: ubuntu-latest permissions: pull-requests: write contents: write strategy: matrix: - python-version: ["3.11", "3.12"] + imagename: ${{ fromJson(needs.get_dockerfiles.outputs.imagenames) }} + # runs within Docker container + container: + image: ghcr.io/ptb-mr/${{ matrix.imagename }}:latest + options: --user runner steps: - name: Checkout repo uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - cache: 'pip' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install .[test] + - name: Install mrpro and dependencies + run: pip install --upgrade --upgrade-strategy "eager" .[test] - name: Install pytest-github-actions-annotate-failures plugin run: pip install pytest-github-actions-annotate-failures - name: Run PyTest - run: | pytest -n 4 -m "not cuda" --junitxml=pytest.xml --cov-report=term-missing:skip-covered --cov=mrpro | tee pytest-coverage.txt @@ -57,11 +90,11 @@ jobs: namedLogo: python - name: Set pipeline status - run: | - if [[ ${{ steps.coverageComment.outputs.errors }} -ne 0 || ${{ steps.coverageComment.outputs.failures }} -ne 0 ]]; then - echo "Errors or failures detected, marking pipeline as failure." - exit 1 - fi + if: steps.coverageComment.outputs.errors != 0 || steps.coverageComment.outputs.failures != 0 + uses: actions/github-script@v7 + with: + script: | + core.setFailed('PyTest workflow failed with ${{ steps.coverageComment.outputs.errors }} errors and ${{ steps.coverageComment.outputs.failures }} failures.') concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} diff --git a/binder/environment.yml b/binder/environment.yml new file mode 100755 index 00000000..55c8d56c --- /dev/null +++ b/binder/environment.yml @@ -0,0 +1,9 @@ +name: Binder +channels: + - conda-forge +dependencies: + - python=3.11 + - pytorch::pytorch + - pytorch::cpuonly + - pip: + - mrpro[notebook] @ git+https://github.com/PTB-MR/mrpro diff --git a/binder/requirements.txt b/binder/requirements.txt deleted file mode 100755 index 71fb8ca1..00000000 --- a/binder/requirements.txt +++ /dev/null @@ -1,276 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile --extra=notebook --output-file=./binder/requirements.txt pyproject.toml -# -asttokens==2.4.1 - # via stack-data -attrs==23.1.0 - # via - # jsonschema - # referencing -build==1.0.3 - # via pip-tools -certifi==2023.11.17 - # via requests -charset-normalizer==3.3.2 - # via requests -click==8.1.7 - # via pip-tools -comm==0.2.0 - # via - # ipykernel - # ipywidgets -contourpy==1.2.0 - # via matplotlib -coverage==7.3.2 - # via pypulseq -cycler==0.12.1 - # via matplotlib -debugpy==1.8.0 - # via ipykernel -decorator==5.1.1 - # via ipython -einops==0.7.0 - # via mrpro (pyproject.toml) -executing==2.0.1 - # via stack-data -fastjsonschema==2.19.0 - # via nbformat -filelock==3.13.1 - # via - # torch - # triton -fonttools==4.46.0 - # via matplotlib -fsspec==2023.12.0 - # via torch -h5py==3.10.0 - # via ismrmrd -idna==3.6 - # via requests -ipykernel==6.27.1 - # via mrpro (pyproject.toml) -ipython==8.18.1 - # via - # ipykernel - # ipywidgets -ipywidgets==8.1.2 - # via mrpro (pyproject.toml) -ismrmrd==1.14.0 - # via mrpro (pyproject.toml) -jedi==0.19.1 - # via ipython -jinja2==3.1.2 - # via torch -jsonschema==4.20.0 - # via nbformat -jsonschema-specifications==2023.11.2 - # via jsonschema -jupyter-client==8.6.0 - # via ipykernel -jupyter-core==5.5.0 - # via - # ipykernel - # jupyter-client - # nbformat -jupyterlab-widgets==3.0.10 - # via ipywidgets -jupytext==1.16.0 - # via mrpro (pyproject.toml) -kiwisolver==1.4.5 - # via matplotlib -llvmlite==0.41.1 - # via numba -markdown-it-py==3.0.0 - # via - # jupytext - # mdit-py-plugins -markupsafe==2.1.3 - # via jinja2 -matplotlib==3.8.2 - # via - # mrpro (pyproject.toml) - # pypulseq -matplotlib-inline==0.1.6 - # via - # ipykernel - # ipython -mdit-py-plugins==0.4.0 - # via jupytext -mdurl==0.1.2 - # via markdown-it-py -mpmath==1.3.0 - # via sympy -nbformat==5.9.2 - # via jupytext -nest-asyncio==1.5.8 - # via ipykernel -networkx==3.2.1 - # via torch -numba==0.58.1 - # via sigpy -numpy==1.26.2 - # via - # contourpy - # h5py - # ismrmrd - # matplotlib - # mrpro (pyproject.toml) - # numba - # pypulseq - # pywavelets - # scipy - # sigpy - # torchkbnufft -nvidia-cublas-cu12==12.1.3.1 - # via - # nvidia-cudnn-cu12 - # nvidia-cusolver-cu12 - # torch -nvidia-cuda-cupti-cu12==12.1.105 - # via torch -nvidia-cuda-nvrtc-cu12==12.1.105 - # via torch -nvidia-cuda-runtime-cu12==12.1.105 - # via torch -nvidia-cudnn-cu12==8.9.2.26 - # via torch -nvidia-cufft-cu12==11.0.2.54 - # via torch -nvidia-curand-cu12==10.3.2.106 - # via torch -nvidia-cusolver-cu12==11.4.5.107 - # via torch -nvidia-cusparse-cu12==12.1.0.106 - # via - # nvidia-cusolver-cu12 - # torch -nvidia-nccl-cu12==2.19.3 - # via torch -nvidia-nvjitlink-cu12==12.3.101 - # via - # nvidia-cusolver-cu12 - # nvidia-cusparse-cu12 -nvidia-nvtx-cu12==12.1.105 - # via torch -packaging==23.2 - # via - # build - # ipykernel - # jupytext - # matplotlib -parso==0.8.3 - # via jedi -pexpect==4.9.0 - # via ipython -pillow==10.1.0 - # via matplotlib -pip-tools==7.3.0 - # via mrpro (pyproject.toml) -platformdirs==4.0.0 - # via jupyter-core -prompt-toolkit==3.0.41 - # via ipython -psutil==5.9.6 - # via ipykernel -ptyprocess==0.7.0 - # via pexpect -pure-eval==0.2.2 - # via stack-data -pydicom==2.4.3 - # via mrpro (pyproject.toml) -pygments==2.17.2 - # via ipython -pyparsing==3.1.1 - # via matplotlib -pyproject-hooks==1.0.0 - # via build -pypulseq @ git+https://github.com/imr-framework/pypulseq - # via mrpro (pyproject.toml) -python-dateutil==2.8.2 - # via - # jupyter-client - # matplotlib -pywavelets==1.5.0 - # via sigpy -pyyaml==6.0.1 - # via jupytext -pyzmq==25.1.1 - # via - # ipykernel - # jupyter-client -referencing==0.31.1 - # via - # jsonschema - # jsonschema-specifications -requests==2.31.0 - # via zenodo-get -rpds-py==0.13.2 - # via - # jsonschema - # referencing -scipy==1.11.4 - # via - # pypulseq - # sigpy - # torchkbnufft -sigpy==0.1.26 - # via pypulseq -six==1.16.0 - # via - # asttokens - # python-dateutil -stack-data==0.6.3 - # via ipython -sympy==1.12 - # via torch -toml==0.10.2 - # via jupytext -torch==2.2.0 - # via - # mrpro (pyproject.toml) - # torchkbnufft -torchkbnufft==1.4.0 - # via mrpro (pyproject.toml) -tornado==6.4 - # via - # ipykernel - # jupyter-client -tqdm==4.66.1 - # via sigpy -traitlets==5.14.0 - # via - # comm - # ipykernel - # ipython - # ipywidgets - # jupyter-client - # jupyter-core - # matplotlib-inline - # nbformat -triton==2.2.0 - # via torch -typing-extensions==4.8.0 - # via torch -urllib3==2.1.0 - # via requests -wcwidth==0.2.12 - # via prompt-toolkit -wget==3.2 - # via zenodo-get -wheel==0.42.0 - # via pip-tools -widgetsnbextension==4.0.10 - # via ipywidgets -xsdata==22.12 - # via - # ismrmrd - # mrpro (pyproject.toml) -zenodo-get==1.5.1 - # via mrpro (pyproject.toml) - -# The following packages are considered to be unsafe in a requirements file: -# pip -# setuptools diff --git a/docker/Dockerfile_py311 b/docker/Dockerfile_py311 new file mode 100644 index 00000000..14da19bd --- /dev/null +++ b/docker/Dockerfile_py311 @@ -0,0 +1,14 @@ +ARG BASE_IMAGE=ubuntu:22.04 +FROM ${BASE_IMAGE} as base + +ARG DEBIAN_FRONTEND=noninteractive + +ARG PYTHON="python3.11" + +# install python and dependencies +COPY reco.sh . +RUN bash reco.sh +RUN rm reco.sh + +# set user +USER runner diff --git a/docker/Dockerfile_py312 b/docker/Dockerfile_py312 new file mode 100644 index 00000000..c4224f32 --- /dev/null +++ b/docker/Dockerfile_py312 @@ -0,0 +1,14 @@ +ARG BASE_IMAGE=ubuntu:22.04 +FROM ${BASE_IMAGE} as base + +ARG DEBIAN_FRONTEND=noninteractive + +ARG PYTHON="python3.12" + +# install python and dependencies +COPY reco.sh . +RUN bash reco.sh +RUN rm reco.sh + +# set user +USER runner diff --git a/docker/reco.sh b/docker/reco.sh new file mode 100644 index 00000000..64df5047 --- /dev/null +++ b/docker/reco.sh @@ -0,0 +1,49 @@ +# define apt-get installation command +APT_GET_INSTALL="apt-get install -yq --no-install-recommends" + +# update, qq: quiet +apt-get update -qq + +# ensure certificates are up to date +${APT_GET_INSTALL} --reinstall ca-certificates + +# base utilities +${APT_GET_INSTALL} git software-properties-common gpg-agent + +# add repo for python installation +add-apt-repository ppa:deadsnakes/ppa +apt update -qq +${APT_GET_INSTALL} $PYTHON-full + +# create alias for installed python version +ln -s /usr/bin/$PYTHON /usr/local/bin/python +ln -s /usr/bin/$PYTHON /usr/local/bin/python3 + +pip install matplotlib + +# clone repo to get requirements +git clone https://github.com/PTB-MR/mrpro --depth 1 /opt/mrpro +cd /opt/mrpro +python -m ensurepip --upgrade +python -m pip install --upgrade pip + +# create alias to ensure pip works in the same way as pip3 +ln -s /usr/local/bin/pip3 /usr/local/bin/pip + +# pre-install cpu-version of torch to avoid installation of cuda-version via dependencies +python -m pip install --no-cache-dir torch --index-url https://download.pytorch.org/whl/cpu + +# install mrpro and dependencies +python -m pip install --upgrade --upgrade-strategy "eager" .[notebook,test,docs] + +# clean up +rm -r /opt/mrpro +apt-get clean && rm -rf /var/lib/apt/lists/* + +# add user runner +adduser --disabled-password --gecos "" --uid 1001 runner \ + && groupadd docker --gid 123 \ + && usermod -aG sudo runner \ + && usermod -aG docker runner \ + && echo "%sudo ALL=(ALL:ALL) NOPASSWD:ALL" > /etc/sudoers \ + && echo "Defaults env_keep += \"DEBIAN_FRONTEND\"" >> /etc/sudoers From 1ae648f3331289c26cafd9190614e7bc5ba9e1b6 Mon Sep 17 00:00:00 2001 From: ckolbPTB Date: Wed, 13 Mar 2024 11:43:51 +0100 Subject: [PATCH 2/3] remove unnecessary quotes --- .github/workflows/docker.yml | 8 ++++---- .github/workflows/docs.yml | 6 +++--- .github/workflows/pre-commit.yml | 2 +- .github/workflows/pytest.yml | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 3c7817e0..2d6e5780 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -7,7 +7,7 @@ on: jobs: get_dockerfiles: - name: "Get list of dockerfiles for different containers" + name: Get list of dockerfiles for different containers runs-on: ubuntu-latest outputs: docker_toml: ${{ steps.filter.outputs.docker_toml }} @@ -48,7 +48,7 @@ jobs: echo ${{ steps.set-matrix.outputs.imagenames }} push_test: - name: "Create test images and push to GCR" + name: Create test images and push to GCR needs: get_dockerfiles if: ${{ needs.get_dockerfiles.outputs.docker_toml == 'true' }} runs-on: ubuntu-latest @@ -83,7 +83,7 @@ jobs: tags: ${{ steps.image_name.outputs.image_name }}:test test: - name: "Test docker containers" + name: Test docker containers needs: [get_dockerfiles, push_test] if: ${{ needs.get_dockerfiles.outputs.docker_toml == 'true' }} runs-on: ubuntu-latest @@ -113,7 +113,7 @@ jobs: pytest -n 4 -m "not cuda" push_latest: - name: "Create latest images and push to GCR" + name: Create latest images and push to GCR needs: [get_dockerfiles, test] if: ${{ needs.get_dockerfiles.outputs.docker_toml == 'true' }} runs-on: ubuntu-latest diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index e3dd1a36..eb922830 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -15,7 +15,7 @@ defaults: jobs: convert_scripts: - name: "Translate scripts to notebooks" + name: Translate scripts to notebooks runs-on: ubuntu-latest permissions: pull-requests: write @@ -60,7 +60,7 @@ jobs: run: echo "commit_hash=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT get_notebooks: - name: "Get list of notebooks" + name: Get list of notebooks needs: convert_scripts runs-on: ubuntu-latest steps: @@ -81,7 +81,7 @@ jobs: notebooks: ${{ steps.set-matrix.outputs.notebooks }} run_notebook: - name: "Run notebook" + name: Run notebook needs: [convert_scripts, get_notebooks] runs-on: ubuntu-latest permissions: diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 67a798e0..137c848b 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -5,7 +5,7 @@ on: jobs: pre-commit: - name: "Pre-commit" + name: Pre-commit runs-on: ubuntu-latest container: image: ghcr.io/ptb-mr/mrpro_py311:latest diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index a5316340..889ff991 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -7,7 +7,7 @@ on: jobs: get_dockerfiles: - name: "Get list of dockerfiles for different containers" + name: Get list of dockerfiles for different containers runs-on: ubuntu-latest permissions: packages: read @@ -43,7 +43,7 @@ jobs: test: - name: "Run tests and get coverage report" + name: Run tests and get coverage report needs: get_dockerfiles runs-on: ubuntu-latest permissions: From 79afbc9329bcc38fe16c635ef382bc96da2c0f1a Mon Sep 17 00:00:00 2001 From: ckolbPTB Date: Wed, 13 Mar 2024 14:35:26 +0100 Subject: [PATCH 3/3] review comments --- .github/workflows/docker.yml | 11 ++++++----- .github/workflows/docs.yml | 10 ++++++---- .github/workflows/pytest.yml | 8 ++++---- docker/Dockerfile_py311 | 8 ++++---- docker/Dockerfile_py312 | 8 ++++---- docker/{reco.sh => install_mrpro.sh} | 0 6 files changed, 24 insertions(+), 21 deletions(-) rename docker/{reco.sh => install_mrpro.sh} (100%) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 2d6e5780..b4b62946 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -27,7 +27,8 @@ jobs: - 'pyproject.toml' - '/.github/docker.yml' - - run: echo ${{ steps.filter.outputs.docker_toml }} + - run: | + echo "Rebuild containers? ${{ steps.filter.outputs.docker_toml }}" - id: set-matrix if: steps.filter.outputs.docker_toml == 'true' @@ -35,17 +36,17 @@ jobs: cd ./docker/ ls dockerfiles=$(ls Dockerfile_* | jq -R -s -c 'split("\n")[:-1]') - echo $dockerfiles + echo "dockerfiles: $dockerfiles" echo "dockerfiles=$dockerfiles" >> $GITHUB_OUTPUT imagenames=$(ls Dockerfile_* | sed -e 's/Dockerfile_/ghcr.io\/ptb-mr\/mrpro_/' | jq -R -s -c 'split("\n")[:-1]') - echo $imagenames + echo "image names: $imagenames" echo "imagenames=$imagenames" >> $GITHUB_OUTPUT - name: Dockerfile overview if: steps.filter.outputs.docker_toml == 'true' run: | - echo ${{ steps.set-matrix.outputs.dockerfiles }} - echo ${{ steps.set-matrix.outputs.imagenames }} + echo "final list of dockerfiles: ${{ steps.set-matrix.outputs.dockerfiles }}" + echo "final list of images: ${{ steps.set-matrix.outputs.imagenames }}" push_test: name: Create test images and push to GCR diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index eb922830..5cbb1d76 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -44,7 +44,7 @@ jobs: done - name: Check if any notebooks have been changed - uses: tj-actions/verify-changed-files@v18 + uses: tj-actions/verify-changed-files@v19 id: verify-changed-notebooks with: files: ./examples/*.ipynb @@ -76,7 +76,8 @@ jobs: echo "notebooks=$(ls *.ipynb | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT - name: Notebook overview - run: echo ${{ steps.set-matrix.outputs.notebooks }} + run: | + echo "jupyter-notebooks: ${{ steps.set-matrix.outputs.notebooks }}" outputs: notebooks: ${{ steps.set-matrix.outputs.notebooks }} @@ -103,7 +104,8 @@ jobs: run: pip install --upgrade --upgrade-strategy "eager" .[notebook] - name: Notebook name - run: echo ${{ matrix.notebook }} + run: | + echo "current jupyter-notebook: ${{ matrix.notebook }}" - name: Run notebook uses: fzimmermann89/run-notebook@v3 @@ -157,7 +159,7 @@ jobs: notebooks=$(grep -rl --include='*' './') for nb in $notebooks do - echo $nb + echo "current jupyter-notebook: $nb" cp ./$nb ../_notebooks/ done diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 889ff991..9d58ba4a 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -21,7 +21,7 @@ jobs: imagenames=$(curl -s --request GET \ --url "https://api.github.com/orgs/PTB-MR/packages?package_type=container" \ --header "Authorization: Bearer $GH_TOKEN" | jq -r '.[].name') - echo $imagenames + echo "image names: $imagenames" imagenames_latest=() for image in $(echo $imagenames) @@ -32,14 +32,14 @@ jobs: imagenames_latest+=$image":" fi done - echo $imagenames_latest + echo "image names with tag latest: $imagenames_latest" imagenames_latest=$(echo $imagenames_latest | jq -R -c 'split(":")[:-1]') - echo $imagenames_latest + echo "image names with tag latest: $imagenames_latest" echo "imagenames=$imagenames_latest" >> $GITHUB_OUTPUT - name: Dockerfile overview run: | - echo ${{ steps.set-matrix.outputs.imagenames }} + echo "final list of images with tag latest: ${{ steps.set-matrix.outputs.imagenames }}" test: diff --git a/docker/Dockerfile_py311 b/docker/Dockerfile_py311 index 14da19bd..1e775160 100644 --- a/docker/Dockerfile_py311 +++ b/docker/Dockerfile_py311 @@ -5,10 +5,10 @@ ARG DEBIAN_FRONTEND=noninteractive ARG PYTHON="python3.11" -# install python and dependencies -COPY reco.sh . -RUN bash reco.sh -RUN rm reco.sh +# install python and mrpro dependencies +COPY install_mrpro.sh . +RUN bash install_mrpro.sh +RUN rm install_mrpro.sh # set user USER runner diff --git a/docker/Dockerfile_py312 b/docker/Dockerfile_py312 index c4224f32..fe223467 100644 --- a/docker/Dockerfile_py312 +++ b/docker/Dockerfile_py312 @@ -5,10 +5,10 @@ ARG DEBIAN_FRONTEND=noninteractive ARG PYTHON="python3.12" -# install python and dependencies -COPY reco.sh . -RUN bash reco.sh -RUN rm reco.sh +# install python and mrpro dependencies +COPY install_mrpro.sh . +RUN bash install_mrpro.sh +RUN rm install_mrpro.sh # set user USER runner diff --git a/docker/reco.sh b/docker/install_mrpro.sh similarity index 100% rename from docker/reco.sh rename to docker/install_mrpro.sh