From fed247b209e1cf6428c41d7f0bdadfb1e7a9764f Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Sat, 23 Oct 2021 01:25:31 +0000 Subject: [PATCH 001/115] Initial try --- .gitpod.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .gitpod.yml diff --git a/.gitpod.yml b/.gitpod.yml new file mode 100644 index 00000000000..ea6879cd176 --- /dev/null +++ b/.gitpod.yml @@ -0,0 +1,20 @@ +# List the start up tasks. Learn more https://www.gitpod.io/docs/config-start-tasks/ +tasks: + - init: | + sudo apt-get update + # Needed for bootstrap + sudo apt-get install gettext + # From dockerfile + sudo apt-get install -y --no-install-recommends gfortran gcc g++ libstdc++-9-dev sudo openssl + sudo apt-get install -y wget build-essential automake m4 dpkg-dev python libssl-dev git rdfind + # Recommended by ./configure + sudo apt-get install -y libflint-arb-dev libbrial-dev libbrial-groebner-dev libcdd-dev libcdd-tools ecl libec-dev eclib-tools fflas-ffpack flintqs libgc-dev gfan libgiac-dev xcas libgsl-dev libiml-dev lcalc liblfunction-dev libhomfly-dev libopenblas-dev palp pari-gp2c libpari-dev pari-doc pari-elldata pari-galdata pari-galpol pari-seadata libppl-dev ppl-dev python3 libpython3-dev python3-distutils r-base-dev r-cran-lattice librw-dev libsuitesparse-dev libsymmetrica2-dev sympow tachyon libzmq3-dev libzn-poly-dev + ./bootstrap + ./configure -enable-editable + MAKE='make -j8' make + command: echo 'start script' + +# List the ports to expose. Learn more https://www.gitpod.io/docs/config-ports/ +ports: + - port: 3000 + onOpen: open-preview From 5a46c568beb46a5f29438d5e61ed748d799956e3 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Sat, 23 Oct 2021 10:46:45 +0000 Subject: [PATCH 002/115] Try to fix missing pip in venv --- .gitpod.yml | 4 ++-- build/bin/sage-venv | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index ea6879cd176..b960d8a7c27 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -3,7 +3,7 @@ tasks: - init: | sudo apt-get update # Needed for bootstrap - sudo apt-get install gettext + sudo apt-get install -y gettext # From dockerfile sudo apt-get install -y --no-install-recommends gfortran gcc g++ libstdc++-9-dev sudo openssl sudo apt-get install -y wget build-essential automake m4 dpkg-dev python libssl-dev git rdfind @@ -11,7 +11,7 @@ tasks: sudo apt-get install -y libflint-arb-dev libbrial-dev libbrial-groebner-dev libcdd-dev libcdd-tools ecl libec-dev eclib-tools fflas-ffpack flintqs libgc-dev gfan libgiac-dev xcas libgsl-dev libiml-dev lcalc liblfunction-dev libhomfly-dev libopenblas-dev palp pari-gp2c libpari-dev pari-doc pari-elldata pari-galdata pari-galpol pari-seadata libppl-dev ppl-dev python3 libpython3-dev python3-distutils r-base-dev r-cran-lattice librw-dev libsuitesparse-dev libsymmetrica2-dev sympow tachyon libzmq3-dev libzn-poly-dev ./bootstrap ./configure -enable-editable - MAKE='make -j8' make + MAKE='make -j8' make build command: echo 'start script' # List the ports to expose. Learn more https://www.gitpod.io/docs/config-ports/ diff --git a/build/bin/sage-venv b/build/bin/sage-venv index 9ee9fc794be..fb144a8ea92 100755 --- a/build/bin/sage-venv +++ b/build/bin/sage-venv @@ -49,7 +49,8 @@ else: b = venv.EnvBuilder(system_site_packages=options.system_site, clear=options.clear, upgrade=options.upgrade, - symlinks=use_symlinks) + symlinks=use_symlinks, + with_pip=True) c = b.ensure_directories(options.env_dir) b.setup_python(c) b.create_configuration(c) From 4eb4f7f0e32a72edc6cfc40add6f38845c10f46b Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Sat, 23 Oct 2021 11:06:04 +0000 Subject: [PATCH 003/115] Try to add venv package to install pip in venv --- .gitpod.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitpod.yml b/.gitpod.yml index b960d8a7c27..fbf0e941ace 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -9,6 +9,7 @@ tasks: sudo apt-get install -y wget build-essential automake m4 dpkg-dev python libssl-dev git rdfind # Recommended by ./configure sudo apt-get install -y libflint-arb-dev libbrial-dev libbrial-groebner-dev libcdd-dev libcdd-tools ecl libec-dev eclib-tools fflas-ffpack flintqs libgc-dev gfan libgiac-dev xcas libgsl-dev libiml-dev lcalc liblfunction-dev libhomfly-dev libopenblas-dev palp pari-gp2c libpari-dev pari-doc pari-elldata pari-galdata pari-galpol pari-seadata libppl-dev ppl-dev python3 libpython3-dev python3-distutils r-base-dev r-cran-lattice librw-dev libsuitesparse-dev libsymmetrica2-dev sympow tachyon libzmq3-dev libzn-poly-dev + sudo apt-get install -y python3-venv ./bootstrap ./configure -enable-editable MAKE='make -j8' make build From 597b45b68812c110f9ea44f74f0c11162f69cd47 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Sat, 23 Oct 2021 12:01:38 +0000 Subject: [PATCH 004/115] Add more deps and try to fix pip problem --- .gitpod.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitpod.yml b/.gitpod.yml index fbf0e941ace..6c628e3aa34 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -9,11 +9,15 @@ tasks: sudo apt-get install -y wget build-essential automake m4 dpkg-dev python libssl-dev git rdfind # Recommended by ./configure sudo apt-get install -y libflint-arb-dev libbrial-dev libbrial-groebner-dev libcdd-dev libcdd-tools ecl libec-dev eclib-tools fflas-ffpack flintqs libgc-dev gfan libgiac-dev xcas libgsl-dev libiml-dev lcalc liblfunction-dev libhomfly-dev libopenblas-dev palp pari-gp2c libpari-dev pari-doc pari-elldata pari-galdata pari-galpol pari-seadata libppl-dev ppl-dev python3 libpython3-dev python3-distutils r-base-dev r-cran-lattice librw-dev libsuitesparse-dev libsymmetrica2-dev sympow tachyon libzmq3-dev libzn-poly-dev - sudo apt-get install -y python3-venv + # They are not recommended for some reason + sudo apt-get install -y python3-venv tox libgf2x-dev cliquer libcliquer-dev gmp-ecm libecm-dev glpk-utils libglpk-dev libbraiding-dev liblrcalc-dev libm4rie-dev libmpc-dev libmpfi-dev libmpfr-dev nauty libntl-dev libplanarity-dev planarity ./bootstrap ./configure -enable-editable MAKE='make -j8' make build command: echo 'start script' + env: + # Gitpod sets PIP_USER: yes by default, which leads to problems during build (e.g pip not being installed in the venv) + PIP_USER: no # List the ports to expose. Learn more https://www.gitpod.io/docs/config-ports/ ports: From 5755460ab6ec6123beaefdd4c51d53cbc5f76bd5 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Sat, 23 Oct 2021 12:21:53 +0000 Subject: [PATCH 005/115] Set pip_user to no --- .gitpod.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitpod.yml b/.gitpod.yml index 6c628e3aa34..a8c3651d129 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,5 +1,6 @@ # List the start up tasks. Learn more https://www.gitpod.io/docs/config-start-tasks/ tasks: + - before: "echo 'export PIP_USER=no' >> ~/.bashrc && export PIP_USER=no" - init: | sudo apt-get update # Needed for bootstrap @@ -13,6 +14,7 @@ tasks: sudo apt-get install -y python3-venv tox libgf2x-dev cliquer libcliquer-dev gmp-ecm libecm-dev glpk-utils libglpk-dev libbraiding-dev liblrcalc-dev libm4rie-dev libmpc-dev libmpfi-dev libmpfr-dev nauty libntl-dev libplanarity-dev planarity ./bootstrap ./configure -enable-editable + #export PIP_USER=no MAKE='make -j8' make build command: echo 'start script' env: From de42170be4a21e0c9f364d82513aad1af3f60ac1 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Sat, 23 Oct 2021 15:06:37 +0000 Subject: [PATCH 006/115] Fix pip install by disabling pyenv --- .gitpod.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index a8c3651d129..aeaa9470648 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,6 +1,5 @@ # List the start up tasks. Learn more https://www.gitpod.io/docs/config-start-tasks/ tasks: - - before: "echo 'export PIP_USER=no' >> ~/.bashrc && export PIP_USER=no" - init: | sudo apt-get update # Needed for bootstrap @@ -11,15 +10,16 @@ tasks: # Recommended by ./configure sudo apt-get install -y libflint-arb-dev libbrial-dev libbrial-groebner-dev libcdd-dev libcdd-tools ecl libec-dev eclib-tools fflas-ffpack flintqs libgc-dev gfan libgiac-dev xcas libgsl-dev libiml-dev lcalc liblfunction-dev libhomfly-dev libopenblas-dev palp pari-gp2c libpari-dev pari-doc pari-elldata pari-galdata pari-galpol pari-seadata libppl-dev ppl-dev python3 libpython3-dev python3-distutils r-base-dev r-cran-lattice librw-dev libsuitesparse-dev libsymmetrica2-dev sympow tachyon libzmq3-dev libzn-poly-dev # They are not recommended for some reason - sudo apt-get install -y python3-venv tox libgf2x-dev cliquer libcliquer-dev gmp-ecm libecm-dev glpk-utils libglpk-dev libbraiding-dev liblrcalc-dev libm4rie-dev libmpc-dev libmpfi-dev libmpfr-dev nauty libntl-dev libplanarity-dev planarity + sudo apt-get install -y python3-venv libgf2x-dev cliquer libcliquer-dev gmp-ecm libecm-dev glpk-utils libglpk-dev libbraiding-dev liblrcalc-dev libm4rie-dev libmpc-dev libmpfi-dev libmpfr-dev nauty libntl-dev libplanarity-dev planarity + # We do not install tox, since it pulls in javascript-common which does not install for some reason ./bootstrap ./configure -enable-editable - #export PIP_USER=no + # Gitpod sets PIP_USER: yes by default, which leads to problems during build (e.g pip not being installed in the venv) + unset PIP_USER + # Gitpod installs pyenv by default, and sage's pip install targets the pyenv python for some reason + pyenv global system MAKE='make -j8' make build command: echo 'start script' - env: - # Gitpod sets PIP_USER: yes by default, which leads to problems during build (e.g pip not being installed in the venv) - PIP_USER: no # List the ports to expose. Learn more https://www.gitpod.io/docs/config-ports/ ports: From db1386d67c6342bc655e1fbfab49b0c818595463 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Sat, 23 Oct 2021 16:46:25 +0000 Subject: [PATCH 007/115] Try to speed things up --- .gitpod.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitpod.yml b/.gitpod.yml index aeaa9470648..6c1550a7dc6 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -12,6 +12,7 @@ tasks: # They are not recommended for some reason sudo apt-get install -y python3-venv libgf2x-dev cliquer libcliquer-dev gmp-ecm libecm-dev glpk-utils libglpk-dev libbraiding-dev liblrcalc-dev libm4rie-dev libmpc-dev libmpfi-dev libmpfr-dev nauty libntl-dev libplanarity-dev planarity # We do not install tox, since it pulls in javascript-common which does not install for some reason + brew install arb ecl flint fplll pari pari-elldata pari-galdata pari-galpol pari-seadata tox ./bootstrap ./configure -enable-editable # Gitpod sets PIP_USER: yes by default, which leads to problems during build (e.g pip not being installed in the venv) @@ -19,6 +20,8 @@ tasks: # Gitpod installs pyenv by default, and sage's pip install targets the pyenv python for some reason pyenv global system MAKE='make -j8' make build + env: + SAGE_NUM_THREADS: 4 command: echo 'start script' # List the ports to expose. Learn more https://www.gitpod.io/docs/config-ports/ From 341a4ea741ae558e2e563a938ff2446500186f71 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Sat, 23 Oct 2021 18:01:57 +0000 Subject: [PATCH 008/115] Prio to brew --- .gitpod.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitpod.yml b/.gitpod.yml index 6c1550a7dc6..9cd8704de69 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -13,6 +13,7 @@ tasks: sudo apt-get install -y python3-venv libgf2x-dev cliquer libcliquer-dev gmp-ecm libecm-dev glpk-utils libglpk-dev libbraiding-dev liblrcalc-dev libm4rie-dev libmpc-dev libmpfi-dev libmpfr-dev nauty libntl-dev libplanarity-dev planarity # We do not install tox, since it pulls in javascript-common which does not install for some reason brew install arb ecl flint fplll pari pari-elldata pari-galdata pari-galpol pari-seadata tox + export PATH="/home/linuxbrew/.linuxbrew/bin:$PATH" ./bootstrap ./configure -enable-editable # Gitpod sets PIP_USER: yes by default, which leads to problems during build (e.g pip not being installed in the venv) From 79519fe27c084c5fe5b47b0c82cad107d1c8047e Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Sat, 23 Oct 2021 19:26:17 +0000 Subject: [PATCH 009/115] Update brew before installing anything else --- .gitpod.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitpod.yml b/.gitpod.yml index 9cd8704de69..acea6ad1986 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -12,6 +12,8 @@ tasks: # They are not recommended for some reason sudo apt-get install -y python3-venv libgf2x-dev cliquer libcliquer-dev gmp-ecm libecm-dev glpk-utils libglpk-dev libbraiding-dev liblrcalc-dev libm4rie-dev libmpc-dev libmpfi-dev libmpfr-dev nauty libntl-dev libplanarity-dev planarity # We do not install tox, since it pulls in javascript-common which does not install for some reason + brew update + brew upgrade brew install arb ecl flint fplll pari pari-elldata pari-galdata pari-galpol pari-seadata tox export PATH="/home/linuxbrew/.linuxbrew/bin:$PATH" ./bootstrap From bd1fe072c2e993713243495a820240c27aa616fe Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Sat, 23 Oct 2021 20:38:30 +0000 Subject: [PATCH 010/115] Add trac config --- .gitpod.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.gitpod.yml b/.gitpod.yml index acea6ad1986..426aeb0b807 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,6 +1,18 @@ # List the start up tasks. Learn more https://www.gitpod.io/docs/config-start-tasks/ tasks: - - init: | + - name: Setup + before: | + # Setup ssh key for authentication with trac + # In order to use this, generate a new key with `ssh-keygen -f tempkey` and save the private key to gitpod `gp env PRIVATE_SSH_KEY="$( ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + init: | + # Setup trac repo + git remote add trac git@trac.sagemath.org:sage.git -t master + git remote set-url --push trac git@trac.sagemath.org:sage.git + sudo apt-get update # Needed for bootstrap sudo apt-get install -y gettext From 6bb485fcc8be870ceb1af9bd42259b6acaebdbcc Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Sun, 24 Oct 2021 07:58:17 +0000 Subject: [PATCH 011/115] Do not install ecl from brew --- .gitpod.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitpod.yml b/.gitpod.yml index 426aeb0b807..aa04a5c55ef 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -26,7 +26,8 @@ tasks: # We do not install tox, since it pulls in javascript-common which does not install for some reason brew update brew upgrade - brew install arb ecl flint fplll pari pari-elldata pari-galdata pari-galpol pari-seadata tox + brew install arb flint fplll pari pari-elldata pari-galdata pari-galpol pari-seadata tox + # We do not install ecl from brew, since this breaks the build of maxima export PATH="/home/linuxbrew/.linuxbrew/bin:$PATH" ./bootstrap ./configure -enable-editable From 1ea950dc983dd926cae46c2dc40ca6e01766b907 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Sun, 24 Oct 2021 12:20:08 +0000 Subject: [PATCH 012/115] Cleanup gitpod config --- .gitpod.yml | 47 +++++++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index aa04a5c55ef..15670ae2402 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,10 +1,10 @@ -# List the start up tasks. Learn more https://www.gitpod.io/docs/config-start-tasks/ +# Start up tasks. https://www.gitpod.io/docs/config-start-tasks/ tasks: - name: Setup before: | # Setup ssh key for authentication with trac - # In order to use this, generate a new key with `ssh-keygen -f tempkey` and save the private key to gitpod `gp env PRIVATE_SSH_KEY="$( ~/.ssh/id_rsa chmod 600 ~/.ssh/id_rsa @@ -13,34 +13,41 @@ tasks: git remote add trac git@trac.sagemath.org:sage.git -t master git remote set-url --push trac git@trac.sagemath.org:sage.git + # Install system packages sudo apt-get update - # Needed for bootstrap + ## Needed for bootstrap sudo apt-get install -y gettext - # From dockerfile + ## From dockerfile sudo apt-get install -y --no-install-recommends gfortran gcc g++ libstdc++-9-dev sudo openssl sudo apt-get install -y wget build-essential automake m4 dpkg-dev python libssl-dev git rdfind - # Recommended by ./configure - sudo apt-get install -y libflint-arb-dev libbrial-dev libbrial-groebner-dev libcdd-dev libcdd-tools ecl libec-dev eclib-tools fflas-ffpack flintqs libgc-dev gfan libgiac-dev xcas libgsl-dev libiml-dev lcalc liblfunction-dev libhomfly-dev libopenblas-dev palp pari-gp2c libpari-dev pari-doc pari-elldata pari-galdata pari-galpol pari-seadata libppl-dev ppl-dev python3 libpython3-dev python3-distutils r-base-dev r-cran-lattice librw-dev libsuitesparse-dev libsymmetrica2-dev sympow tachyon libzmq3-dev libzn-poly-dev - # They are not recommended for some reason - sudo apt-get install -y python3-venv libgf2x-dev cliquer libcliquer-dev gmp-ecm libecm-dev glpk-utils libglpk-dev libbraiding-dev liblrcalc-dev libm4rie-dev libmpc-dev libmpfi-dev libmpfr-dev nauty libntl-dev libplanarity-dev planarity - # We do not install tox, since it pulls in javascript-common which does not install for some reason + ## Recommended by ./configure + ### We do not install tox, since it pulls in javascript-common which does not install for some reason + sudo apt-get install -y libflint-arb-dev libbrial-dev libbrial-groebner-dev libcdd-dev libcdd-tools ecl libec-dev eclib-tools fflas-ffpack flintqs libgc-dev gfan libgiac-dev xcas libgsl-dev libiml-dev lcalc liblfunction-dev libhomfly-dev libopenblas-dev palp pari-gp2c libpari-dev pari-doc pari-elldata pari-galdata pari-galpol pari-seadata libppl-dev ppl-dev python3 libpython3-dev python3-distutils r-base-dev r-cran-lattice librw-dev libsuitesparse-dev libsymmetrica2-dev sympow tachyon libzmq3-dev libzn-poly-dev python3-venv libgf2x-dev cliquer libcliquer-dev gmp-ecm libecm-dev glpk-utils libglpk-dev libbraiding-dev liblrcalc-dev libm4rie-dev libmpc-dev libmpfi-dev libmpfr-dev nauty libntl-dev libplanarity-dev planarity + ## Homebrew has more up-to-date package versions + ### We do not install ecl from brew, since this breaks the build of maxima brew update brew upgrade brew install arb flint fplll pari pari-elldata pari-galdata pari-galpol pari-seadata tox - # We do not install ecl from brew, since this breaks the build of maxima + ### Give prio to brew over system packages export PATH="/home/linuxbrew/.linuxbrew/bin:$PATH" - ./bootstrap - ./configure -enable-editable - # Gitpod sets PIP_USER: yes by default, which leads to problems during build (e.g pip not being installed in the venv) + + # Configure + ## Gitpod sets PIP_USER: yes by default, which leads to problems during build (e.g pip not being installed in the venv) unset PIP_USER - # Gitpod installs pyenv by default, and sage's pip install targets the pyenv python for some reason + ## Gitpod installs pyenv by default, and sage's pip install targets the pyenv python for some reason pyenv global system + + # Start build + ./bootstrap + ./configure -enable-editable MAKE='make -j8' make build env: SAGE_NUM_THREADS: 4 - command: echo 'start script' -# List the ports to expose. Learn more https://www.gitpod.io/docs/config-ports/ -ports: - - port: 3000 - onOpen: open-preview +# Preinstalled VS Code extensions. https://www.gitpod.io/docs/vscode-extensions +vscode: + extensions: + - ms-pyright.pyright + - ms-python.python + - lextudio.restructuredtext + - streetsidesoftware.code-spell-checker From d792c6b2a8b13ceb99834c045ff3fa652695ab5a Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Sun, 24 Oct 2021 12:20:25 +0000 Subject: [PATCH 013/115] Add docs --- src/doc/en/developer/index.rst | 9 ++++++++ src/doc/en/developer/workspace.rst | 34 ++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 src/doc/en/developer/workspace.rst diff --git a/src/doc/en/developer/index.rst b/src/doc/en/developer/index.rst index 899ebe45a88..d5dadb87f96 100644 --- a/src/doc/en/developer/index.rst +++ b/src/doc/en/developer/index.rst @@ -41,6 +41,14 @@ development! `_ on your system. + As an easy way to get started, you can run and edit Sage's code and contribute + your changes using `Gitpod `_, + a free online development environment based on VS Code. + It will launch a pre-made workspace with all dependencies and tools installed + so that you can start contributing straight away. + Start by `going to Gitpod `_, + and read :ref:`our Gitpod guidelines ` to learn more. + - **Conventions:** read our :ref:`conventions and guidelines ` for code and documentation. @@ -115,6 +123,7 @@ Writing Code for Sage .. toctree:: :maxdepth: 3 + workspace coding_basics reviewer_checklist diff --git a/src/doc/en/developer/workspace.rst b/src/doc/en/developer/workspace.rst new file mode 100644 index 00000000000..cbc2b12d87f --- /dev/null +++ b/src/doc/en/developer/workspace.rst @@ -0,0 +1,34 @@ +.. highlight:: shell-session + +.. _chapter-workspace-setup: + +========================= +Setting up your workspace +========================= + +.. _section-gitpod: + +Gitpod +================= + +`Gitpod `_ is a free service that will let you build and +run Sage from an online development environment based on VS Code. +Without needing to install anything on your computer, Gitpod creates a virtual +fully-functional workspace with all the dependencies and tools pre-installed. + +To get started, `go to Gitpod `_ +and log in using your GitHub or GitLab account. +Wait while Gitpod creates a workspace. +The first time, it may take some time to build Sage. + +You can now run and edit Sage's code. Contributing your changes follows the normal +:ref:`Git workflow `. +For this to work, you first have to authorize Gitpod with Trac: + + 1. In the running Gitpod workspace, generate a new SSH key pair by `ssh-keygen -f tempkey`. + 2. Save the private key as a secure environment variable in Gitpod using `gp env PRIVATE_SSH_KEY="$(`_. + 3. Register the public key with Trac following the instructions in :ref:`section-trac-ssh-key`. + +You can also `use your VS Code Desktop `_ to keep +your local IDE configuration while still benefiting from Gitpod’s high-spec servers and automated prebuilds. \ No newline at end of file From 1ae507de86dcd8f8bcc992138451976eda8a6f03 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Sun, 24 Oct 2021 12:35:55 +0000 Subject: [PATCH 014/115] Remove uunnecessary changes to sage-venv --- build/bin/sage-venv | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build/bin/sage-venv b/build/bin/sage-venv index fb144a8ea92..9ee9fc794be 100755 --- a/build/bin/sage-venv +++ b/build/bin/sage-venv @@ -49,8 +49,7 @@ else: b = venv.EnvBuilder(system_site_packages=options.system_site, clear=options.clear, upgrade=options.upgrade, - symlinks=use_symlinks, - with_pip=True) + symlinks=use_symlinks) c = b.ensure_directories(options.env_dir) b.setup_python(c) b.create_configuration(c) From a973734cc6855fb4a162fd3e6c24230784eb7412 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Mon, 25 Oct 2021 12:02:19 +0000 Subject: [PATCH 015/115] Move system setup to dockerfile --- .gitpod.yml | 29 +++++------------------------ docker/.gitpod.Dockerfile | 24 ++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 24 deletions(-) create mode 100644 docker/.gitpod.Dockerfile diff --git a/.gitpod.yml b/.gitpod.yml index 15670ae2402..2a617c3211b 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,3 +1,7 @@ +# Use custom docker image. https://www.gitpod.io/docs/config-docker +image: + file: docker/.gitpod.Dockerfile + # Start up tasks. https://www.gitpod.io/docs/config-start-tasks/ tasks: - name: Setup @@ -13,30 +17,6 @@ tasks: git remote add trac git@trac.sagemath.org:sage.git -t master git remote set-url --push trac git@trac.sagemath.org:sage.git - # Install system packages - sudo apt-get update - ## Needed for bootstrap - sudo apt-get install -y gettext - ## From dockerfile - sudo apt-get install -y --no-install-recommends gfortran gcc g++ libstdc++-9-dev sudo openssl - sudo apt-get install -y wget build-essential automake m4 dpkg-dev python libssl-dev git rdfind - ## Recommended by ./configure - ### We do not install tox, since it pulls in javascript-common which does not install for some reason - sudo apt-get install -y libflint-arb-dev libbrial-dev libbrial-groebner-dev libcdd-dev libcdd-tools ecl libec-dev eclib-tools fflas-ffpack flintqs libgc-dev gfan libgiac-dev xcas libgsl-dev libiml-dev lcalc liblfunction-dev libhomfly-dev libopenblas-dev palp pari-gp2c libpari-dev pari-doc pari-elldata pari-galdata pari-galpol pari-seadata libppl-dev ppl-dev python3 libpython3-dev python3-distutils r-base-dev r-cran-lattice librw-dev libsuitesparse-dev libsymmetrica2-dev sympow tachyon libzmq3-dev libzn-poly-dev python3-venv libgf2x-dev cliquer libcliquer-dev gmp-ecm libecm-dev glpk-utils libglpk-dev libbraiding-dev liblrcalc-dev libm4rie-dev libmpc-dev libmpfi-dev libmpfr-dev nauty libntl-dev libplanarity-dev planarity - ## Homebrew has more up-to-date package versions - ### We do not install ecl from brew, since this breaks the build of maxima - brew update - brew upgrade - brew install arb flint fplll pari pari-elldata pari-galdata pari-galpol pari-seadata tox - ### Give prio to brew over system packages - export PATH="/home/linuxbrew/.linuxbrew/bin:$PATH" - - # Configure - ## Gitpod sets PIP_USER: yes by default, which leads to problems during build (e.g pip not being installed in the venv) - unset PIP_USER - ## Gitpod installs pyenv by default, and sage's pip install targets the pyenv python for some reason - pyenv global system - # Start build ./bootstrap ./configure -enable-editable @@ -51,3 +31,4 @@ vscode: - ms-python.python - lextudio.restructuredtext - streetsidesoftware.code-spell-checker + - ms-toolsai.jupyter diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile new file mode 100644 index 00000000000..2864c167c8a --- /dev/null +++ b/docker/.gitpod.Dockerfile @@ -0,0 +1,24 @@ +FROM gitpod/workspace-full + +# Install system packages +RUN sudo apt-get update +## Needed for bootstrap +RUN sudo apt-get install -y gettext +## From dockerfile +RUN sudo apt-get install -y --no-install-recommends gfortran gcc g++ libstdc++-9-dev sudo openssl +RUN sudo apt-get install -y wget build-essential automake m4 dpkg-dev python libssl-dev git rdfind +## Recommended by ./configure +### We do not install tox, since it pulls in javascript-common which does not install for some reason +RUN sudo apt-get install -y libflint-arb-dev libbrial-dev libbrial-groebner-dev libcdd-dev libcdd-tools ecl libec-dev eclib-tools fflas-ffpack flintqs libgc-dev gfan libgiac-dev xcas libgsl-dev libiml-dev lcalc liblfunction-dev libhomfly-dev libopenblas-dev palp pari-gp2c libpari-dev pari-doc pari-elldata pari-galdata pari-galpol pari-seadata libppl-dev ppl-dev python3 libpython3-dev python3-distutils r-base-dev r-cran-lattice librw-dev libsuitesparse-dev libsymmetrica2-dev sympow tachyon libzmq3-dev libzn-poly-dev python3-venv libgf2x-dev cliquer libcliquer-dev gmp-ecm libecm-dev glpk-utils libglpk-dev libbraiding-dev liblrcalc-dev libm4rie-dev libmpc-dev libmpfi-dev libmpfr-dev nauty libntl-dev libplanarity-dev planarity +## Homebrew has more up-to-date package versions +### We do not install ecl from brew, since this breaks the build of maxima +RUN brew update && brew upgrade +RUN brew install arb flint fplll pari pari-elldata pari-galdata pari-galpol pari-seadata tox || true +### Give prio to brew over system packages +ENV PATH="/home/linuxbrew/.linuxbrew/bin:$PATH" + +# Configure +## Gitpod sets PIP_USER: yes by default, which leads to problems during build (e.g pip not being installed in the venv) +RUN unset PIP_USER +## Gitpod installs pyenv by default, and sage's pip install targets the pyenv python for some reason +RUN pyenv global system From 5d3e6da15d0406b5139a9ad54d99d07876ecbb60 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Mon, 25 Oct 2021 12:59:26 +0000 Subject: [PATCH 016/115] Disable brew prio --- docker/.gitpod.Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile index 2864c167c8a..d9ae9400abe 100644 --- a/docker/.gitpod.Dockerfile +++ b/docker/.gitpod.Dockerfile @@ -15,7 +15,8 @@ RUN sudo apt-get install -y libflint-arb-dev libbrial-dev libbrial-groebner-dev RUN brew update && brew upgrade RUN brew install arb flint fplll pari pari-elldata pari-galdata pari-galpol pari-seadata tox || true ### Give prio to brew over system packages -ENV PATH="/home/linuxbrew/.linuxbrew/bin:$PATH" +#Currently doesn't work as gitpod gp executable is then hidden by pari/gp +#ENV PATH="/home/linuxbrew/.linuxbrew/bin:$PATH" # Configure ## Gitpod sets PIP_USER: yes by default, which leads to problems during build (e.g pip not being installed in the venv) From 51c69e3ffdbeff930e810e6df6492542d4b8269f Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Fri, 26 Nov 2021 16:12:16 +0100 Subject: [PATCH 017/115] No need to intstall git again --- docker/.gitpod.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile index d9ae9400abe..c0dff8aae0e 100644 --- a/docker/.gitpod.Dockerfile +++ b/docker/.gitpod.Dockerfile @@ -6,7 +6,7 @@ RUN sudo apt-get update RUN sudo apt-get install -y gettext ## From dockerfile RUN sudo apt-get install -y --no-install-recommends gfortran gcc g++ libstdc++-9-dev sudo openssl -RUN sudo apt-get install -y wget build-essential automake m4 dpkg-dev python libssl-dev git rdfind +RUN sudo apt-get install -y wget build-essential automake m4 dpkg-dev python libssl-dev rdfind ## Recommended by ./configure ### We do not install tox, since it pulls in javascript-common which does not install for some reason RUN sudo apt-get install -y libflint-arb-dev libbrial-dev libbrial-groebner-dev libcdd-dev libcdd-tools ecl libec-dev eclib-tools fflas-ffpack flintqs libgc-dev gfan libgiac-dev xcas libgsl-dev libiml-dev lcalc liblfunction-dev libhomfly-dev libopenblas-dev palp pari-gp2c libpari-dev pari-doc pari-elldata pari-galdata pari-galpol pari-seadata libppl-dev ppl-dev python3 libpython3-dev python3-distutils r-base-dev r-cran-lattice librw-dev libsuitesparse-dev libsymmetrica2-dev sympow tachyon libzmq3-dev libzn-poly-dev python3-venv libgf2x-dev cliquer libcliquer-dev gmp-ecm libecm-dev glpk-utils libglpk-dev libbraiding-dev liblrcalc-dev libm4rie-dev libmpc-dev libmpfi-dev libmpfr-dev nauty libntl-dev libplanarity-dev planarity From 8cf8b387a768ead8f391b3f909e6fc212d002098 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Wed, 8 Dec 2021 23:10:04 +0100 Subject: [PATCH 018/115] Prebuild non-python packages in docker --- docker/.gitpod.Dockerfile | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile index c0dff8aae0e..d0d94eb127d 100644 --- a/docker/.gitpod.Dockerfile +++ b/docker/.gitpod.Dockerfile @@ -1,4 +1,7 @@ -FROM gitpod/workspace-full +## +## Install system packages +## +FROM gitpod/workspace-full as prepare # Install system packages RUN sudo apt-get update @@ -11,12 +14,32 @@ RUN sudo apt-get install -y wget build-essential automake m4 dpkg-dev python lib ### We do not install tox, since it pulls in javascript-common which does not install for some reason RUN sudo apt-get install -y libflint-arb-dev libbrial-dev libbrial-groebner-dev libcdd-dev libcdd-tools ecl libec-dev eclib-tools fflas-ffpack flintqs libgc-dev gfan libgiac-dev xcas libgsl-dev libiml-dev lcalc liblfunction-dev libhomfly-dev libopenblas-dev palp pari-gp2c libpari-dev pari-doc pari-elldata pari-galdata pari-galpol pari-seadata libppl-dev ppl-dev python3 libpython3-dev python3-distutils r-base-dev r-cran-lattice librw-dev libsuitesparse-dev libsymmetrica2-dev sympow tachyon libzmq3-dev libzn-poly-dev python3-venv libgf2x-dev cliquer libcliquer-dev gmp-ecm libecm-dev glpk-utils libglpk-dev libbraiding-dev liblrcalc-dev libm4rie-dev libmpc-dev libmpfi-dev libmpfr-dev nauty libntl-dev libplanarity-dev planarity ## Homebrew has more up-to-date package versions -### We do not install ecl from brew, since this breaks the build of maxima RUN brew update && brew upgrade -RUN brew install arb flint fplll pari pari-elldata pari-galdata pari-galpol pari-seadata tox || true -### Give prio to brew over system packages -#Currently doesn't work as gitpod gp executable is then hidden by pari/gp -#ENV PATH="/home/linuxbrew/.linuxbrew/bin:$PATH" +RUN brew install arb flint fplll tox || true +### We do not install ecl from brew, since this breaks the build of maxima +### Installing pari from brew doesn't work as gitpod gp executable is then hidden by pari/gp +### RUN brew install pari pari-elldata pari-galdata pari-galpol pari-seadata + +## Give prio to brew over other system packages +ENV PATH="/home/linuxbrew/.linuxbrew/bin:$PATH" + +## +## Prebuild non-Python packages that have no (working) system-installed package +## +FROM prepare as prebuild +USER gitpod +COPY --chown=gitpod:gitpod . . +RUN ./bootstrap +RUN ./configure --prefix=/home/gitpod/sage-prebuild +RUN MAKE='make -j8' make build-local + +## +## Build final image +## +FROM prepare +# Reuse the prebuild packages +COPY --from=prebuild /home/gitpod/sage-prebuild /home/gitpod/sage-prebuild +ENV PATH=/home/gitpod/sage-prebuild/bin:$PATH # Configure ## Gitpod sets PIP_USER: yes by default, which leads to problems during build (e.g pip not being installed in the venv) From 1aa950023bb150c47099f6e820658a9d1d652f81 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Thu, 9 Dec 2021 13:04:55 +0100 Subject: [PATCH 019/115] Fix log output --- docker/.gitpod.Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile index d0d94eb127d..c4a632a29a7 100644 --- a/docker/.gitpod.Dockerfile +++ b/docker/.gitpod.Dockerfile @@ -13,9 +13,9 @@ RUN sudo apt-get install -y wget build-essential automake m4 dpkg-dev python lib ## Recommended by ./configure ### We do not install tox, since it pulls in javascript-common which does not install for some reason RUN sudo apt-get install -y libflint-arb-dev libbrial-dev libbrial-groebner-dev libcdd-dev libcdd-tools ecl libec-dev eclib-tools fflas-ffpack flintqs libgc-dev gfan libgiac-dev xcas libgsl-dev libiml-dev lcalc liblfunction-dev libhomfly-dev libopenblas-dev palp pari-gp2c libpari-dev pari-doc pari-elldata pari-galdata pari-galpol pari-seadata libppl-dev ppl-dev python3 libpython3-dev python3-distutils r-base-dev r-cran-lattice librw-dev libsuitesparse-dev libsymmetrica2-dev sympow tachyon libzmq3-dev libzn-poly-dev python3-venv libgf2x-dev cliquer libcliquer-dev gmp-ecm libecm-dev glpk-utils libglpk-dev libbraiding-dev liblrcalc-dev libm4rie-dev libmpc-dev libmpfi-dev libmpfr-dev nauty libntl-dev libplanarity-dev planarity -## Homebrew has more up-to-date package versions +## Homebrew has some more up-to-date packages RUN brew update && brew upgrade -RUN brew install arb flint fplll tox || true +RUN brew install arb flint fplll tox ### We do not install ecl from brew, since this breaks the build of maxima ### Installing pari from brew doesn't work as gitpod gp executable is then hidden by pari/gp ### RUN brew install pari pari-elldata pari-galdata pari-galpol pari-seadata @@ -31,7 +31,8 @@ USER gitpod COPY --chown=gitpod:gitpod . . RUN ./bootstrap RUN ./configure --prefix=/home/gitpod/sage-prebuild -RUN MAKE='make -j8' make build-local +### Hide output since otherwise we would reach log limit +RUN MAKE='make -j8' make build-local > /dev/null ## ## Build final image From 8c83420d615be972979f62f0a148b8ee3f3f2ce5 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Thu, 9 Dec 2021 16:24:28 +0100 Subject: [PATCH 020/115] Reduce image build time --- docker/.gitpod.Dockerfile | 92 ++++++++++++++++++++++++++++++++------- 1 file changed, 76 insertions(+), 16 deletions(-) diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile index c4a632a29a7..aedaaf8fba1 100644 --- a/docker/.gitpod.Dockerfile +++ b/docker/.gitpod.Dockerfile @@ -5,23 +5,82 @@ FROM gitpod/workspace-full as prepare # Install system packages RUN sudo apt-get update -## Needed for bootstrap -RUN sudo apt-get install -y gettext -## From dockerfile -RUN sudo apt-get install -y --no-install-recommends gfortran gcc g++ libstdc++-9-dev sudo openssl -RUN sudo apt-get install -y wget build-essential automake m4 dpkg-dev python libssl-dev rdfind -## Recommended by ./configure -### We do not install tox, since it pulls in javascript-common which does not install for some reason -RUN sudo apt-get install -y libflint-arb-dev libbrial-dev libbrial-groebner-dev libcdd-dev libcdd-tools ecl libec-dev eclib-tools fflas-ffpack flintqs libgc-dev gfan libgiac-dev xcas libgsl-dev libiml-dev lcalc liblfunction-dev libhomfly-dev libopenblas-dev palp pari-gp2c libpari-dev pari-doc pari-elldata pari-galdata pari-galpol pari-seadata libppl-dev ppl-dev python3 libpython3-dev python3-distutils r-base-dev r-cran-lattice librw-dev libsuitesparse-dev libsymmetrica2-dev sympow tachyon libzmq3-dev libzn-poly-dev python3-venv libgf2x-dev cliquer libcliquer-dev gmp-ecm libecm-dev glpk-utils libglpk-dev libbraiding-dev liblrcalc-dev libm4rie-dev libmpc-dev libmpfi-dev libmpfr-dev nauty libntl-dev libplanarity-dev planarity -## Homebrew has some more up-to-date packages -RUN brew update && brew upgrade -RUN brew install arb flint fplll tox +RUN sudo apt-get install -y --no-install-recommends \ + ## From main dockerfile and/or needed for bootstrap + wget \ + build-essential \ + automake \ + m4 \ + dpkg-dev \ + python \ + libssl-dev \ + rdfind \ + gettext \ + gfortran \ + gcc \ + g++ \ + libstdc++-9-dev \ + sudo \ + openssl \ + ## Recommended by ./configure + libflint-arb-dev \ + libbrial-dev libbrial-groebner-dev \ + libcdd-dev libcdd-tools \ + ecl eclib-tools \ + libec-dev \ + fflas-ffpack \ + flintqs \ + libgc-dev \ + gfan \ + libgiac-dev \ + xcas \ + libgsl-dev \ + libiml-dev \ + lcalc \ + liblfunction-dev \ + libhomfly-dev \ + libopenblas-dev \ + palp \ + # pari is not recognized even if installed + #pari-gp2c libpari-dev pari-doc pari-elldata pari-galdata pari-galpol pari-seadata + libppl-dev ppl-dev \ + python3 libpython3-dev python3-distutils python3-venv \ + r-base-dev r-cran-lattice \ + librw-dev \ + libsuitesparse-dev \ + libsymmetrica2-dev \ + sympow \ + tachyon \ + libzmq3-dev \ + libzn-poly-dev \ + libgf2x-dev \ + cliquer \ + libcliquer-dev \ + gmp-ecm \ + libecm-dev \ + glpk-utils \ + libglpk-dev \ + libbraiding-dev \ + liblrcalc-dev \ + libm4rie-dev \ + libmpc-dev \ + libmpfi-dev \ + libmpfr-dev \ + nauty \ + libntl-dev \ + libplanarity-dev \ + planarity + ## We do not install tox, since it pulls in javascript-common which does not install for some reason + #tox + +## Homebrew has some more up-to-date packages (but sage is not yet able to find them) +### RUN brew update && brew upgrade +### RUN brew install arb flint fplll tox ### We do not install ecl from brew, since this breaks the build of maxima ### Installing pari from brew doesn't work as gitpod gp executable is then hidden by pari/gp ### RUN brew install pari pari-elldata pari-galdata pari-galpol pari-seadata - -## Give prio to brew over other system packages -ENV PATH="/home/linuxbrew/.linuxbrew/bin:$PATH" +### Give prio to brew over other system packages +### ENV PATH="/home/linuxbrew/.linuxbrew/bin:$PATH" ## ## Prebuild non-Python packages that have no (working) system-installed package @@ -31,8 +90,9 @@ USER gitpod COPY --chown=gitpod:gitpod . . RUN ./bootstrap RUN ./configure --prefix=/home/gitpod/sage-prebuild -### Hide output since otherwise we would reach log limit -RUN MAKE='make -j8' make build-local > /dev/null +### Hide output since otherwise we would reach log limit +### Gitpod also puts a timeout at 1h, so we cannot install everything here with `make build-local` +RUN MAKE='make -j8' make arb ecl > /dev/null ## ## Build final image From ea6c0a3f8c30aedf53b39fde71edb23947645efd Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Thu, 9 Dec 2021 17:40:34 +0100 Subject: [PATCH 021/115] Prebuild a few more libs --- docker/.gitpod.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile index aedaaf8fba1..c4f5d8e93d5 100644 --- a/docker/.gitpod.Dockerfile +++ b/docker/.gitpod.Dockerfile @@ -92,7 +92,7 @@ RUN ./bootstrap RUN ./configure --prefix=/home/gitpod/sage-prebuild ### Hide output since otherwise we would reach log limit ### Gitpod also puts a timeout at 1h, so we cannot install everything here with `make build-local` -RUN MAKE='make -j8' make arb ecl > /dev/null +RUN MAKE='make -j8' make arb ecl flint cddlib eclib fplll gap giac > /dev/null ## ## Build final image From 40a07dd6aefeb01f930e633f53406a7be627ec9d Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Thu, 9 Dec 2021 19:01:12 +0100 Subject: [PATCH 022/115] Install pari instead of gap, and set env correctly --- docker/.gitpod.Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile index c4f5d8e93d5..8a21da09042 100644 --- a/docker/.gitpod.Dockerfile +++ b/docker/.gitpod.Dockerfile @@ -92,7 +92,7 @@ RUN ./bootstrap RUN ./configure --prefix=/home/gitpod/sage-prebuild ### Hide output since otherwise we would reach log limit ### Gitpod also puts a timeout at 1h, so we cannot install everything here with `make build-local` -RUN MAKE='make -j8' make arb ecl flint cddlib eclib fplll gap giac > /dev/null +RUN MAKE='make -j8' make arb ecl flint cddlib eclib fplll giac gengetopt pari > /dev/null ## ## Build final image @@ -101,6 +101,9 @@ FROM prepare # Reuse the prebuild packages COPY --from=prebuild /home/gitpod/sage-prebuild /home/gitpod/sage-prebuild ENV PATH=/home/gitpod/sage-prebuild/bin:$PATH +ENV PKG_CONFIG_PATH=/home/gitpod/sage-prebuild/lib/pkgconfig:$PKG_CONFIG_PATH +ENV CPPFLAGS=-I/home/gitpod/sage-prebuild/include $CPPFLAGS +ENV LDFLAGS=-L/home/gitpod/sage-prebuild/lib $LDFLAGS # Configure ## Gitpod sets PIP_USER: yes by default, which leads to problems during build (e.g pip not being installed in the venv) From 2ec7921c5297d3306a0d1e691777c158b0f51311 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Thu, 9 Dec 2021 19:04:04 +0100 Subject: [PATCH 023/115] Fix syntax --- docker/.gitpod.Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile index 8a21da09042..4b05db91539 100644 --- a/docker/.gitpod.Dockerfile +++ b/docker/.gitpod.Dockerfile @@ -102,8 +102,8 @@ FROM prepare COPY --from=prebuild /home/gitpod/sage-prebuild /home/gitpod/sage-prebuild ENV PATH=/home/gitpod/sage-prebuild/bin:$PATH ENV PKG_CONFIG_PATH=/home/gitpod/sage-prebuild/lib/pkgconfig:$PKG_CONFIG_PATH -ENV CPPFLAGS=-I/home/gitpod/sage-prebuild/include $CPPFLAGS -ENV LDFLAGS=-L/home/gitpod/sage-prebuild/lib $LDFLAGS +ENV CPPFLAGS="-I/home/gitpod/sage-prebuild/include $CPPFLAGS" +ENV LDFLAGS="-L/home/gitpod/sage-prebuild/lib $LDFLAGS" # Configure ## Gitpod sets PIP_USER: yes by default, which leads to problems during build (e.g pip not being installed in the venv) From 00394d2ff668acb13563761e14889788bd5c040b Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Thu, 9 Dec 2021 20:07:40 +0100 Subject: [PATCH 024/115] Don't need to install sudo --- docker/.gitpod.Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile index 4b05db91539..3911b32a92c 100644 --- a/docker/.gitpod.Dockerfile +++ b/docker/.gitpod.Dockerfile @@ -20,7 +20,6 @@ RUN sudo apt-get install -y --no-install-recommends \ gcc \ g++ \ libstdc++-9-dev \ - sudo \ openssl \ ## Recommended by ./configure libflint-arb-dev \ From e43768f7839a862c85d47a5298ccbafc5a673393 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Thu, 9 Dec 2021 22:33:01 +0100 Subject: [PATCH 025/115] Try again without pari --- docker/.gitpod.Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile index 3911b32a92c..08c0b3fba44 100644 --- a/docker/.gitpod.Dockerfile +++ b/docker/.gitpod.Dockerfile @@ -91,7 +91,8 @@ RUN ./bootstrap RUN ./configure --prefix=/home/gitpod/sage-prebuild ### Hide output since otherwise we would reach log limit ### Gitpod also puts a timeout at 1h, so we cannot install everything here with `make build-local` -RUN MAKE='make -j8' make arb ecl flint cddlib eclib fplll giac gengetopt pari > /dev/null +RUN MAKE='make -j8' make arb ecl flint cddlib eclib fplll giac gengetopt > /dev/null + ## ## Build final image From 962ab18db098ef638a7aa2fa36c242e0a9981800 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Thu, 9 Dec 2021 22:40:45 +0100 Subject: [PATCH 026/115] Try disabling most of the docker build... --- docker/.gitpod.Dockerfile | 58 +++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile index 08c0b3fba44..ad0f3cc1f95 100644 --- a/docker/.gitpod.Dockerfile +++ b/docker/.gitpod.Dockerfile @@ -81,32 +81,32 @@ RUN sudo apt-get install -y --no-install-recommends \ ### Give prio to brew over other system packages ### ENV PATH="/home/linuxbrew/.linuxbrew/bin:$PATH" -## -## Prebuild non-Python packages that have no (working) system-installed package -## -FROM prepare as prebuild -USER gitpod -COPY --chown=gitpod:gitpod . . -RUN ./bootstrap -RUN ./configure --prefix=/home/gitpod/sage-prebuild -### Hide output since otherwise we would reach log limit -### Gitpod also puts a timeout at 1h, so we cannot install everything here with `make build-local` -RUN MAKE='make -j8' make arb ecl flint cddlib eclib fplll giac gengetopt > /dev/null - - -## -## Build final image -## -FROM prepare -# Reuse the prebuild packages -COPY --from=prebuild /home/gitpod/sage-prebuild /home/gitpod/sage-prebuild -ENV PATH=/home/gitpod/sage-prebuild/bin:$PATH -ENV PKG_CONFIG_PATH=/home/gitpod/sage-prebuild/lib/pkgconfig:$PKG_CONFIG_PATH -ENV CPPFLAGS="-I/home/gitpod/sage-prebuild/include $CPPFLAGS" -ENV LDFLAGS="-L/home/gitpod/sage-prebuild/lib $LDFLAGS" - -# Configure -## Gitpod sets PIP_USER: yes by default, which leads to problems during build (e.g pip not being installed in the venv) -RUN unset PIP_USER -## Gitpod installs pyenv by default, and sage's pip install targets the pyenv python for some reason -RUN pyenv global system +### +### Prebuild non-Python packages that have no (working) system-installed package +### +#FROM prepare as prebuild +#USER gitpod +#COPY --chown=gitpod:gitpod . . +#RUN ./bootstrap +#RUN ./configure --prefix=/home/gitpod/sage-prebuild +#### Hide output since otherwise we would reach log limit +#### Gitpod also puts a timeout at 1h, so we cannot install everything here with `make build-local` +#RUN MAKE='make -j8' make arb ecl flint cddlib eclib fplll giac gengetopt > /dev/null +# +# +### +### Build final image +### +#FROM prepare +## Reuse the prebuild packages +#COPY --from=prebuild /home/gitpod/sage-prebuild /home/gitpod/sage-prebuild +#ENV PATH=/home/gitpod/sage-prebuild/bin:$PATH +#ENV PKG_CONFIG_PATH=/home/gitpod/sage-prebuild/lib/pkgconfig:$PKG_CONFIG_PATH +#ENV CPPFLAGS="-I/home/gitpod/sage-prebuild/include $CPPFLAGS" +#ENV LDFLAGS="-L/home/gitpod/sage-prebuild/lib $LDFLAGS" +# +## Configure +### Gitpod sets PIP_USER: yes by default, which leads to problems during build (e.g pip not being installed in the venv) +#RUN unset PIP_USER +### Gitpod installs pyenv by default, and sage's pip install targets the pyenv python for some reason +#RUN pyenv global system From 22d046267b1fd7a7e8bc368ae67f8bfc216a13a2 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Thu, 9 Dec 2021 23:14:39 +0100 Subject: [PATCH 027/115] Reenable env config --- docker/.gitpod.Dockerfile | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile index ad0f3cc1f95..94c5c3cbf9f 100644 --- a/docker/.gitpod.Dockerfile +++ b/docker/.gitpod.Dockerfile @@ -93,20 +93,20 @@ RUN sudo apt-get install -y --no-install-recommends \ #### Gitpod also puts a timeout at 1h, so we cannot install everything here with `make build-local` #RUN MAKE='make -j8' make arb ecl flint cddlib eclib fplll giac gengetopt > /dev/null # -# -### -### Build final image -### -#FROM prepare -## Reuse the prebuild packages + +## +## Build final image +## +FROM prepare +# Reuse the prebuild packages #COPY --from=prebuild /home/gitpod/sage-prebuild /home/gitpod/sage-prebuild -#ENV PATH=/home/gitpod/sage-prebuild/bin:$PATH -#ENV PKG_CONFIG_PATH=/home/gitpod/sage-prebuild/lib/pkgconfig:$PKG_CONFIG_PATH -#ENV CPPFLAGS="-I/home/gitpod/sage-prebuild/include $CPPFLAGS" -#ENV LDFLAGS="-L/home/gitpod/sage-prebuild/lib $LDFLAGS" -# -## Configure -### Gitpod sets PIP_USER: yes by default, which leads to problems during build (e.g pip not being installed in the venv) -#RUN unset PIP_USER -### Gitpod installs pyenv by default, and sage's pip install targets the pyenv python for some reason -#RUN pyenv global system +ENV PATH=/home/gitpod/sage-prebuild/bin:$PATH +ENV PKG_CONFIG_PATH=/home/gitpod/sage-prebuild/lib/pkgconfig:$PKG_CONFIG_PATH +ENV CPPFLAGS="-I/home/gitpod/sage-prebuild/include $CPPFLAGS" +ENV LDFLAGS="-L/home/gitpod/sage-prebuild/lib $LDFLAGS" + +# Configure +## Gitpod sets PIP_USER: yes by default, which leads to problems during build (e.g pip not being installed in the venv) +RUN unset PIP_USER +## Gitpod installs pyenv by default, and sage's pip install targets the pyenv python for some reason +RUN pyenv global system From 743e8fc91bf74cd0f236bbd4543cc34e11916b27 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Thu, 9 Dec 2021 23:39:50 +0100 Subject: [PATCH 028/115] Reenable everything --- docker/.gitpod.Dockerfile | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile index 94c5c3cbf9f..5600b1b9bb0 100644 --- a/docker/.gitpod.Dockerfile +++ b/docker/.gitpod.Dockerfile @@ -81,25 +81,25 @@ RUN sudo apt-get install -y --no-install-recommends \ ### Give prio to brew over other system packages ### ENV PATH="/home/linuxbrew/.linuxbrew/bin:$PATH" -### -### Prebuild non-Python packages that have no (working) system-installed package -### -#FROM prepare as prebuild -#USER gitpod -#COPY --chown=gitpod:gitpod . . -#RUN ./bootstrap -#RUN ./configure --prefix=/home/gitpod/sage-prebuild -#### Hide output since otherwise we would reach log limit -#### Gitpod also puts a timeout at 1h, so we cannot install everything here with `make build-local` -#RUN MAKE='make -j8' make arb ecl flint cddlib eclib fplll giac gengetopt > /dev/null -# +## +## Prebuild non-Python packages that have no (working) system-installed package +## +FROM prepare as prebuild +USER gitpod +COPY --chown=gitpod:gitpod . . +RUN ./bootstrap +RUN ./configure --prefix=/home/gitpod/sage-prebuild +### Hide output since otherwise we would reach log limit +### Gitpod also puts a timeout at 1h, so we cannot install everything here with `make build-local` +RUN MAKE='make -j8' make arb ecl flint cddlib eclib fplll giac gengetopt pari > /dev/null + ## ## Build final image ## FROM prepare # Reuse the prebuild packages -#COPY --from=prebuild /home/gitpod/sage-prebuild /home/gitpod/sage-prebuild +COPY --from=prebuild /home/gitpod/sage-prebuild /home/gitpod/sage-prebuild ENV PATH=/home/gitpod/sage-prebuild/bin:$PATH ENV PKG_CONFIG_PATH=/home/gitpod/sage-prebuild/lib/pkgconfig:$PKG_CONFIG_PATH ENV CPPFLAGS="-I/home/gitpod/sage-prebuild/include $CPPFLAGS" From 12cdb6acc6be7bfd3f4cef8edfe153604ec38b75 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Thu, 9 Dec 2021 23:44:32 +0100 Subject: [PATCH 029/115] More bisecting... --- docker/.gitpod.Dockerfile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile index 5600b1b9bb0..49a79b92f98 100644 --- a/docker/.gitpod.Dockerfile +++ b/docker/.gitpod.Dockerfile @@ -86,13 +86,13 @@ RUN sudo apt-get install -y --no-install-recommends \ ## FROM prepare as prebuild USER gitpod -COPY --chown=gitpod:gitpod . . -RUN ./bootstrap -RUN ./configure --prefix=/home/gitpod/sage-prebuild -### Hide output since otherwise we would reach log limit -### Gitpod also puts a timeout at 1h, so we cannot install everything here with `make build-local` -RUN MAKE='make -j8' make arb ecl flint cddlib eclib fplll giac gengetopt pari > /dev/null - +#COPY --chown=gitpod:gitpod . . +#RUN ./bootstrap +#RUN ./configure --prefix=/home/gitpod/sage-prebuild +#### Hide output since otherwise we would reach log limit +#### Gitpod also puts a timeout at 1h, so we cannot install everything here with `make build-local` +#RUN MAKE='make -j8' make arb ecl flint cddlib eclib fplll giac gengetopt > /dev/null +# ## ## Build final image From 23a210cf3d032072c3f6649f5eb8b8003b362cce Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Thu, 9 Dec 2021 23:48:20 +0100 Subject: [PATCH 030/115] Another one... --- docker/.gitpod.Dockerfile | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile index 49a79b92f98..3417fe2326c 100644 --- a/docker/.gitpod.Dockerfile +++ b/docker/.gitpod.Dockerfile @@ -87,12 +87,11 @@ RUN sudo apt-get install -y --no-install-recommends \ FROM prepare as prebuild USER gitpod #COPY --chown=gitpod:gitpod . . -#RUN ./bootstrap -#RUN ./configure --prefix=/home/gitpod/sage-prebuild -#### Hide output since otherwise we would reach log limit -#### Gitpod also puts a timeout at 1h, so we cannot install everything here with `make build-local` -#RUN MAKE='make -j8' make arb ecl flint cddlib eclib fplll giac gengetopt > /dev/null -# +RUN ./bootstrap +RUN ./configure --prefix=/home/gitpod/sage-prebuild +### Hide output since otherwise we would reach log limit +### Gitpod also puts a timeout at 1h, so we cannot install everything here with `make build-local` +RUN MAKE='make -j8' make arb ecl flint cddlib eclib fplll giac gengetopt pari > /dev/null ## ## Build final image From 5eeae07b406ebcb74187ead704bd20167998b21c Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Thu, 9 Dec 2021 23:58:22 +0100 Subject: [PATCH 031/115] Reactivate everything? --- docker/.gitpod.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile index 3417fe2326c..3911b32a92c 100644 --- a/docker/.gitpod.Dockerfile +++ b/docker/.gitpod.Dockerfile @@ -86,7 +86,7 @@ RUN sudo apt-get install -y --no-install-recommends \ ## FROM prepare as prebuild USER gitpod -#COPY --chown=gitpod:gitpod . . +COPY --chown=gitpod:gitpod . . RUN ./bootstrap RUN ./configure --prefix=/home/gitpod/sage-prebuild ### Hide output since otherwise we would reach log limit From 8f578efbc097517b66c442750647fbaf453781c1 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Fri, 10 Dec 2021 00:23:14 +0100 Subject: [PATCH 032/115] Try to ignore workspace folder --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 91838a1627c..2412ccd65b2 100644 --- a/.gitignore +++ b/.gitignore @@ -203,3 +203,6 @@ src/venv.bak/ # git worktree worktree* **/worktree* + +# Gitpod +workspace From c0ce19ed03806b6b636b3af5762d3fd4bf32247b Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Fri, 10 Dec 2021 00:33:57 +0100 Subject: [PATCH 033/115] Copy only what we need --- .gitignore | 3 --- docker/.gitpod.Dockerfile | 13 ++++++++++++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 2412ccd65b2..91838a1627c 100644 --- a/.gitignore +++ b/.gitignore @@ -203,6 +203,3 @@ src/venv.bak/ # git worktree worktree* **/worktree* - -# Gitpod -workspace diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile index 3911b32a92c..0a7cc012180 100644 --- a/docker/.gitpod.Dockerfile +++ b/docker/.gitpod.Dockerfile @@ -86,7 +86,18 @@ RUN sudo apt-get install -y --no-install-recommends \ ## FROM prepare as prebuild USER gitpod -COPY --chown=gitpod:gitpod . . +### We cannot copy everything due to https://github.com/gitpod-io/gitpod/issues/7157 +### COPY --chown=gitpod:gitpod . . +### Thus only selectively copy the files we need +COPY --chown=gitpod:gitpod ./build ./build +COPY --chown=gitpod:gitpod ./bootstrap ./bootstrap +COPY --chown=gitpod:gitpod ./configure.ac ./configure.ac +COPY --chown=gitpod:gitpod ./src/doc/bootstrap ./src/doc/bootstrap +COPY --chown=gitpod:gitpod ./src/bin ./src/bin +COPY --chown=gitpod:gitpod ./m4 ./m4 +COPY --chown=gitpod:gitpod ./pkgs ./pkgs +COPY --chown=gitpod:gitpod ./sage ./sage +COPY --chown=gitpod:gitpod ./Makefile ./Makefile RUN ./bootstrap RUN ./configure --prefix=/home/gitpod/sage-prebuild ### Hide output since otherwise we would reach log limit From a0ba1d263cc5483d4c2d3d8e7ddf6f15444780c9 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Fri, 10 Dec 2021 12:04:49 +0100 Subject: [PATCH 034/115] Only create ssh folder if it not exists --- .gitpod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitpod.yml b/.gitpod.yml index 2a617c3211b..ddd3fd31db0 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -9,7 +9,7 @@ tasks: # Setup ssh key for authentication with trac ## In order to use this, generate a new key with `ssh-keygen -f tempkey` and save the private key to gitpod `gp env PRIVATE_SSH_KEY="$( ~/.ssh/id_rsa chmod 600 ~/.ssh/id_rsa init: | From 64022527b1d740efbefc4d31ea51d04b45bbfda1 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Fri, 10 Dec 2021 12:20:50 +0100 Subject: [PATCH 035/115] Install other pari packages as wel --- docker/.gitpod.Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile index 0a7cc012180..3f9bbc1938b 100644 --- a/docker/.gitpod.Dockerfile +++ b/docker/.gitpod.Dockerfile @@ -102,7 +102,10 @@ RUN ./bootstrap RUN ./configure --prefix=/home/gitpod/sage-prebuild ### Hide output since otherwise we would reach log limit ### Gitpod also puts a timeout at 1h, so we cannot install everything here with `make build-local` -RUN MAKE='make -j8' make arb ecl flint cddlib eclib fplll giac gengetopt pari > /dev/null +RUN MAKE='make -j8' make \ + arb ecl flint cddlib eclib fplll giac gengetopt \ + pari pari_elldata pari_galdata pari_galpol pari_seadata \ + > /dev/null ## ## Build final image From a382c87af8d8d89da37a3bd23a297890e1f97a5c Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Fri, 10 Dec 2021 13:59:05 +0100 Subject: [PATCH 036/115] Disable pyenv --- .gitpod.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitpod.yml b/.gitpod.yml index ddd3fd31db0..addbd990ca3 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -12,11 +12,16 @@ tasks: mkdir -p ~/.ssh echo $PRIVATE_SSH_KEY > ~/.ssh/id_rsa chmod 600 ~/.ssh/id_rsa + # Disable pyenv since it would interfer with sages build process + pyenv shell --unset init: | # Setup trac repo git remote add trac git@trac.sagemath.org:sage.git -t master git remote set-url --push trac git@trac.sagemath.org:sage.git + # Disable pyenv since it would interfer with sages build process + pyenv shell --unset + # Start build ./bootstrap ./configure -enable-editable From 3bd6512cd2b1f310d313f16eef431b2acfdf4496 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Fri, 10 Dec 2021 13:59:25 +0100 Subject: [PATCH 037/115] Add LD_lib path so that pari finds everything correctly --- docker/.gitpod.Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile index 3f9bbc1938b..5cd422f1167 100644 --- a/docker/.gitpod.Dockerfile +++ b/docker/.gitpod.Dockerfile @@ -117,6 +117,7 @@ ENV PATH=/home/gitpod/sage-prebuild/bin:$PATH ENV PKG_CONFIG_PATH=/home/gitpod/sage-prebuild/lib/pkgconfig:$PKG_CONFIG_PATH ENV CPPFLAGS="-I/home/gitpod/sage-prebuild/include $CPPFLAGS" ENV LDFLAGS="-L/home/gitpod/sage-prebuild/lib $LDFLAGS" +ENV LD_LIBRARY_PATH="/home/gitpod/sage-prebuild/lib:$LD_LIBRARY_PATH" # Configure ## Gitpod sets PIP_USER: yes by default, which leads to problems during build (e.g pip not being installed in the venv) From 0961ed31981bbc3d8dd492a660c585bf6671f661 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Fri, 10 Dec 2021 15:53:34 +0100 Subject: [PATCH 038/115] Also install simple-rst for rst support --- .gitpod.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitpod.yml b/.gitpod.yml index addbd990ca3..9fdbd00b658 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -34,6 +34,7 @@ vscode: extensions: - ms-pyright.pyright - ms-python.python + - trond-snekvik.simple-rst - lextudio.restructuredtext - streetsidesoftware.code-spell-checker - ms-toolsai.jupyter From 1ce6c771f512f9ba481318f7dec43aa20d4b2a4e Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Fri, 10 Dec 2021 16:01:30 +0100 Subject: [PATCH 039/115] Also preinstall singular --- docker/.gitpod.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile index 5cd422f1167..53b7d3fe45b 100644 --- a/docker/.gitpod.Dockerfile +++ b/docker/.gitpod.Dockerfile @@ -103,7 +103,7 @@ RUN ./configure --prefix=/home/gitpod/sage-prebuild ### Hide output since otherwise we would reach log limit ### Gitpod also puts a timeout at 1h, so we cannot install everything here with `make build-local` RUN MAKE='make -j8' make \ - arb ecl flint cddlib eclib fplll giac gengetopt \ + arb ecl flint cddlib eclib fplll giac gengetopt singular \ pari pari_elldata pari_galdata pari_galpol pari_seadata \ > /dev/null From eab7958341e493f2fe95952783412900e1b916ba Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 28 Dec 2021 11:32:02 -0800 Subject: [PATCH 040/115] docker/.gitpod.Dockerfile: Use sage-get-system-packages --- docker/.gitpod.Dockerfile | 82 +++++++-------------------------------- 1 file changed, 14 insertions(+), 68 deletions(-) diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile index 53b7d3fe45b..8ece7af4e1d 100644 --- a/docker/.gitpod.Dockerfile +++ b/docker/.gitpod.Dockerfile @@ -3,74 +3,22 @@ ## FROM gitpod/workspace-full as prepare +USER gitpod +# Only copy build, for package information needed for the system package install. +# configure.ac is needed because build/sage_bootstrap uses it to recognize SAGE_ROOT. +COPY --chown=gitpod:gitpod ./configure.ac ./configure.ac +COPY --chown=gitpod:gitpod ./build ./build + # Install system packages RUN sudo apt-get update -RUN sudo apt-get install -y --no-install-recommends \ - ## From main dockerfile and/or needed for bootstrap - wget \ - build-essential \ - automake \ - m4 \ - dpkg-dev \ - python \ - libssl-dev \ - rdfind \ - gettext \ - gfortran \ - gcc \ - g++ \ - libstdc++-9-dev \ - openssl \ - ## Recommended by ./configure - libflint-arb-dev \ - libbrial-dev libbrial-groebner-dev \ - libcdd-dev libcdd-tools \ - ecl eclib-tools \ - libec-dev \ - fflas-ffpack \ - flintqs \ - libgc-dev \ - gfan \ - libgiac-dev \ - xcas \ - libgsl-dev \ - libiml-dev \ - lcalc \ - liblfunction-dev \ - libhomfly-dev \ - libopenblas-dev \ - palp \ - # pari is not recognized even if installed - #pari-gp2c libpari-dev pari-doc pari-elldata pari-galdata pari-galpol pari-seadata - libppl-dev ppl-dev \ - python3 libpython3-dev python3-distutils python3-venv \ - r-base-dev r-cran-lattice \ - librw-dev \ - libsuitesparse-dev \ - libsymmetrica2-dev \ - sympow \ - tachyon \ - libzmq3-dev \ - libzn-poly-dev \ - libgf2x-dev \ - cliquer \ - libcliquer-dev \ - gmp-ecm \ - libecm-dev \ - glpk-utils \ - libglpk-dev \ - libbraiding-dev \ - liblrcalc-dev \ - libm4rie-dev \ - libmpc-dev \ - libmpfi-dev \ - libmpfr-dev \ - nauty \ - libntl-dev \ - libplanarity-dev \ - planarity - ## We do not install tox, since it pulls in javascript-common which does not install for some reason - #tox +RUN sudo apt-get install -y --no-install-recommends \ + $(build/bin/sage-get-system-packages debian \ + _bootstrap \ + $(PATH=build/bin:$PATH build/bin/sage-package list \ + --has-file=spkg-configure.m4 :standard: \ + | grep -E -v "pari|tox" )) + # We do not install pari, as it is not recognized even if installed + # We do not install tox, since it pulls in javascript-common which does not install for some reason ## Homebrew has some more up-to-date packages (but sage is not yet able to find them) ### RUN brew update && brew upgrade @@ -89,9 +37,7 @@ USER gitpod ### We cannot copy everything due to https://github.com/gitpod-io/gitpod/issues/7157 ### COPY --chown=gitpod:gitpod . . ### Thus only selectively copy the files we need -COPY --chown=gitpod:gitpod ./build ./build COPY --chown=gitpod:gitpod ./bootstrap ./bootstrap -COPY --chown=gitpod:gitpod ./configure.ac ./configure.ac COPY --chown=gitpod:gitpod ./src/doc/bootstrap ./src/doc/bootstrap COPY --chown=gitpod:gitpod ./src/bin ./src/bin COPY --chown=gitpod:gitpod ./m4 ./m4 From ecaa007affce524ed7c7f77ee22b9992c3a938fb Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 29 Dec 2021 10:55:45 -0800 Subject: [PATCH 041/115] docker/.gitpod.Dockerfile: Filter out flint --- docker/.gitpod.Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile index 8ece7af4e1d..1fe521c0263 100644 --- a/docker/.gitpod.Dockerfile +++ b/docker/.gitpod.Dockerfile @@ -16,7 +16,9 @@ RUN sudo apt-get install -y --no-install-recommends \ _bootstrap \ $(PATH=build/bin:$PATH build/bin/sage-package list \ --has-file=spkg-configure.m4 :standard: \ - | grep -E -v "pari|tox" )) + | grep -E -v "pari|tox|flint" )) + # As of 2021-12, gitpod uses ubuntu-focal. To save space, we filter out some packages that are + # too old and will be rejected by our configure script. # We do not install pari, as it is not recognized even if installed # We do not install tox, since it pulls in javascript-common which does not install for some reason From 20617a4af28236bd35a466ef3a9403eb7f341a11 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 29 Dec 2021 21:54:23 -0800 Subject: [PATCH 042/115] .gitpod.yml, .gitpod.Dockerfile: Use prefix=/workspace/sage-local --- .gitpod.yml | 2 +- docker/.gitpod.Dockerfile | 17 ++++++----------- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index 9fdbd00b658..5a6b08d7a1f 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -24,7 +24,7 @@ tasks: # Start build ./bootstrap - ./configure -enable-editable + ./configure --enable-editable --prefix=/workspace/sage-local --with-sage-venv MAKE='make -j8' make build env: SAGE_NUM_THREADS: 4 diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile index 1fe521c0263..abfac63da4a 100644 --- a/docker/.gitpod.Dockerfile +++ b/docker/.gitpod.Dockerfile @@ -47,25 +47,20 @@ COPY --chown=gitpod:gitpod ./pkgs ./pkgs COPY --chown=gitpod:gitpod ./sage ./sage COPY --chown=gitpod:gitpod ./Makefile ./Makefile RUN ./bootstrap -RUN ./configure --prefix=/home/gitpod/sage-prebuild -### Hide output since otherwise we would reach log limit +RUN sudo mkdir -p /workspace/sage-local && sudo chown gitpod:gitpod /workspace/sage-local +RUN ./configure --prefix=/workspace/sage-local --with-sage-venv +### V=0 since otherwise we would reach log limit ### Gitpod also puts a timeout at 1h, so we cannot install everything here with `make build-local` -RUN MAKE='make -j8' make \ +RUN MAKE='make -j8' make V=0 \ arb ecl flint cddlib eclib fplll giac gengetopt singular \ - pari pari_elldata pari_galdata pari_galpol pari_seadata \ - > /dev/null + pari pari_elldata pari_galdata pari_galpol pari_seadata ## ## Build final image ## FROM prepare # Reuse the prebuild packages -COPY --from=prebuild /home/gitpod/sage-prebuild /home/gitpod/sage-prebuild -ENV PATH=/home/gitpod/sage-prebuild/bin:$PATH -ENV PKG_CONFIG_PATH=/home/gitpod/sage-prebuild/lib/pkgconfig:$PKG_CONFIG_PATH -ENV CPPFLAGS="-I/home/gitpod/sage-prebuild/include $CPPFLAGS" -ENV LDFLAGS="-L/home/gitpod/sage-prebuild/lib $LDFLAGS" -ENV LD_LIBRARY_PATH="/home/gitpod/sage-prebuild/lib:$LD_LIBRARY_PATH" +COPY --from=prebuild /workspace/sage-local /workspace/sage-local # Configure ## Gitpod sets PIP_USER: yes by default, which leads to problems during build (e.g pip not being installed in the venv) From cfa47b0dd3cd498103bf67103c3c32c168773163 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 29 Dec 2021 23:30:13 -0800 Subject: [PATCH 043/115] .gitpod.yml, .gitpod.Dockerfile: Stage SAGE_LOCAL in /home/gitpod/sage-local, move into place in 'before' script --- .gitpod.yml | 1 + docker/.gitpod.Dockerfile | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index 5a6b08d7a1f..f1fe69c9d5c 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -6,6 +6,7 @@ image: tasks: - name: Setup before: | + mv /home/gitpod/sage-local /workspace/sage-local # Setup ssh key for authentication with trac ## In order to use this, generate a new key with `ssh-keygen -f tempkey` and save the private key to gitpod `gp env PRIVATE_SSH_KEY="$( Date: Thu, 30 Dec 2021 08:42:20 -0800 Subject: [PATCH 044/115] .gitpod.yml: sage-local setup in 'init', not 'before' --- .gitpod.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitpod.yml b/.gitpod.yml index f1fe69c9d5c..90b21b54cf6 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -6,7 +6,6 @@ image: tasks: - name: Setup before: | - mv /home/gitpod/sage-local /workspace/sage-local # Setup ssh key for authentication with trac ## In order to use this, generate a new key with `ssh-keygen -f tempkey` and save the private key to gitpod `gp env PRIVATE_SSH_KEY="$( Date: Thu, 30 Dec 2021 08:46:48 -0800 Subject: [PATCH 045/115] .gitpod.yml: Use make V=0 --- .gitpod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitpod.yml b/.gitpod.yml index 90b21b54cf6..2cc4499f43f 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -27,7 +27,7 @@ tasks: # Start build ./bootstrap ./configure --enable-editable --prefix=/workspace/sage-local --with-sage-venv - MAKE='make -j8' make build + MAKE='make -j8' make build V=0 env: SAGE_NUM_THREADS: 4 From 7ba5aadf7bdf31efab0cadc6b9d44dd771e6dbbe Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 30 Dec 2021 08:47:30 -0800 Subject: [PATCH 046/115] docker/.gitpod.Dockerfile: Set ownership of sage-local --- docker/.gitpod.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile index 003267a9126..05a1c9ce61b 100644 --- a/docker/.gitpod.Dockerfile +++ b/docker/.gitpod.Dockerfile @@ -60,7 +60,7 @@ RUN MAKE='make -j8' make V=0 \ ## FROM prepare # Reuse the prebuild packages -COPY --from=prebuild /home/gitpod/sage-local /home/gitpod/sage-local +COPY --chown=gitpod:gitpod --from=prebuild /home/gitpod/sage-local /home/gitpod/sage-local # Configure ## Gitpod sets PIP_USER: yes by default, which leads to problems during build (e.g pip not being installed in the venv) From 74eeddbaab16de675af2642d0a4ea4434d32e80c Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Thu, 30 Dec 2021 20:02:48 +0000 Subject: [PATCH 047/115] Use timeout to make sure docker build and prebuild don't fail --- .gitpod.yml | 8 +++++--- docker/.gitpod.Dockerfile | 11 +++++------ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index 2cc4499f43f..f46db238979 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -12,22 +12,24 @@ tasks: mkdir -p ~/.ssh echo $PRIVATE_SSH_KEY > ~/.ssh/id_rsa chmod 600 ~/.ssh/id_rsa - # Disable pyenv since it would interfer with sages build process + # Disable pyenv since it would interfere with sages build process pyenv shell --unset init: | + # Reuse non-Python packages compiled during the docker build mv /home/gitpod/sage-local /workspace/sage-local # Setup trac repo git remote add trac git@trac.sagemath.org:sage.git -t master git remote set-url --push trac git@trac.sagemath.org:sage.git - # Disable pyenv since it would interfer with sages build process + # Disable pyenv since it would interfere with sages build process pyenv shell --unset # Start build ./bootstrap ./configure --enable-editable --prefix=/workspace/sage-local --with-sage-venv - MAKE='make -j8' make build V=0 + ## Gitpod has a timeout of 1h, so make sure we are below this to ensure that the prebuild is always successful + MAKE='make -j8' timeout 55m make build V=0 || true env: SAGE_NUM_THREADS: 4 diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile index 05a1c9ce61b..a987151d0b5 100644 --- a/docker/.gitpod.Dockerfile +++ b/docker/.gitpod.Dockerfile @@ -17,9 +17,8 @@ RUN sudo apt-get install -y --no-install-recommends \ $(PATH=build/bin:$PATH build/bin/sage-package list \ --has-file=spkg-configure.m4 :standard: \ | grep -E -v "pari|tox|flint" )) - # As of 2021-12, gitpod uses ubuntu-focal. To save space, we filter out some packages that are + # As of 2021-12, gitpod uses ubuntu-focal. To save space, we filter out some packages (pari, flint) that are # too old and will be rejected by our configure script. - # We do not install pari, as it is not recognized even if installed # We do not install tox, since it pulls in javascript-common which does not install for some reason ## Homebrew has some more up-to-date packages (but sage is not yet able to find them) @@ -50,10 +49,10 @@ RUN ./bootstrap RUN mkdir -p sage-local && sudo ln -s /home/gitpod/sage-local /workspace/sage-local RUN ./configure --prefix=/workspace/sage-local --with-sage-venv ### V=0 since otherwise we would reach log limit -### Gitpod also puts a timeout at 1h, so we cannot install everything here with `make build-local` -RUN MAKE='make -j8' make V=0 \ - arb ecl flint cddlib eclib fplll giac gengetopt singular \ - pari pari_elldata pari_galdata pari_galpol pari_seadata +### Gitpod also puts a timeout at 1h +### So we use the construction timeout ... || true +### to make sure we are below this limit and ensure that the docker build doesn't fail due to hitting this limit +RUN MAKE='make -j8' timeout 45m make build-local V=0 || true ## ## Build final image From d5b96111f30954d9086a6f9b5b81f05e0c64df0f Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Thu, 30 Dec 2021 20:30:26 +0000 Subject: [PATCH 048/115] Fix ssh key format --- .gitpod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitpod.yml b/.gitpod.yml index f46db238979..49df5869be4 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -10,7 +10,7 @@ tasks: ## In order to use this, generate a new key with `ssh-keygen -f tempkey` and save the private key to gitpod `gp env PRIVATE_SSH_KEY="$( ~/.ssh/id_rsa + echo $PRIVATE_SSH_KEY | sed 's/\(-----\(BEGIN\|END\) OPENSSH PRIVATE KEY-----\)/\n\1\n/g' > ~/.ssh/id_rsa chmod 600 ~/.ssh/id_rsa # Disable pyenv since it would interfere with sages build process pyenv shell --unset From f686b2ed2df93506186f08479a1c929dd1aad6ae Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Thu, 30 Dec 2021 20:37:03 +0000 Subject: [PATCH 049/115] We also don't want new lines in our ssh key --- .gitpod.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitpod.yml b/.gitpod.yml index 49df5869be4..4f8a15c38a2 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -11,6 +11,7 @@ tasks: ## then follow https://doc.sagemath.org/html/en/developer/trac.html#linking-your-public-key-to-your-trac-account to register the public key with trac. mkdir -p ~/.ssh echo $PRIVATE_SSH_KEY | sed 's/\(-----\(BEGIN\|END\) OPENSSH PRIVATE KEY-----\)/\n\1\n/g' > ~/.ssh/id_rsa + sed -i '/^$/d' file.txt chmod 600 ~/.ssh/id_rsa # Disable pyenv since it would interfere with sages build process pyenv shell --unset From 1fa1165120b6fcf8f81aa4875eb856438de77094 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Thu, 30 Dec 2021 20:44:32 +0000 Subject: [PATCH 050/115] Use more build jobs --- .gitpod.yml | 4 ++-- docker/.gitpod.Dockerfile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index 4f8a15c38a2..ab4b0d9d757 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -11,7 +11,7 @@ tasks: ## then follow https://doc.sagemath.org/html/en/developer/trac.html#linking-your-public-key-to-your-trac-account to register the public key with trac. mkdir -p ~/.ssh echo $PRIVATE_SSH_KEY | sed 's/\(-----\(BEGIN\|END\) OPENSSH PRIVATE KEY-----\)/\n\1\n/g' > ~/.ssh/id_rsa - sed -i '/^$/d' file.txt + sed -i '/^$/d' ~/.ssh/id_rsa chmod 600 ~/.ssh/id_rsa # Disable pyenv since it would interfere with sages build process pyenv shell --unset @@ -30,7 +30,7 @@ tasks: ./bootstrap ./configure --enable-editable --prefix=/workspace/sage-local --with-sage-venv ## Gitpod has a timeout of 1h, so make sure we are below this to ensure that the prebuild is always successful - MAKE='make -j8' timeout 55m make build V=0 || true + MAKE='make -j16' timeout 55m make build V=0 || true env: SAGE_NUM_THREADS: 4 diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile index a987151d0b5..a88036e7038 100644 --- a/docker/.gitpod.Dockerfile +++ b/docker/.gitpod.Dockerfile @@ -52,7 +52,7 @@ RUN ./configure --prefix=/workspace/sage-local --with-sage-venv ### Gitpod also puts a timeout at 1h ### So we use the construction timeout ... || true ### to make sure we are below this limit and ensure that the docker build doesn't fail due to hitting this limit -RUN MAKE='make -j8' timeout 45m make build-local V=0 || true +RUN MAKE='make -j16' timeout 45m make build-local V=0 || true ## ## Build final image From f07f04a8d247386811518185d738a06b6bd8dec7 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Fri, 31 Dec 2021 11:32:04 +0000 Subject: [PATCH 051/115] Use minimal gitpod docker image as base --- docker/.gitpod.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile index a88036e7038..8535c173756 100644 --- a/docker/.gitpod.Dockerfile +++ b/docker/.gitpod.Dockerfile @@ -1,7 +1,7 @@ ## ## Install system packages ## -FROM gitpod/workspace-full as prepare +FROM gitpod/workspace-base as prepare USER gitpod # Only copy build, for package information needed for the system package install. From 052d8fed36f6f81083da8f7503050df798b2ada3 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Fri, 31 Dec 2021 11:34:28 +0000 Subject: [PATCH 052/115] Remove PRIVATE_SSH_KEY after we made use of it for security reasons --- .gitpod.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitpod.yml b/.gitpod.yml index ab4b0d9d757..d48118c60c0 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -13,6 +13,7 @@ tasks: echo $PRIVATE_SSH_KEY | sed 's/\(-----\(BEGIN\|END\) OPENSSH PRIVATE KEY-----\)/\n\1\n/g' > ~/.ssh/id_rsa sed -i '/^$/d' ~/.ssh/id_rsa chmod 600 ~/.ssh/id_rsa + unset PRIVATE_SSH_KEY # Disable pyenv since it would interfere with sages build process pyenv shell --unset init: | From 4b760c8ff6f944f7854f60ecd8a33f732154233a Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Fri, 31 Dec 2021 11:35:10 +0000 Subject: [PATCH 053/115] Sligtly reduce prebuild timeout --- .gitpod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitpod.yml b/.gitpod.yml index d48118c60c0..794333011ed 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -31,7 +31,7 @@ tasks: ./bootstrap ./configure --enable-editable --prefix=/workspace/sage-local --with-sage-venv ## Gitpod has a timeout of 1h, so make sure we are below this to ensure that the prebuild is always successful - MAKE='make -j16' timeout 55m make build V=0 || true + MAKE='make -j16' timeout 52m make build V=0 || true env: SAGE_NUM_THREADS: 4 From d9babe8d5d5a487784ff6bd5defd7f2233d85f76 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Fri, 31 Dec 2021 12:13:46 +0000 Subject: [PATCH 054/115] Create workspace folder --- docker/.gitpod.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile index 8535c173756..fe9eb4c196d 100644 --- a/docker/.gitpod.Dockerfile +++ b/docker/.gitpod.Dockerfile @@ -46,7 +46,7 @@ COPY --chown=gitpod:gitpod ./pkgs ./pkgs COPY --chown=gitpod:gitpod ./sage ./sage COPY --chown=gitpod:gitpod ./Makefile ./Makefile RUN ./bootstrap -RUN mkdir -p sage-local && sudo ln -s /home/gitpod/sage-local /workspace/sage-local +RUN mkdir -p sage-local && mkdir -p /workspace && sudo ln -s /home/gitpod/sage-local /workspace/sage-local RUN ./configure --prefix=/workspace/sage-local --with-sage-venv ### V=0 since otherwise we would reach log limit ### Gitpod also puts a timeout at 1h From 312a3643f827573e35619b2a66845fe24de1417f Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Fri, 31 Dec 2021 12:22:58 +0000 Subject: [PATCH 055/115] Add trac to known hosts --- .gitpod.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitpod.yml b/.gitpod.yml index 794333011ed..24a1693a586 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -14,6 +14,7 @@ tasks: sed -i '/^$/d' ~/.ssh/id_rsa chmod 600 ~/.ssh/id_rsa unset PRIVATE_SSH_KEY + ssh-keyscan -H trac.sagemath.org >> ~/.ssh/known_hosts # Disable pyenv since it would interfere with sages build process pyenv shell --unset init: | From d3e4ede68b3dd53d9bfbc8daadd5b6068cedd6a5 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Fri, 31 Dec 2021 12:31:16 +0000 Subject: [PATCH 056/115] Create folder with sudo --- docker/.gitpod.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile index fe9eb4c196d..78a28e37066 100644 --- a/docker/.gitpod.Dockerfile +++ b/docker/.gitpod.Dockerfile @@ -46,7 +46,7 @@ COPY --chown=gitpod:gitpod ./pkgs ./pkgs COPY --chown=gitpod:gitpod ./sage ./sage COPY --chown=gitpod:gitpod ./Makefile ./Makefile RUN ./bootstrap -RUN mkdir -p sage-local && mkdir -p /workspace && sudo ln -s /home/gitpod/sage-local /workspace/sage-local +RUN mkdir -p sage-local && sudo mkdir -p /workspace && sudo ln -s /home/gitpod/sage-local /workspace/sage-local RUN ./configure --prefix=/workspace/sage-local --with-sage-venv ### V=0 since otherwise we would reach log limit ### Gitpod also puts a timeout at 1h From fb7f7f6d0b2b3a2e05fb5b09f946ee17df2cd889 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Fri, 31 Dec 2021 13:24:05 +0000 Subject: [PATCH 057/115] Don't set pyenv when using workspace-base --- docker/.gitpod.Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile index 78a28e37066..49011e5db7d 100644 --- a/docker/.gitpod.Dockerfile +++ b/docker/.gitpod.Dockerfile @@ -64,5 +64,5 @@ COPY --chown=gitpod:gitpod --from=prebuild /home/gitpod/sage-local /home/gitpod/ # Configure ## Gitpod sets PIP_USER: yes by default, which leads to problems during build (e.g pip not being installed in the venv) RUN unset PIP_USER -## Gitpod installs pyenv by default, and sage's pip install targets the pyenv python for some reason -RUN pyenv global system +## Gitpod installs pyenv by default (in the full workspace image), and sage's pip install targets the pyenv python for some reason +## RUN pyenv global system From 7481266f6278de218f38ceabf4c97e0a22f5a22c Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Fri, 31 Dec 2021 13:27:41 +0000 Subject: [PATCH 058/115] Slightly increase build timeout --- docker/.gitpod.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile index 49011e5db7d..f650b975640 100644 --- a/docker/.gitpod.Dockerfile +++ b/docker/.gitpod.Dockerfile @@ -52,7 +52,7 @@ RUN ./configure --prefix=/workspace/sage-local --with-sage-venv ### Gitpod also puts a timeout at 1h ### So we use the construction timeout ... || true ### to make sure we are below this limit and ensure that the docker build doesn't fail due to hitting this limit -RUN MAKE='make -j16' timeout 45m make build-local V=0 || true +RUN MAKE='make -j16' timeout 50m make build-local V=0 || true ## ## Build final image From 3ef2a76f235c4da7719399741df2ae147d14c310 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Fri, 31 Dec 2021 16:18:15 +0000 Subject: [PATCH 059/115] Uncomment pyenv stuff --- .gitpod.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index 24a1693a586..f0129007c7c 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -16,7 +16,7 @@ tasks: unset PRIVATE_SSH_KEY ssh-keyscan -H trac.sagemath.org >> ~/.ssh/known_hosts # Disable pyenv since it would interfere with sages build process - pyenv shell --unset + # pyenv shell --unset init: | # Reuse non-Python packages compiled during the docker build mv /home/gitpod/sage-local /workspace/sage-local @@ -26,7 +26,7 @@ tasks: git remote set-url --push trac git@trac.sagemath.org:sage.git # Disable pyenv since it would interfere with sages build process - pyenv shell --unset + # pyenv shell --unset # Start build ./bootstrap From fcb7f92136932d43021875a334028eaa78aa449f Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Fri, 31 Dec 2021 17:30:23 +0000 Subject: [PATCH 060/115] Try again with smaller timeout --- .gitpod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitpod.yml b/.gitpod.yml index f0129007c7c..f4327e3081d 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -32,7 +32,7 @@ tasks: ./bootstrap ./configure --enable-editable --prefix=/workspace/sage-local --with-sage-venv ## Gitpod has a timeout of 1h, so make sure we are below this to ensure that the prebuild is always successful - MAKE='make -j16' timeout 52m make build V=0 || true + MAKE='make -j16' timeout 50m make build V=0 || true env: SAGE_NUM_THREADS: 4 From e3adf517761e492663abc2d4d37d6fe07631ed8f Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Sat, 1 Jan 2022 12:37:59 +0000 Subject: [PATCH 061/115] Try with way shorter timeout --- .gitpod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitpod.yml b/.gitpod.yml index f4327e3081d..955f63af4e6 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -32,7 +32,7 @@ tasks: ./bootstrap ./configure --enable-editable --prefix=/workspace/sage-local --with-sage-venv ## Gitpod has a timeout of 1h, so make sure we are below this to ensure that the prebuild is always successful - MAKE='make -j16' timeout 50m make build V=0 || true + MAKE='make -j16' timeout 35m make build V=0 || true env: SAGE_NUM_THREADS: 4 From 93150cec8ecfb56e1777af787a8bdb48bf93262f Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sat, 1 Jan 2022 10:12:45 -0800 Subject: [PATCH 062/115] tox.ini: Add docker-gitpod-standard --- tox.ini | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tox.ini b/tox.ini index 6dd379b1920..6aacc49b6b5 100644 --- a/tox.ini +++ b/tox.ini @@ -424,6 +424,12 @@ setenv = ubuntu-bionic-nvidia-cuda: BASE_TAG=ubuntu18.04 ubuntu-xenial-nvidia-cuda: BASE_TAG=ubuntu16.04 # + # https://hub.docker.com/r/gitpod/workspace-full + # + gitpod: SYSTEM=debian + gitpod: BASE_IMAGE=gitpod/workspace-full + gitpod: CONFIG_CONFIGURE_ARGS_ROOT=--enable-build-as-root --prefix=/workspace/sage-local --with-sage-venv + # # Resulting full image:tag name # docker: FULL_BASE_IMAGE_AND_TAG={env:ARCH_IMAGE_PREFIX:}{env:BASE_IMAGE}{env:ARCH_IMAGE_SUFFIX:}:{env:ARCH_TAG_PREFIX:}{env:BASE_TAG}{env:ARCH_TAG_SUFFIX:} From cbf03d1786d24bc9ec6d437f2b820bfa687c52fd Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sat, 1 Jan 2022 10:21:06 -0800 Subject: [PATCH 063/115] .github/workflows/tox.yml: Add docker-gitpod --- .github/workflows/tox.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index 67de9c01593..34ed055f6c4 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -38,7 +38,7 @@ jobs: fail-fast: false max-parallel: 20 matrix: - tox_system_factor: [ubuntu-trusty, ubuntu-xenial, ubuntu-bionic, ubuntu-focal, ubuntu-hirsute, ubuntu-impish, ubuntu-jammy, debian-stretch, debian-buster, debian-bullseye, debian-bookworm, debian-sid, linuxmint-17, linuxmint-18, linuxmint-19, linuxmint-19.3, linuxmint-20.1, linuxmint-20.2, linuxmint-20.3, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, fedora-32, fedora-33, fedora-34, fedora-35, fedora-36, centos-7, centos-8, gentoo-python3.9, gentoo-python3.10, archlinux-latest, opensuse-15, opensuse-15.3, opensuse-tumbleweed, slackware-14.2, conda-forge, ubuntu-bionic-i386, manylinux-2_24-i686, debian-buster-i386, centos-7-i386] + tox_system_factor: [gitpod, ubuntu-trusty, ubuntu-xenial, ubuntu-bionic, ubuntu-focal, ubuntu-hirsute, ubuntu-impish, ubuntu-jammy, debian-stretch, debian-buster, debian-bullseye, debian-bookworm, debian-sid, linuxmint-17, linuxmint-18, linuxmint-19, linuxmint-19.3, linuxmint-20.1, linuxmint-20.2, linuxmint-20.3, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, fedora-32, fedora-33, fedora-34, fedora-35, fedora-36, centos-7, centos-8, gentoo-python3.9, gentoo-python3.10, archlinux-latest, opensuse-15, opensuse-15.3, opensuse-tumbleweed, slackware-14.2, conda-forge, ubuntu-bionic-i386, manylinux-2_24-i686, debian-buster-i386, centos-7-i386] tox_packages_factor: [minimal, standard] env: TOX_ENV: docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }} From 4a9edfcc73c5ccbaadb064beccd80df75f8f9b9f Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sat, 1 Jan 2022 10:51:26 -0800 Subject: [PATCH 064/115] build/bin/write-dockerfile.sh: Handle __SUDO --- build/bin/write-dockerfile.sh | 11 ++++++++--- tox.ini | 3 ++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/build/bin/write-dockerfile.sh b/build/bin/write-dockerfile.sh index bd709957ef1..9b3388c38d0 100755 --- a/build/bin/write-dockerfile.sh +++ b/build/bin/write-dockerfile.sh @@ -43,10 +43,15 @@ EOF RUN sed -i.bak $DIST_UPGRADE /etc/apt/sources.list && apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y dist-upgrade EOF fi + if [ -n "$__SUDO" ]; then + SUDO="sudo" + else + unset SUDO + fi EXISTS="2>/dev/null >/dev/null apt-cache show" - UPDATE="apt-get update &&" - INSTALL="DEBIAN_FRONTEND=noninteractive apt-get install -qqq --no-install-recommends --yes" - CLEAN="&& apt-get clean" + UPDATE="$SUDO apt-get update &&" + INSTALL="$SUDO DEBIAN_FRONTEND=noninteractive apt-get install -qqq --no-install-recommends --yes" + CLEAN="&& $SUDO apt-get clean" ;; fedora*|redhat*|centos*) cat < Date: Sat, 1 Jan 2022 10:58:09 -0800 Subject: [PATCH 065/115] build/bin/write-dockerfile.sh: Use relative workdir 'sage' --- build/bin/write-dockerfile.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/bin/write-dockerfile.sh b/build/bin/write-dockerfile.sh index 9b3388c38d0..04ffa7e501c 100755 --- a/build/bin/write-dockerfile.sh +++ b/build/bin/write-dockerfile.sh @@ -190,8 +190,8 @@ cat < Date: Sat, 1 Jan 2022 11:17:47 -0800 Subject: [PATCH 066/115] .gitpod.yml, docker/.gitpod.Dockerfile: Reverse SAGE_LOCAL symlink --- .gitpod.yml | 3 ++- docker/.gitpod.Dockerfile | 3 +-- tox.ini | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index 955f63af4e6..13a6a21c79c 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -20,6 +20,7 @@ tasks: init: | # Reuse non-Python packages compiled during the docker build mv /home/gitpod/sage-local /workspace/sage-local + ln -s /workspace/sage-local /home/gitpod/sage-local # Setup trac repo git remote add trac git@trac.sagemath.org:sage.git -t master @@ -30,7 +31,7 @@ tasks: # Start build ./bootstrap - ./configure --enable-editable --prefix=/workspace/sage-local --with-sage-venv + ./configure --enable-editable --prefix=$HOME/sage-local --with-sage-venv ## Gitpod has a timeout of 1h, so make sure we are below this to ensure that the prebuild is always successful MAKE='make -j16' timeout 35m make build V=0 || true env: diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile index f650b975640..e890e4f7160 100644 --- a/docker/.gitpod.Dockerfile +++ b/docker/.gitpod.Dockerfile @@ -46,8 +46,7 @@ COPY --chown=gitpod:gitpod ./pkgs ./pkgs COPY --chown=gitpod:gitpod ./sage ./sage COPY --chown=gitpod:gitpod ./Makefile ./Makefile RUN ./bootstrap -RUN mkdir -p sage-local && sudo mkdir -p /workspace && sudo ln -s /home/gitpod/sage-local /workspace/sage-local -RUN ./configure --prefix=/workspace/sage-local --with-sage-venv +RUN ./configure --prefix=$HOME/sage-local --with-sage-venv ### V=0 since otherwise we would reach log limit ### Gitpod also puts a timeout at 1h ### So we use the construction timeout ... || true diff --git a/tox.ini b/tox.ini index 9614d7baa36..69bae200998 100644 --- a/tox.ini +++ b/tox.ini @@ -434,7 +434,7 @@ setenv = # gitpod: SYSTEM=debian gitpod: BASE_IMAGE=gitpod/workspace-full - gitpod: CONFIG_CONFIGURE_ARGS_ROOT=--prefix=/workspace/sage-local --with-sage-venv + gitpod: CONFIG_CONFIGURE_ARGS_ROOT=--prefix=/home/gitpod/sage-local --with-sage-venv gitpod: __SUDO=--sudo # # Resulting full image:tag name From bde9f74bc7be3ca657f3d6ca8ab420709abfbeda Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sat, 1 Jan 2022 11:22:14 -0800 Subject: [PATCH 067/115] build/bin/write-dockerfile.sh: Handle $__CHOWN --- build/bin/write-dockerfile.sh | 15 ++++++++------- tox.ini | 1 + 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/build/bin/write-dockerfile.sh b/build/bin/write-dockerfile.sh index 04ffa7e501c..357386c0ff0 100755 --- a/build/bin/write-dockerfile.sh +++ b/build/bin/write-dockerfile.sh @@ -31,6 +31,7 @@ done echo "# Automatically generated by SAGE_ROOT/build/bin/write-dockerfile.sh" echo "# the :comments: separate the generated file into sections" echo "# to simplify writing scripts that customize this file" +ADD="ADD $__CHOWN" RUN=RUN case $SYSTEM in debian*|ubuntu*) @@ -192,18 +193,18 @@ FROM with-system-packages as bootstrapped #:bootstrapping: RUN mkdir -p sage WORKDIR sage -ADD Makefile VERSION.txt README.md bootstrap configure.ac sage ./ -ADD src/doc/bootstrap src/doc/bootstrap -ADD src/bin src/bin -ADD m4 ./m4 -ADD build ./build +$ADD Makefile VERSION.txt README.md bootstrap configure.ac sage ./ +$ADD src/doc/bootstrap src/doc/bootstrap +$ADD src/bin src/bin +$ADD m4 ./m4 +$ADD build ./build ARG BOOTSTRAP=./bootstrap $RUN sh -x -c "\${BOOTSTRAP}" $ENDRUN FROM bootstrapped as configured #:configuring: # config.status needs to write in pkgs/sage-conf/ -ADD pkgs pkgs +$ADD pkgs pkgs RUN mkdir -p logs/pkgs; ln -s logs/pkgs/config.log config.log ARG EXTRA_CONFIGURE_ARGS="" EOF @@ -244,7 +245,7 @@ ENV MAKE="make -j\${NUMPROC}" ARG USE_MAKEFLAGS="-k V=0" ENV SAGE_CHECK=warn ENV SAGE_CHECK_PACKAGES="!gfan,!cython,!r,!python3,!gap,!cysignals,!linbox,!git,!ppl,!cmake,!rpy2,!sage_sws2rst" -ADD src src +$ADD src src ARG TARGETS="build" $RUN make SAGE_SPKG="sage-spkg -y -o" \${USE_MAKEFLAGS} \${TARGETS} $ENDRUN diff --git a/tox.ini b/tox.ini index 69bae200998..99476fc6c6a 100644 --- a/tox.ini +++ b/tox.ini @@ -436,6 +436,7 @@ setenv = gitpod: BASE_IMAGE=gitpod/workspace-full gitpod: CONFIG_CONFIGURE_ARGS_ROOT=--prefix=/home/gitpod/sage-local --with-sage-venv gitpod: __SUDO=--sudo + gitpod: __CHOWN=--chown=gitpod:gitpod # # Resulting full image:tag name # From 7ef6a6e3333a43153b92e54dceb3b06ab0713387 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sat, 1 Jan 2022 12:27:29 -0800 Subject: [PATCH 068/115] docker/.gitpod.Dockerfile: Remove, use custom image from GH Packages instead --- .gitpod.yml | 10 +++--- docker/.gitpod.Dockerfile | 67 --------------------------------------- 2 files changed, 6 insertions(+), 71 deletions(-) delete mode 100644 docker/.gitpod.Dockerfile diff --git a/.gitpod.yml b/.gitpod.yml index 13a6a21c79c..440b8db9011 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,6 +1,5 @@ # Use custom docker image. https://www.gitpod.io/docs/config-docker -image: - file: docker/.gitpod.Dockerfile +image: ghcr.io/mkoeppe/sage/sage-docker-gitpod-standard-with-targets:9.5.beta9-3-g76b5e76ab2 # Start up tasks. https://www.gitpod.io/docs/config-start-tasks/ tasks: @@ -18,8 +17,11 @@ tasks: # Disable pyenv since it would interfere with sages build process # pyenv shell --unset init: | - # Reuse non-Python packages compiled during the docker build - mv /home/gitpod/sage-local /workspace/sage-local + # Only /workspace is preserved during build. + # If the Docker image contains a built SAGE_LOCAL, move it into the workspace. + if [ ! -d /workspace/sage-local -a -d /home/gitpod/sage-local ]; then + mv /home/gitpod/sage-local /workspace/sage-local + fi ln -s /workspace/sage-local /home/gitpod/sage-local # Setup trac repo diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile deleted file mode 100644 index e890e4f7160..00000000000 --- a/docker/.gitpod.Dockerfile +++ /dev/null @@ -1,67 +0,0 @@ -## -## Install system packages -## -FROM gitpod/workspace-base as prepare - -USER gitpod -# Only copy build, for package information needed for the system package install. -# configure.ac is needed because build/sage_bootstrap uses it to recognize SAGE_ROOT. -COPY --chown=gitpod:gitpod ./configure.ac ./configure.ac -COPY --chown=gitpod:gitpod ./build ./build - -# Install system packages -RUN sudo apt-get update -RUN sudo apt-get install -y --no-install-recommends \ - $(build/bin/sage-get-system-packages debian \ - _bootstrap \ - $(PATH=build/bin:$PATH build/bin/sage-package list \ - --has-file=spkg-configure.m4 :standard: \ - | grep -E -v "pari|tox|flint" )) - # As of 2021-12, gitpod uses ubuntu-focal. To save space, we filter out some packages (pari, flint) that are - # too old and will be rejected by our configure script. - # We do not install tox, since it pulls in javascript-common which does not install for some reason - -## Homebrew has some more up-to-date packages (but sage is not yet able to find them) -### RUN brew update && brew upgrade -### RUN brew install arb flint fplll tox -### We do not install ecl from brew, since this breaks the build of maxima -### Installing pari from brew doesn't work as gitpod gp executable is then hidden by pari/gp -### RUN brew install pari pari-elldata pari-galdata pari-galpol pari-seadata -### Give prio to brew over other system packages -### ENV PATH="/home/linuxbrew/.linuxbrew/bin:$PATH" - -## -## Prebuild non-Python packages that have no (working) system-installed package -## -FROM prepare as prebuild -USER gitpod -### We cannot copy everything due to https://github.com/gitpod-io/gitpod/issues/7157 -### COPY --chown=gitpod:gitpod . . -### Thus only selectively copy the files we need -COPY --chown=gitpod:gitpod ./bootstrap ./bootstrap -COPY --chown=gitpod:gitpod ./src/doc/bootstrap ./src/doc/bootstrap -COPY --chown=gitpod:gitpod ./src/bin ./src/bin -COPY --chown=gitpod:gitpod ./m4 ./m4 -COPY --chown=gitpod:gitpod ./pkgs ./pkgs -COPY --chown=gitpod:gitpod ./sage ./sage -COPY --chown=gitpod:gitpod ./Makefile ./Makefile -RUN ./bootstrap -RUN ./configure --prefix=$HOME/sage-local --with-sage-venv -### V=0 since otherwise we would reach log limit -### Gitpod also puts a timeout at 1h -### So we use the construction timeout ... || true -### to make sure we are below this limit and ensure that the docker build doesn't fail due to hitting this limit -RUN MAKE='make -j16' timeout 50m make build-local V=0 || true - -## -## Build final image -## -FROM prepare -# Reuse the prebuild packages -COPY --chown=gitpod:gitpod --from=prebuild /home/gitpod/sage-local /home/gitpod/sage-local - -# Configure -## Gitpod sets PIP_USER: yes by default, which leads to problems during build (e.g pip not being installed in the venv) -RUN unset PIP_USER -## Gitpod installs pyenv by default (in the full workspace image), and sage's pip install targets the pyenv python for some reason -## RUN pyenv global system From 6b41c4af670b9e8a653e8c2e4049848e4e87d8a1 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sat, 1 Jan 2022 12:58:56 -0800 Subject: [PATCH 069/115] .github/workflows/tox.yml: Also push ...-with-targets-pre --- .github/workflows/tox.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index 22d90aa0137..aafe80411e4 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -43,7 +43,7 @@ jobs: env: TOX_ENV: docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }} LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }} - DOCKER_TARGETS: configured with-targets with-targets-optional + DOCKER_TARGETS: configured with-targets-pre with-targets with-targets-optional steps: - uses: actions/checkout@v2 with: From fad8942c93702e6c098fe6b9cd2d68704b256334 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sat, 1 Jan 2022 10:58:09 -0800 Subject: [PATCH 070/115] build/bin/write-dockerfile.sh: Use relative workdir 'sage' (again) --- build/bin/write-dockerfile.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/bin/write-dockerfile.sh b/build/bin/write-dockerfile.sh index e674a3ebe23..f6954091fe7 100755 --- a/build/bin/write-dockerfile.sh +++ b/build/bin/write-dockerfile.sh @@ -191,8 +191,8 @@ cat < Date: Sat, 1 Jan 2022 21:02:36 -0800 Subject: [PATCH 071/115] .gitpod.yml: Remove pyenv workarounds --- .gitpod.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index 440b8db9011..9298187fbdb 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -14,8 +14,6 @@ tasks: chmod 600 ~/.ssh/id_rsa unset PRIVATE_SSH_KEY ssh-keyscan -H trac.sagemath.org >> ~/.ssh/known_hosts - # Disable pyenv since it would interfere with sages build process - # pyenv shell --unset init: | # Only /workspace is preserved during build. # If the Docker image contains a built SAGE_LOCAL, move it into the workspace. @@ -28,9 +26,6 @@ tasks: git remote add trac git@trac.sagemath.org:sage.git -t master git remote set-url --push trac git@trac.sagemath.org:sage.git - # Disable pyenv since it would interfere with sages build process - # pyenv shell --unset - # Start build ./bootstrap ./configure --enable-editable --prefix=$HOME/sage-local --with-sage-venv From 1258c702dac22a428cf0d1ede65ccea35cf50d0f Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sat, 1 Jan 2022 21:02:56 -0800 Subject: [PATCH 072/115] .gitpod.yml: configure --enable-download-from-upstream-url --- .gitpod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitpod.yml b/.gitpod.yml index 9298187fbdb..51b775bf2c5 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -28,7 +28,7 @@ tasks: # Start build ./bootstrap - ./configure --enable-editable --prefix=$HOME/sage-local --with-sage-venv + ./configure --enable-editable --enable-download-from-upstream-url --prefix=$HOME/sage-local --with-sage-venv ## Gitpod has a timeout of 1h, so make sure we are below this to ensure that the prebuild is always successful MAKE='make -j16' timeout 35m make build V=0 || true env: From fdb49a1d7865f8105c50470715368af9a8ede02d Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 2 Jan 2022 13:40:15 -0800 Subject: [PATCH 073/115] tox.ini, .github/workflows/tox*.yml: Also tag the Docker images with GITHUB_REF_NAME --- .github/workflows/tox-experimental.yml | 1 + .github/workflows/tox-optional.yml | 1 + .github/workflows/tox.yml | 1 + tox.ini | 20 ++++++++++++++++---- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tox-experimental.yml b/.github/workflows/tox-experimental.yml index 5e0c8f7cc31..2e1679d22d1 100644 --- a/.github/workflows/tox-experimental.yml +++ b/.github/workflows/tox-experimental.yml @@ -48,6 +48,7 @@ jobs: TOX_ENV: docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }} LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }} DOCKER_TARGETS: configured with-targets with-targets-optional + EXTRA_DOCKER_TAGS: ${{ env.GITHUB_REF_NAME }} # Test all non-dummy experimental packages, but do not test huge packages # and do not test packages that require external software TARGETS_OPTIONAL: "$( echo $(export PATH=build/bin:$PATH && sage-package list :experimental: --has-file spkg-install.in && sage-package list :experimental: --has-file spkg-install && sage-package list :experimental: --has-file requirements.txt | grep -v ^_ | grep -v database_stein_watkins\\$ | grep -v polytopes_db_4d | grep -v cplex | grep -v gurobi | grep '^[${{ matrix.targets_pattern }}]' ) )" diff --git a/.github/workflows/tox-optional.yml b/.github/workflows/tox-optional.yml index 95383e9e1de..896b6809b15 100644 --- a/.github/workflows/tox-optional.yml +++ b/.github/workflows/tox-optional.yml @@ -48,6 +48,7 @@ jobs: TOX_ENV: docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }} LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }} DOCKER_TARGETS: configured with-targets with-targets-optional + EXTRA_DOCKER_TAGS: ${{ env.GITHUB_REF_NAME }} # Test all non-dummy optional packages, but do not test huge packages # and do not test packages that require external software TARGETS_OPTIONAL: "$( echo $(export PATH=build/bin:$PATH && (sage-package list :optional: --has-file spkg-install.in && sage-package list :optional: --has-file spkg-install && sage-package list :optional: --has-file requirements.txt) | grep -v ^_ | grep -v database_stein_watkins\\$ | grep -v polytopes_db_4d | grep -v cplex | grep -v gurobi | grep '^[${{ matrix.targets_pattern }}]' ) )" diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index aafe80411e4..d028f49a872 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -44,6 +44,7 @@ jobs: TOX_ENV: docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }} LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }} DOCKER_TARGETS: configured with-targets-pre with-targets with-targets-optional + EXTRA_DOCKER_TAGS: ${{ env.GITHUB_REF_NAME }} steps: - uses: actions/checkout@v2 with: diff --git a/tox.ini b/tox.ini index 50b29e3df9a..87cda7bdf60 100644 --- a/tox.ini +++ b/tox.ini @@ -131,6 +131,7 @@ passenv = TARGETS_PRE TARGETS_OPTIONAL docker: EXTRA_DOCKER_BUILD_ARGS + docker: EXTRA_DOCKER_TAGS # Use DOCKER_BUILDKIT=1 for new version - for which unfortunately we cannot save failed builds as an image docker: DOCKER_BUILDKIT # Set for example to "with-system-packages configured with-targets-pre with-targets" @@ -605,11 +606,13 @@ commands = docker-{arm64,armhf}: docker run --rm --privileged multiarch/qemu-user-static:register --reset docker: bash -c 'if [ x"{env:DOCKER_CONFIG_FILE:}" != x ]; then mkdir -p {envdir}/.docker && ln -sf $(realpath "{env:DOCKER_CONFIG_FILE:}") {envdir}/.docker/; fi' docker: bash -c 'for docker_target in {env:DOCKER_TARGETS:with-targets}; do \ - docker: BUILD_TAG={env:DOCKER_PUSH_REPOSITORY:}sage-{envname}-$docker_target:$(git describe --dirty --always); \ + docker: BUILD_IMAGE={env:DOCKER_PUSH_REPOSITORY:}sage-{envname}-$docker_target; \ + docker: BUILD_TAG=$BUILD_IMAGE:$(git describe --dirty --always); \ + docker: TAG_ARGS=$(echo --tag $BUILD_TAG; for tag in {env:EXTRA_DOCKER_TAGS:}; do echo --tag $BUILD_IMAGE:$tag; done); \ docker: DOCKER_BUILDKIT={env:DOCKER_BUILDKIT:0} \ docker: docker build . -f {envdir}/Dockerfile \ docker: --target $docker_target \ - docker: --tag $BUILD_TAG \ + docker: $TAG_ARGS \ docker: --build-arg EXTRA_CONFIGURE_ARGS="{env:CONFIGURE_ARGS}" \ docker: --build-arg BASE_IMAGE={env:FULL_BASE_IMAGE_AND_TAG} \ docker-conda: --build-arg USE_CONDARC="{env:USE_CONDARC}" \ @@ -618,9 +621,18 @@ commands = docker: --build-arg TARGETS="{posargs:build}" \ docker: --build-arg TARGETS_OPTIONAL="{env:TARGETS_OPTIONAL:ptest}" \ docker: {env:EXTRA_DOCKER_BUILD_ARGS:}; status=$?; \ - docker: if [ $status != 0 ]; then BUILD_TAG="$BUILD_TAG-failed"; docker commit $(docker ps -l -q) $BUILD_TAG; fi; \ + docker: if [ $status != 0 ]; then \ + docker: BUILD_TAG="$BUILD_TAG-failed"; docker commit $(docker ps -l -q) $BUILD_TAG; PUSH_TAGS=$BUILD_TAG; \ + docker: else \ + docker: PUSH_TAGS=$(echo $BUILD_TAG; for tag in {env:EXTRA_DOCKER_TAGS:}; do echo "$BUILD_IMAGE:$tag"; done); \ + docker: fi; \ docker: echo $BUILD_TAG >> {envdir}/Dockertags; \ - docker: if [ x"{env:DOCKER_PUSH_REPOSITORY:}" != x ]; then echo Pushing $BUILD_TAG; docker push $BUILD_TAG || echo "(ignoring errors)"; fi; \ + docker: if [ x"{env:DOCKER_PUSH_REPOSITORY:}" != x ]; then \ + docker: echo Pushing $PUSH_TAGS; \ + docker: for tag in $PUSH_TAGS; do \ + docker: docker push $tag || echo "(ignoring errors)"; \ + docker: done; \ + docker: fi; \ docker: if [ $status != 0 ]; then exit $status; fi; \ docker: done' # #28728: gap fails its test suite. From ab757ba44a661a0c358764d3aa40c49709f4c6cd Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 2 Jan 2022 14:29:25 -0800 Subject: [PATCH 074/115] .github/workflows/tox.yml: Fix passing of EXTRA_DOCKER_TAGS --- .github/workflows/tox-experimental.yml | 3 +-- .github/workflows/tox-optional.yml | 3 +-- .github/workflows/tox.yml | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tox-experimental.yml b/.github/workflows/tox-experimental.yml index 2e1679d22d1..2aae06bc339 100644 --- a/.github/workflows/tox-experimental.yml +++ b/.github/workflows/tox-experimental.yml @@ -48,7 +48,6 @@ jobs: TOX_ENV: docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }} LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }} DOCKER_TARGETS: configured with-targets with-targets-optional - EXTRA_DOCKER_TAGS: ${{ env.GITHUB_REF_NAME }} # Test all non-dummy experimental packages, but do not test huge packages # and do not test packages that require external software TARGETS_OPTIONAL: "$( echo $(export PATH=build/bin:$PATH && sage-package list :experimental: --has-file spkg-install.in && sage-package list :experimental: --has-file spkg-install && sage-package list :experimental: --has-file requirements.txt | grep -v ^_ | grep -v database_stein_watkins\\$ | grep -v polytopes_db_4d | grep -v cplex | grep -v gurobi | grep '^[${{ matrix.targets_pattern }}]' ) )" @@ -87,7 +86,7 @@ jobs: echo "DOCKER_CONFIG_FILE=$HOME/.docker/config.json" >> $GITHUB_ENV fi - run: | - set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;" + set -o pipefail; EXTRA_DOCKER_TAGS="$GITHUB_REF_NAME" EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;" - name: Copy logs from the docker image or build container run: | mkdir -p "artifacts/$LOGS_ARTIFACT_NAME" diff --git a/.github/workflows/tox-optional.yml b/.github/workflows/tox-optional.yml index 896b6809b15..08d9db8d1c0 100644 --- a/.github/workflows/tox-optional.yml +++ b/.github/workflows/tox-optional.yml @@ -48,7 +48,6 @@ jobs: TOX_ENV: docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }} LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }} DOCKER_TARGETS: configured with-targets with-targets-optional - EXTRA_DOCKER_TAGS: ${{ env.GITHUB_REF_NAME }} # Test all non-dummy optional packages, but do not test huge packages # and do not test packages that require external software TARGETS_OPTIONAL: "$( echo $(export PATH=build/bin:$PATH && (sage-package list :optional: --has-file spkg-install.in && sage-package list :optional: --has-file spkg-install && sage-package list :optional: --has-file requirements.txt) | grep -v ^_ | grep -v database_stein_watkins\\$ | grep -v polytopes_db_4d | grep -v cplex | grep -v gurobi | grep '^[${{ matrix.targets_pattern }}]' ) )" @@ -87,7 +86,7 @@ jobs: echo "DOCKER_CONFIG_FILE=$HOME/.docker/config.json" >> $GITHUB_ENV fi - run: | - set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;" + set -o pipefail; EXTRA_DOCKER_TAGS="$GITHUB_REF_NAME" EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;" - name: Copy logs from the docker image or build container run: | mkdir -p "artifacts/$LOGS_ARTIFACT_NAME" diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index d028f49a872..e5c35fa7d41 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -44,7 +44,6 @@ jobs: TOX_ENV: docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }} LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }} DOCKER_TARGETS: configured with-targets-pre with-targets with-targets-optional - EXTRA_DOCKER_TAGS: ${{ env.GITHUB_REF_NAME }} steps: - uses: actions/checkout@v2 with: @@ -80,7 +79,7 @@ jobs: echo "DOCKER_CONFIG_FILE=$HOME/.docker/config.json" >> $GITHUB_ENV fi - run: | - set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;" + set -o pipefail; EXTRA_DOCKER_TAGS="$GITHUB_REF_NAME" EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;" - name: Copy logs from the docker image or build container run: | mkdir -p "artifacts/$LOGS_ARTIFACT_NAME" From 96528ebee43305123a813b340ad3e027ba2f8712 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 2 Jan 2022 15:51:38 -0800 Subject: [PATCH 075/115] .github/workflows/tox*.yml: Tag docker images 'latest' or 'dev' when building a release tag --- .github/workflows/tox-experimental.yml | 11 ++++++++++- .github/workflows/tox-optional.yml | 11 ++++++++++- .github/workflows/tox.yml | 11 ++++++++++- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tox-experimental.yml b/.github/workflows/tox-experimental.yml index 2aae06bc339..98df6e645b1 100644 --- a/.github/workflows/tox-experimental.yml +++ b/.github/workflows/tox-experimental.yml @@ -85,8 +85,17 @@ jobs: echo "DOCKER_PUSH_REPOSITORY=docker.pkg.github.com/${{ github.repository }}/" >> $GITHUB_ENV echo "DOCKER_CONFIG_FILE=$HOME/.docker/config.json" >> $GITHUB_ENV fi + EXTRA_DOCKER_TAGS="$GITHUB_REF_NAME" + shopt -s extglob + case "$GITHUB_REF_NAME" in + +([0-9]).+([0-9])?(.+([0-9])) ) + EXTRA_DOCKER_TAGS="latest $EXTRA_DOCKER_TAGS";; + +([0-9]).+([0-9])?(.+([0-9])).@(beta|rc)+([0-9]) ) + EXTRA_DOCKER_TAGS="dev $EXTRA_DOCKER_TAGS";; + esac + echo "EXTRA_DOCKER_TAGS=$EXTRA_DOCKER_TAGS" >> $GITHUB_ENV - run: | - set -o pipefail; EXTRA_DOCKER_TAGS="$GITHUB_REF_NAME" EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;" + set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;" - name: Copy logs from the docker image or build container run: | mkdir -p "artifacts/$LOGS_ARTIFACT_NAME" diff --git a/.github/workflows/tox-optional.yml b/.github/workflows/tox-optional.yml index 08d9db8d1c0..dafb50bb680 100644 --- a/.github/workflows/tox-optional.yml +++ b/.github/workflows/tox-optional.yml @@ -85,8 +85,17 @@ jobs: echo "DOCKER_PUSH_REPOSITORY=docker.pkg.github.com/${{ github.repository }}/" >> $GITHUB_ENV echo "DOCKER_CONFIG_FILE=$HOME/.docker/config.json" >> $GITHUB_ENV fi + EXTRA_DOCKER_TAGS="$GITHUB_REF_NAME" + shopt -s extglob + case "$GITHUB_REF_NAME" in + +([0-9]).+([0-9])?(.+([0-9])) ) + EXTRA_DOCKER_TAGS="latest $EXTRA_DOCKER_TAGS";; + +([0-9]).+([0-9])?(.+([0-9])).@(beta|rc)+([0-9]) ) + EXTRA_DOCKER_TAGS="dev $EXTRA_DOCKER_TAGS";; + esac + echo "EXTRA_DOCKER_TAGS=$EXTRA_DOCKER_TAGS" >> $GITHUB_ENV - run: | - set -o pipefail; EXTRA_DOCKER_TAGS="$GITHUB_REF_NAME" EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;" + set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;" - name: Copy logs from the docker image or build container run: | mkdir -p "artifacts/$LOGS_ARTIFACT_NAME" diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index e5c35fa7d41..54cc2473b79 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -78,8 +78,17 @@ jobs: echo "DOCKER_PUSH_REPOSITORY=docker.pkg.github.com/${{ github.repository }}/" >> $GITHUB_ENV echo "DOCKER_CONFIG_FILE=$HOME/.docker/config.json" >> $GITHUB_ENV fi + EXTRA_DOCKER_TAGS="$GITHUB_REF_NAME" + shopt -s extglob + case "$GITHUB_REF_NAME" in + +([0-9]).+([0-9])?(.+([0-9])) ) + EXTRA_DOCKER_TAGS="latest $EXTRA_DOCKER_TAGS";; + +([0-9]).+([0-9])?(.+([0-9])).@(beta|rc)+([0-9]) ) + EXTRA_DOCKER_TAGS="dev $EXTRA_DOCKER_TAGS";; + esac + echo "EXTRA_DOCKER_TAGS=$EXTRA_DOCKER_TAGS" >> $GITHUB_ENV - run: | - set -o pipefail; EXTRA_DOCKER_TAGS="$GITHUB_REF_NAME" EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;" + set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;" - name: Copy logs from the docker image or build container run: | mkdir -p "artifacts/$LOGS_ARTIFACT_NAME" From 5d1dc0fa92a7192bee0bb56e9ea13944da97007f Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 2 Jan 2022 18:44:34 -0800 Subject: [PATCH 076/115] .gitpod.yml: Use tag :dev --- .gitpod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitpod.yml b/.gitpod.yml index 51b775bf2c5..09c8b752b15 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,5 +1,5 @@ # Use custom docker image. https://www.gitpod.io/docs/config-docker -image: ghcr.io/mkoeppe/sage/sage-docker-gitpod-standard-with-targets:9.5.beta9-3-g76b5e76ab2 +image: ghcr.io/mkoeppe/sage/sage-docker-gitpod-standard-with-targets:dev # Start up tasks. https://www.gitpod.io/docs/config-start-tasks/ tasks: From 3e28e13afa9d3a61aceb4d6b3c3d66451f47133f Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 3 Jan 2022 10:20:22 -0800 Subject: [PATCH 077/115] .gitpod.yml: Set github prebuild options to true --- .gitpod.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.gitpod.yml b/.gitpod.yml index 09c8b752b15..92061085b68 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -43,3 +43,21 @@ vscode: - lextudio.restructuredtext - streetsidesoftware.code-spell-checker - ms-toolsai.jupyter + +# https://www.gitpod.io/docs/prebuilds#github-specific-configuration +github: + prebuilds: + # enable for the default branch (defaults to true) + master: true + # enable for all branches in this repo (defaults to false) + branches: true + # enable for pull requests coming from this repo (defaults to true) + pullRequests: true + # enable for pull requests coming from forks (defaults to false) + pullRequestsFromForks: true + # add a check to pull requests (defaults to true) + addCheck: true + # add a "Review in Gitpod" button as a comment to pull requests (defaults to false) + addComment: true + # add a "Review in Gitpod" button to the pull request's description (defaults to false) + addBadge: true From 49575f56f74343fb81b401a6077a3f05bb037c61 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 3 Jan 2022 10:49:35 -0800 Subject: [PATCH 078/115] .gitpod.yml: Switch to image ghcr.io/sagemath/sage/sage-docker-gitpod-standard-with-targets:dev --- .gitpod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitpod.yml b/.gitpod.yml index 92061085b68..5f24cbf7c9e 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,5 +1,5 @@ # Use custom docker image. https://www.gitpod.io/docs/config-docker -image: ghcr.io/mkoeppe/sage/sage-docker-gitpod-standard-with-targets:dev +image: ghcr.io/sagemath/sage/sage-docker-gitpod-standard-with-targets:dev # Start up tasks. https://www.gitpod.io/docs/config-start-tasks/ tasks: From e7d4e966d8d25281befe14c5b66bae991c4736e3 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 3 Jan 2022 11:19:31 -0800 Subject: [PATCH 079/115] .gitpod.yml: Increase timeout for make in prebuild, increase parallelism --- .gitpod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitpod.yml b/.gitpod.yml index 5f24cbf7c9e..d4725f3c9fa 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -30,7 +30,7 @@ tasks: ./bootstrap ./configure --enable-editable --enable-download-from-upstream-url --prefix=$HOME/sage-local --with-sage-venv ## Gitpod has a timeout of 1h, so make sure we are below this to ensure that the prebuild is always successful - MAKE='make -j16' timeout 35m make build V=0 || true + MAKE='make -j24' timeout 51m make build V=0 || true env: SAGE_NUM_THREADS: 4 From fdb332d2a52db9f63fc54960d029a5ad9c529a61 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 3 Jan 2022 11:43:08 -0800 Subject: [PATCH 080/115] .gitpod.yml: Run 'make' again in 'before' task if prebuild 'make' timed out --- .gitpod.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitpod.yml b/.gitpod.yml index d4725f3c9fa..abde319651a 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -14,6 +14,10 @@ tasks: chmod 600 ~/.ssh/id_rsa unset PRIVATE_SSH_KEY ssh-keyscan -H trac.sagemath.org >> ~/.ssh/known_hosts + if [ -d /workspace/sage-local ]; then + ## Prebuild init script has been run. Run "make" again in case it timed out. + MAKE='make -j24' timeout 51m make build V=0 + fi init: | # Only /workspace is preserved during build. # If the Docker image contains a built SAGE_LOCAL, move it into the workspace. @@ -30,7 +34,7 @@ tasks: ./bootstrap ./configure --enable-editable --enable-download-from-upstream-url --prefix=$HOME/sage-local --with-sage-venv ## Gitpod has a timeout of 1h, so make sure we are below this to ensure that the prebuild is always successful - MAKE='make -j24' timeout 51m make build V=0 || true + MAKE='make -j24' timeout 51m make build V=0 || echo "(ignoring error)" env: SAGE_NUM_THREADS: 4 From aaa97778417b2142d9f3cf83c29dce9d38d1853e Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 3 Jan 2022 11:53:55 -0800 Subject: [PATCH 081/115] .gitpod.yml: Use SAGE_NUM_THREADS=8 --- .gitpod.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index abde319651a..bf1612130c8 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -35,8 +35,8 @@ tasks: ./configure --enable-editable --enable-download-from-upstream-url --prefix=$HOME/sage-local --with-sage-venv ## Gitpod has a timeout of 1h, so make sure we are below this to ensure that the prebuild is always successful MAKE='make -j24' timeout 51m make build V=0 || echo "(ignoring error)" - env: - SAGE_NUM_THREADS: 4 + env: + SAGE_NUM_THREADS: 8 # Preinstalled VS Code extensions. https://www.gitpod.io/docs/vscode-extensions vscode: From 87900a7d0188247be6d0c6db7b87349759621ca3 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 3 Jan 2022 12:00:09 -0800 Subject: [PATCH 082/115] .gitpod.yml: Refer to /home/gitpod as /Users/mkoeppe --- .gitpod.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index bf1612130c8..5dc8241fa39 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -21,10 +21,10 @@ tasks: init: | # Only /workspace is preserved during build. # If the Docker image contains a built SAGE_LOCAL, move it into the workspace. - if [ ! -d /workspace/sage-local -a -d /home/gitpod/sage-local ]; then - mv /home/gitpod/sage-local /workspace/sage-local + if [ ! -d /workspace/sage-local -a -d $HOME/sage-local ]; then + mv $HOME/sage-local /workspace/sage-local fi - ln -s /workspace/sage-local /home/gitpod/sage-local + ln -s /workspace/sage-local $HOME/sage-local # Setup trac repo git remote add trac git@trac.sagemath.org:sage.git -t master From 0864c6ec3e00d24566b586472ffb88e1471a2e30 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 3 Jan 2022 12:10:46 -0800 Subject: [PATCH 083/115] .github/workflows/tox*.yml: Push stable releases also to Docker tag 'dev' --- .github/workflows/tox-experimental.yml | 2 +- .github/workflows/tox-optional.yml | 2 +- .github/workflows/tox.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tox-experimental.yml b/.github/workflows/tox-experimental.yml index 98df6e645b1..2cc00730301 100644 --- a/.github/workflows/tox-experimental.yml +++ b/.github/workflows/tox-experimental.yml @@ -89,7 +89,7 @@ jobs: shopt -s extglob case "$GITHUB_REF_NAME" in +([0-9]).+([0-9])?(.+([0-9])) ) - EXTRA_DOCKER_TAGS="latest $EXTRA_DOCKER_TAGS";; + EXTRA_DOCKER_TAGS="latest dev $EXTRA_DOCKER_TAGS";; +([0-9]).+([0-9])?(.+([0-9])).@(beta|rc)+([0-9]) ) EXTRA_DOCKER_TAGS="dev $EXTRA_DOCKER_TAGS";; esac diff --git a/.github/workflows/tox-optional.yml b/.github/workflows/tox-optional.yml index dafb50bb680..f1d1e57cfcf 100644 --- a/.github/workflows/tox-optional.yml +++ b/.github/workflows/tox-optional.yml @@ -89,7 +89,7 @@ jobs: shopt -s extglob case "$GITHUB_REF_NAME" in +([0-9]).+([0-9])?(.+([0-9])) ) - EXTRA_DOCKER_TAGS="latest $EXTRA_DOCKER_TAGS";; + EXTRA_DOCKER_TAGS="latest dev $EXTRA_DOCKER_TAGS";; +([0-9]).+([0-9])?(.+([0-9])).@(beta|rc)+([0-9]) ) EXTRA_DOCKER_TAGS="dev $EXTRA_DOCKER_TAGS";; esac diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index 54cc2473b79..4a12c2474a1 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -82,7 +82,7 @@ jobs: shopt -s extglob case "$GITHUB_REF_NAME" in +([0-9]).+([0-9])?(.+([0-9])) ) - EXTRA_DOCKER_TAGS="latest $EXTRA_DOCKER_TAGS";; + EXTRA_DOCKER_TAGS="latest dev $EXTRA_DOCKER_TAGS";; +([0-9]).+([0-9])?(.+([0-9])).@(beta|rc)+([0-9]) ) EXTRA_DOCKER_TAGS="dev $EXTRA_DOCKER_TAGS";; esac From 36e0c546caed4bd77400317118b3345ae86b8cc5 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 3 Jan 2022 14:59:08 -0800 Subject: [PATCH 084/115] .gitpod.yml (before): Set timestamps of installation records to same mtime --- .gitpod.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.gitpod.yml b/.gitpod.yml index 5dc8241fa39..74c01755992 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -16,6 +16,22 @@ tasks: ssh-keyscan -H trac.sagemath.org >> ~/.ssh/known_hosts if [ -d /workspace/sage-local ]; then ## Prebuild init script has been run. Run "make" again in case it timed out. + # First set the installation records to the same mtime so that no rebuilds due to dependencies + # among these packages are triggered. This is from .github/workflows/extract-sage-local.sh + SAGE_LOCAL=$HOME/sage-local + dummy="$SAGE_LOCAL"/var/lib/sage/installed/.dummy + if [ -f "$dummy" ]; then + touch "$dummy" + for tree in "$SAGE_LOCAL" "$SAGE_LOCAL"/var/lib/sage/venv*; do + inst="$tree"/var/lib/sage/installed + if [ -d "$inst" ]; then + # -r is --reference; the macOS version of touch does not accept the long option. + (cd "$inst" && touch -r "$dummy" .dummy *) + fi + done + fi + unset SAGE_LOCAL + # Now run make. MAKE='make -j24' timeout 51m make build V=0 fi init: | From c9fa0152e86c3631ae69cb6d7ad75ed133e93bd6 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 3 Jan 2022 16:14:20 -0800 Subject: [PATCH 085/115] .gitpod.yml: Use /workspace/sage/local as physical location of SAGE_LOCAL --- .gitpod.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index 74c01755992..8586e8aa8f5 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -14,11 +14,11 @@ tasks: chmod 600 ~/.ssh/id_rsa unset PRIVATE_SSH_KEY ssh-keyscan -H trac.sagemath.org >> ~/.ssh/known_hosts - if [ -d /workspace/sage-local ]; then + if [ -d local ]; then ## Prebuild init script has been run. Run "make" again in case it timed out. # First set the installation records to the same mtime so that no rebuilds due to dependencies # among these packages are triggered. This is from .github/workflows/extract-sage-local.sh - SAGE_LOCAL=$HOME/sage-local + SAGE_LOCAL=local dummy="$SAGE_LOCAL"/var/lib/sage/installed/.dummy if [ -f "$dummy" ]; then touch "$dummy" @@ -37,11 +37,12 @@ tasks: init: | # Only /workspace is preserved during build. # If the Docker image contains a built SAGE_LOCAL, move it into the workspace. - if [ ! -d /workspace/sage-local -a -d $HOME/sage-local ]; then - mv $HOME/sage-local /workspace/sage-local + if [ ! -d local -a -d $HOME/sage-local ]; then + mv $HOME/sage-local local + fi + if [ ! -d $HOME/sage-local ]; then + ln -sf $(pwd)/local $HOME/sage-local fi - ln -s /workspace/sage-local $HOME/sage-local - # Setup trac repo git remote add trac git@trac.sagemath.org:sage.git -t master git remote set-url --push trac git@trac.sagemath.org:sage.git From 38dafc60837e3de5455e24e6913150791b05e509 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 3 Jan 2022 17:21:09 -0800 Subject: [PATCH 086/115] .gitpod.yml (before): Refer to /workspace/sage/local by full path --- .gitpod.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index 8586e8aa8f5..ca65a679c76 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -14,11 +14,11 @@ tasks: chmod 600 ~/.ssh/id_rsa unset PRIVATE_SSH_KEY ssh-keyscan -H trac.sagemath.org >> ~/.ssh/known_hosts - if [ -d local ]; then - ## Prebuild init script has been run. Run "make" again in case it timed out. + SAGE_LOCAL=/workspace/sage/local + if [ -d "$SAGE_LOCAL" ]; then + echo "Prebuild init script has been run. Running "make" again in case the build had timed out." # First set the installation records to the same mtime so that no rebuilds due to dependencies # among these packages are triggered. This is from .github/workflows/extract-sage-local.sh - SAGE_LOCAL=local dummy="$SAGE_LOCAL"/var/lib/sage/installed/.dummy if [ -f "$dummy" ]; then touch "$dummy" From e6be067e2c7210f5f04e0903a5e45a101461da63 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 3 Jan 2022 17:29:48 -0800 Subject: [PATCH 087/115] .gitpod.yml (init): Make sure to remove the directory /home/gitpod/sage-local --- .gitpod.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index ca65a679c76..420b57c387e 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -40,9 +40,8 @@ tasks: if [ ! -d local -a -d $HOME/sage-local ]; then mv $HOME/sage-local local fi - if [ ! -d $HOME/sage-local ]; then - ln -sf $(pwd)/local $HOME/sage-local - fi + rm -Rf $HOME/sage-local + ln -sf $(pwd)/local $HOME/sage-local # Setup trac repo git remote add trac git@trac.sagemath.org:sage.git -t master git remote set-url --push trac git@trac.sagemath.org:sage.git From cdcf2acb4e53644963484644e2c0e244f28d36b1 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 3 Jan 2022 22:46:23 -0800 Subject: [PATCH 088/115] .gitpod.yml (before): establish SAGE_LOCAL here, not in 'init' --- .gitpod.yml | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index 420b57c387e..e3e4fd7514c 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -17,6 +17,11 @@ tasks: SAGE_LOCAL=/workspace/sage/local if [ -d "$SAGE_LOCAL" ]; then echo "Prebuild init script has been run. Running "make" again in case the build had timed out." + # The init script has already populated the SAGE_LOCAL, + # but only /workspace is preserved; and the $HOME/sage-local may contain a resurrected + # copy of sage-local. Replace it again by a symlink. + rm -Rf $HOME/sage-local + ln -sf $(pwd)/local $HOME/sage-local # First set the installation records to the same mtime so that no rebuilds due to dependencies # among these packages are triggered. This is from .github/workflows/extract-sage-local.sh dummy="$SAGE_LOCAL"/var/lib/sage/installed/.dummy @@ -33,15 +38,17 @@ tasks: unset SAGE_LOCAL # Now run make. MAKE='make -j24' timeout 51m make build V=0 + else + # Prebuild init script has not been run + # Only /workspace is preserved during build. + # If the Docker image contains a built SAGE_LOCAL, use it to populate the SAGE_LOCAL in the workspace. + if [ -d $HOME/sage-local ]; then + mv $HOME/sage-local local + fi + rm -Rf $HOME/sage-local + ln -sf $(pwd)/local $HOME/sage-local fi init: | - # Only /workspace is preserved during build. - # If the Docker image contains a built SAGE_LOCAL, move it into the workspace. - if [ ! -d local -a -d $HOME/sage-local ]; then - mv $HOME/sage-local local - fi - rm -Rf $HOME/sage-local - ln -sf $(pwd)/local $HOME/sage-local # Setup trac repo git remote add trac git@trac.sagemath.org:sage.git -t master git remote set-url --push trac git@trac.sagemath.org:sage.git From 5b44b6b004b97ab73327a4817d0513ea2394097f Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 3 Jan 2022 23:50:27 -0800 Subject: [PATCH 089/115] .gitpod.yml: Remove install-record timestamping --- .gitpod.yml | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index e3e4fd7514c..625c776ff83 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -21,20 +21,7 @@ tasks: # but only /workspace is preserved; and the $HOME/sage-local may contain a resurrected # copy of sage-local. Replace it again by a symlink. rm -Rf $HOME/sage-local - ln -sf $(pwd)/local $HOME/sage-local - # First set the installation records to the same mtime so that no rebuilds due to dependencies - # among these packages are triggered. This is from .github/workflows/extract-sage-local.sh - dummy="$SAGE_LOCAL"/var/lib/sage/installed/.dummy - if [ -f "$dummy" ]; then - touch "$dummy" - for tree in "$SAGE_LOCAL" "$SAGE_LOCAL"/var/lib/sage/venv*; do - inst="$tree"/var/lib/sage/installed - if [ -d "$inst" ]; then - # -r is --reference; the macOS version of touch does not accept the long option. - (cd "$inst" && touch -r "$dummy" .dummy *) - fi - done - fi + ln -sf "$SAGE_LOCAL" $HOME/sage-local unset SAGE_LOCAL # Now run make. MAKE='make -j24' timeout 51m make build V=0 @@ -46,7 +33,8 @@ tasks: mv $HOME/sage-local local fi rm -Rf $HOME/sage-local - ln -sf $(pwd)/local $HOME/sage-local + ln -sf $SAGE_LOCAL $HOME/sage-local + unset SAGE_LOCAL fi init: | # Setup trac repo From f58453d1aeeb84d5508498d6178751405798657a Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 3 Jan 2022 23:58:55 -0800 Subject: [PATCH 090/115] .gitpod.yml: Also remove the source tree used by the Docker build --- .gitpod.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitpod.yml b/.gitpod.yml index 625c776ff83..84b0d859545 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -35,6 +35,12 @@ tasks: rm -Rf $HOME/sage-local ln -sf $SAGE_LOCAL $HOME/sage-local unset SAGE_LOCAL + # Also remove the source tree used by the Docker build + # but keep the logs + if [ -d $HOME/sage/logs ]; then + mv $HOME/sage/logs /workspace/sage/logs + fi + rm -Rf $HOME/sage fi init: | # Setup trac repo From aef75ebe0986f6c499585fcdad36231808e5d21b Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 4 Jan 2022 00:58:22 -0800 Subject: [PATCH 091/115] .gitpod.yml (before): Always remove the source tree used by the Docker build --- .gitpod.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index 84b0d859545..562125f4b88 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -35,13 +35,13 @@ tasks: rm -Rf $HOME/sage-local ln -sf $SAGE_LOCAL $HOME/sage-local unset SAGE_LOCAL - # Also remove the source tree used by the Docker build - # but keep the logs + # Save the logs of the source tree used by the Docker build if [ -d $HOME/sage/logs ]; then mv $HOME/sage/logs /workspace/sage/logs fi - rm -Rf $HOME/sage fi + # Remove the source tree used by the Docker build + rm -Rf $HOME/sage init: | # Setup trac repo git remote add trac git@trac.sagemath.org:sage.git -t master From 6215a9ffd51478d617d6bb3718c7af2a2261a5f3 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 4 Jan 2022 05:11:08 -0800 Subject: [PATCH 092/115] Revert ".gitpod.yml (before): Refer to /workspace/sage/local by full path" This reverts commit d41e233b71c0e960a92517917fa095355258c137. --- .gitpod.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index 562125f4b88..bd7a9e59740 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -14,15 +14,13 @@ tasks: chmod 600 ~/.ssh/id_rsa unset PRIVATE_SSH_KEY ssh-keyscan -H trac.sagemath.org >> ~/.ssh/known_hosts - SAGE_LOCAL=/workspace/sage/local - if [ -d "$SAGE_LOCAL" ]; then + if [ -d local ]; then echo "Prebuild init script has been run. Running "make" again in case the build had timed out." # The init script has already populated the SAGE_LOCAL, # but only /workspace is preserved; and the $HOME/sage-local may contain a resurrected # copy of sage-local. Replace it again by a symlink. rm -Rf $HOME/sage-local - ln -sf "$SAGE_LOCAL" $HOME/sage-local - unset SAGE_LOCAL + ln -sf $(pwd)/local $HOME/sage-local # Now run make. MAKE='make -j24' timeout 51m make build V=0 else @@ -33,11 +31,10 @@ tasks: mv $HOME/sage-local local fi rm -Rf $HOME/sage-local - ln -sf $SAGE_LOCAL $HOME/sage-local - unset SAGE_LOCAL + ln -sf $(pwd)/local $HOME/sage-local # Save the logs of the source tree used by the Docker build if [ -d $HOME/sage/logs ]; then - mv $HOME/sage/logs /workspace/sage/logs + mv $HOME/sage/logs logs fi fi # Remove the source tree used by the Docker build From cec1f44e4cacb9e32a2a15af030e80d9a3d88fad Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 4 Jan 2022 05:15:39 -0800 Subject: [PATCH 093/115] .gitpod.yml (before): No timeout for 'before' task that runs after the prebuild --- .gitpod.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index bd7a9e59740..e5f704ad70e 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -21,8 +21,8 @@ tasks: # copy of sage-local. Replace it again by a symlink. rm -Rf $HOME/sage-local ln -sf $(pwd)/local $HOME/sage-local - # Now run make. - MAKE='make -j24' timeout 51m make build V=0 + # Now run make. No timeout here. + MAKE='make -j24' make build V=0 else # Prebuild init script has not been run # Only /workspace is preserved during build. From fa237c7780e17f5e41babab91bf088908ce721a7 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 4 Jan 2022 05:19:19 -0800 Subject: [PATCH 094/115] .gitpod.yml (before): Disable pyenv --- .gitpod.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitpod.yml b/.gitpod.yml index e5f704ad70e..b2d9b18af6a 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -14,6 +14,8 @@ tasks: chmod 600 ~/.ssh/id_rsa unset PRIVATE_SSH_KEY ssh-keyscan -H trac.sagemath.org >> ~/.ssh/known_hosts + ## No need for pyenv + pyenv shell --unset if [ -d local ]; then echo "Prebuild init script has been run. Running "make" again in case the build had timed out." # The init script has already populated the SAGE_LOCAL, From 8d32070e4d5389244c563db39897a3e61347aeb3 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 4 Jan 2022 05:49:05 -0800 Subject: [PATCH 095/115] .gitpod.yml: Add comments --- .gitpod.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.gitpod.yml b/.gitpod.yml index b2d9b18af6a..1946dc310aa 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,6 +1,26 @@ # Use custom docker image. https://www.gitpod.io/docs/config-docker image: ghcr.io/sagemath/sage/sage-docker-gitpod-standard-with-targets:dev +# We use the following layout: +# +# $HOME/sage Source tree used by the Docker build; see SAGE_ROOT/tox.ini (gitpod). +# - We delete it in every invocation of the 'before' script. +# $HOME/sage/logs Logs of the Docker build. +# - In the first invocation of the 'before' script, we move it +# to /workspace/.../logs +# $HOME/sage-local The configured prefix (SAGE_LOCAL) of the Sage installation. +# - During the Docker build, this is the physical location. +# - In the first invocation of the 'before' script, we move it +# to the new physical location /workspace/.../local +# (because gitpod only preserves the contents of /workspace) +# and replace it by a symlink to the new physical location. +# - In subsequent invocations of the 'before' script, we +# remove it and replace it by a symlink to the physical +# location /workspace/.../local +# /worktree/.../local The physical location of the Sage installation, +# established in the first run of the 'before' script and +# preserved by gitpod. + # Start up tasks. https://www.gitpod.io/docs/config-start-tasks/ tasks: - name: Setup From 7b9eccfd5733d96d888b0734bef5ed29c53ef0cf Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 4 Jan 2022 05:50:49 -0800 Subject: [PATCH 096/115] .gitpod.yml: Fix inconsistent indentation --- .gitpod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitpod.yml b/.gitpod.yml index 1946dc310aa..798ad9e3056 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -50,7 +50,7 @@ tasks: # Only /workspace is preserved during build. # If the Docker image contains a built SAGE_LOCAL, use it to populate the SAGE_LOCAL in the workspace. if [ -d $HOME/sage-local ]; then - mv $HOME/sage-local local + mv $HOME/sage-local local fi rm -Rf $HOME/sage-local ln -sf $(pwd)/local $HOME/sage-local From 081de0e1219da5718d6c25a93f7e1b12089288fb Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 4 Jan 2022 05:56:19 -0800 Subject: [PATCH 097/115] .gitpod.yml: Log to logs/install.log --- .gitpod.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitpod.yml b/.gitpod.yml index 798ad9e3056..39dfa8c72ec 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -37,7 +37,7 @@ tasks: ## No need for pyenv pyenv shell --unset if [ -d local ]; then - echo "Prebuild init script has been run. Running "make" again in case the build had timed out." + mkdir -p logs && echo '### .gitpod.yml Setup.before: Prebuild init script has been run. Running "make" again in case the build had timed out.' >> logs/install.log # The init script has already populated the SAGE_LOCAL, # but only /workspace is preserved; and the $HOME/sage-local may contain a resurrected # copy of sage-local. Replace it again by a symlink. @@ -67,6 +67,7 @@ tasks: git remote set-url --push trac git@trac.sagemath.org:sage.git # Start build + mkdir -p logs && echo '### .gitpod.yml Setup.init: Starting build' >> logs/install.log ./bootstrap ./configure --enable-editable --enable-download-from-upstream-url --prefix=$HOME/sage-local --with-sage-venv ## Gitpod has a timeout of 1h, so make sure we are below this to ensure that the prebuild is always successful From 952bb4e8442ee273fb02eac5977a6203bc5fcd5e Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 4 Jan 2022 11:15:51 -0800 Subject: [PATCH 098/115] .gitpod.yml (before): Create symlink from /home/gitpod/sage to the workspace --- .gitpod.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index 39dfa8c72ec..ab5eaa77bd3 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -4,7 +4,11 @@ image: ghcr.io/sagemath/sage/sage-docker-gitpod-standard-with-targets:dev # We use the following layout: # # $HOME/sage Source tree used by the Docker build; see SAGE_ROOT/tox.ini (gitpod). -# - We delete it in every invocation of the 'before' script. +# - We delete it in every invocation of the 'before' script +# and replace it by a symlink to /workspace/... +# (This symlink is needed because the package-removal scripts +# ({prefix,venv}/var/lib/sage/scripts/*/{spkg-piprm,spkg-prerm,spkg-postrm) +# hardcode SAGE_ROOT and SAGE_SRC from package installation time) # $HOME/sage/logs Logs of the Docker build. # - In the first invocation of the 'before' script, we move it # to /workspace/.../logs @@ -59,8 +63,9 @@ tasks: mv $HOME/sage/logs logs fi fi - # Remove the source tree used by the Docker build + # Remove the source tree used by the Docker build and replace it by a symlink rm -Rf $HOME/sage + ln -s $(pwd) $HOME/sage init: | # Setup trac repo git remote add trac git@trac.sagemath.org:sage.git -t master From 9f89af7a68f9a36a59135f6f541bda204f05896f Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 4 Jan 2022 11:18:58 -0800 Subject: [PATCH 099/115] build/bin/sage-spkg: Run sage-spkg-uninstall on the correct tree --- build/bin/sage-spkg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/bin/sage-spkg b/build/bin/sage-spkg index 5d9a763a13e..9eb47b5310d 100755 --- a/build/bin/sage-spkg +++ b/build/bin/sage-spkg @@ -628,7 +628,7 @@ export SAGE_DESTDIR_LOCAL="${SAGE_DESTDIR}${SAGE_INST_LOCAL}" # First uninstall the previous version of this package, if any if [ "$KEEP_EXISTING" != "yes" ]; then - sage-spkg-uninstall "$PKG_BASE" + sage-spkg-uninstall "$PKG_BASE" "$SAGE_INST_LOCAL" fi # To work around #26996: Create lib and set a symlink so that writes into lib64/ end up in lib/ From 15e112a8b86cdde1a21224edc414e4779488b85d Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 4 Jan 2022 19:41:53 -0800 Subject: [PATCH 100/115] .vscode/settings.json: Update interpreter settings --- .vscode/settings.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 58bb32a4b39..58ed2752dbe 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,7 +1,6 @@ { - "python.venvPath": "./venv/bin/python", "python.defaultInterpreterPath": "./venv/bin/python", - "python.dataScience.jupyterServerURI": "local", + "python.pythonPath": "./venv/bin/python", "files.exclude": { "**/__pycache__": true, "src/**/*.cpp": true, From 2bf20ed651e0914847d82105891f04a04cc23464 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 5 Jan 2022 08:44:05 -0800 Subject: [PATCH 101/115] .github/workflows/tox.yml (docker): Also push -with-system-packages --- .github/workflows/tox.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index 4a12c2474a1..d5a0fc9277c 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -43,7 +43,7 @@ jobs: env: TOX_ENV: docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }} LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }} - DOCKER_TARGETS: configured with-targets-pre with-targets with-targets-optional + DOCKER_TARGETS: with-system-packages configured with-targets-pre with-targets with-targets-optional steps: - uses: actions/checkout@v2 with: From c6edf67b6256c83842a366a2a684e9335745d0eb Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 5 Jan 2022 09:06:59 -0800 Subject: [PATCH 102/115] .gitpod.yml: Use a Dockerfile again --- .gitpod.yml | 9 ++++++++- docker/.gitpod.Dockerfile | 3 +++ 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 docker/.gitpod.Dockerfile diff --git a/.gitpod.yml b/.gitpod.yml index ab5eaa77bd3..80063a761d7 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,5 +1,12 @@ # Use custom docker image. https://www.gitpod.io/docs/config-docker -image: ghcr.io/sagemath/sage/sage-docker-gitpod-standard-with-targets:dev +image: +# Each of these two options works: +# - Directly use the "-with-targets" image built by .github/workflows/tox.yml +# image: ghcr.io/sagemath/sage/sage-docker-gitpod-standard-with-targets:dev +# - or go through a custom Dockerfile, which builds a smaller image +# based on the "-with-system-packages" image built by .github/workflows/tox.yml +# with the built SAGE_LOCAL from the "-with-targets" image copied in. + file: docker/.gitpod.Dockerfile # We use the following layout: # diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile new file mode 100644 index 00000000000..4750408b74f --- /dev/null +++ b/docker/.gitpod.Dockerfile @@ -0,0 +1,3 @@ +FROM ghcr.io/sagemath/sage/sage-docker-gitpod-standard-with-system-packages:dev +COPY --chown=gitpod:gitpod --from=ghcr.io/sagemath/sage/sage-docker-gitpod-standard-with-targets:dev /home/gitpod/sage/logs /home/gitpod/sage/logs +COPY --chown=gitpod:gitpod --from=ghcr.io/sagemath/sage/sage-docker-gitpod-standard-with-targets:dev /home/gitpod/sage-local /home/gitpod/sage-local From cb678a8df1b4dafe0281b6f2def371d955ff7ef7 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 5 Jan 2022 11:19:14 -0800 Subject: [PATCH 103/115] docker/.gitpod.Dockerfile: Go through ARGs BASE_GITHUB_REPOSITORY, BASE_TAG --- docker/.gitpod.Dockerfile | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile index 4750408b74f..4117e2c2155 100644 --- a/docker/.gitpod.Dockerfile +++ b/docker/.gitpod.Dockerfile @@ -1,3 +1,6 @@ -FROM ghcr.io/sagemath/sage/sage-docker-gitpod-standard-with-system-packages:dev -COPY --chown=gitpod:gitpod --from=ghcr.io/sagemath/sage/sage-docker-gitpod-standard-with-targets:dev /home/gitpod/sage/logs /home/gitpod/sage/logs -COPY --chown=gitpod:gitpod --from=ghcr.io/sagemath/sage/sage-docker-gitpod-standard-with-targets:dev /home/gitpod/sage-local /home/gitpod/sage-local +ARG BASE_GITHUB_REPOSITORY=sagemath/sage +ARG BASE_TAG=dev +FROM ghcr.io/${BASE_GITHUB_REPOSITORY}/sage-docker-gitpod-standard-with-targets:${BASE_TAG} as with-targets +FROM ghcr.io/${BASE_GITHUB_REPOSITORY}/sage-docker-gitpod-standard-with-system-packages:${BASE_TAG} +COPY --chown=gitpod:gitpod --from=with-targets /home/gitpod/sage/logs /home/gitpod/sage/logs +COPY --chown=gitpod:gitpod --from=with-targets /home/gitpod/sage-local /home/gitpod/sage-local From e01dd838a52064f37079c7ed62b7a7aa513ccbe6 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 5 Jan 2022 11:24:36 -0800 Subject: [PATCH 104/115] .gitpod.yml (before): Run 'pyenv global system' --- .gitpod.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitpod.yml b/.gitpod.yml index 80063a761d7..d037dec694f 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -47,6 +47,7 @@ tasks: ssh-keyscan -H trac.sagemath.org >> ~/.ssh/known_hosts ## No need for pyenv pyenv shell --unset + pyenv global system if [ -d local ]; then mkdir -p logs && echo '### .gitpod.yml Setup.before: Prebuild init script has been run. Running "make" again in case the build had timed out.' >> logs/install.log # The init script has already populated the SAGE_LOCAL, From 4814334dff7c990489df1441d4244a60ca7b2c59 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 6 Jan 2022 09:53:24 -0800 Subject: [PATCH 105/115] docker/.gitpod.Dockerfile: Remove doctrees/inventory --- docker/.gitpod.Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile index 4117e2c2155..83d0c20e728 100644 --- a/docker/.gitpod.Dockerfile +++ b/docker/.gitpod.Dockerfile @@ -1,6 +1,11 @@ ARG BASE_GITHUB_REPOSITORY=sagemath/sage ARG BASE_TAG=dev FROM ghcr.io/${BASE_GITHUB_REPOSITORY}/sage-docker-gitpod-standard-with-targets:${BASE_TAG} as with-targets +# Fast doc rebuilds do not work because +# "loading pickled environment... failed; source directory has changed" +# So remove the intermediate output to save space (1GB), but keep the built HTML documentation. +RUN (cd /home/gitpod/sage-local/share/doc/sage && rm -Rf doctrees inventory) + FROM ghcr.io/${BASE_GITHUB_REPOSITORY}/sage-docker-gitpod-standard-with-system-packages:${BASE_TAG} COPY --chown=gitpod:gitpod --from=with-targets /home/gitpod/sage/logs /home/gitpod/sage/logs COPY --chown=gitpod:gitpod --from=with-targets /home/gitpod/sage-local /home/gitpod/sage-local From 874dddd4ef5445ace3d8855106f36e37d7d56472 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 6 Jan 2022 11:27:42 -0800 Subject: [PATCH 106/115] tox.ini (gitpod): Use gitpod/workspace-base by default --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 87cda7bdf60..f5bc67f537d 100644 --- a/tox.ini +++ b/tox.ini @@ -436,7 +436,8 @@ setenv = # https://hub.docker.com/r/gitpod/workspace-full # gitpod: SYSTEM=debian - gitpod: BASE_IMAGE=gitpod/workspace-full + gitpod: BASE_IMAGE=gitpod/workspace-base + gitpod-full: BASE_IMAGE=gitpod/workspace-full gitpod: CONFIG_CONFIGURE_ARGS_ROOT=--prefix=/home/gitpod/sage-local --with-sage-venv gitpod: __SUDO=--sudo gitpod: __CHOWN=--chown=gitpod:gitpod From c97dc015a437f7746635e7de3577dc25c422213e Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 6 Jan 2022 15:05:03 -0800 Subject: [PATCH 107/115] build/pkgs/sagelib/spkg-install: Remove build/temp.* directories after build --- build/pkgs/sagelib/spkg-install | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/build/pkgs/sagelib/spkg-install b/build/pkgs/sagelib/spkg-install index 76a306e5762..47e56d49459 100755 --- a/build/pkgs/sagelib/spkg-install +++ b/build/pkgs/sagelib/spkg-install @@ -56,6 +56,12 @@ else time python3 -u setup.py --no-user-cfg build install || exit 1 fi +# Trac #33103: The temp.* directories are large after a full build. +# We remove them to save space; they are not needed for fast rebuilds. +for build_tree in build/pkgs/sagelib/src/build src/build; do + rm -rf $build_tree/temp.* +done + if [ "$UNAME" = "CYGWIN" ]; then sage-rebase.sh "$SAGE_LOCAL" 2>/dev/null; fi From 36050766eb9d84679210ead65650869ba0e46412 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 6 Jan 2022 22:38:35 -0800 Subject: [PATCH 108/115] build/pkgs/sagelib/spkg-install: Fix up deleting temp.* --- build/pkgs/sagelib/spkg-install | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/build/pkgs/sagelib/spkg-install b/build/pkgs/sagelib/spkg-install index 47e56d49459..c703a34a167 100755 --- a/build/pkgs/sagelib/spkg-install +++ b/build/pkgs/sagelib/spkg-install @@ -58,9 +58,7 @@ fi # Trac #33103: The temp.* directories are large after a full build. # We remove them to save space; they are not needed for fast rebuilds. -for build_tree in build/pkgs/sagelib/src/build src/build; do - rm -rf $build_tree/temp.* -done +rm -rf build/temp.* if [ "$UNAME" = "CYGWIN" ]; then sage-rebase.sh "$SAGE_LOCAL" 2>/dev/null; From d87027c8d35dbd268d12de6e6d8fa03b1afaa63d Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Fri, 7 Jan 2022 10:01:22 -0800 Subject: [PATCH 109/115] sage --package list: Add option --exclude --- build/sage_bootstrap/cmdline.py | 5 ++++- build/sage_bootstrap/expand_class.py | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/build/sage_bootstrap/cmdline.py b/build/sage_bootstrap/cmdline.py index 4408b00e9d7..8497633337b 100644 --- a/build/sage_bootstrap/cmdline.py +++ b/build/sage_bootstrap/cmdline.py @@ -207,6 +207,9 @@ def make_parser(): '--has-file', action='append', default=[], metavar='FILENAME', dest='has_files', help=('only include packages that have this file in their metadata directory' '(examples: SPKG.rst, spkg-configure.m4, distros/debian.txt)')) + parser_list.add_argument( + '--exclude', action='append', default=[], metavar='PACKAGE_NAME', + help='exclude package from list') parser_name = subparsers.add_parser( 'name', epilog=epilog_name, formatter_class=argparse.RawDescriptionHelpFormatter, @@ -321,7 +324,7 @@ def run(): if args.subcommand == 'config': app.config() elif args.subcommand == 'list': - app.list_cls(*args.package_class, has_files=args.has_files) + app.list_cls(*args.package_class, has_files=args.has_files, exclude=args.exclude) elif args.subcommand == 'name': app.name(args.tarball_filename) elif args.subcommand == 'tarball': diff --git a/build/sage_bootstrap/expand_class.py b/build/sage_bootstrap/expand_class.py index 4ee92da88ad..1d7d08c1c9c 100644 --- a/build/sage_bootstrap/expand_class.py +++ b/build/sage_bootstrap/expand_class.py @@ -24,10 +24,13 @@ class PackageClass(object): def __init__(self, *package_names_or_classes, **filters): self.names = [] filenames = filters.pop('has_files', []) + excluded = filters.pop('exclude', []) if filters: raise ValueError('filter not supported') def included_in_filter(pkg): + if pkg.name in excluded: + return False return all(pkg.has_file(filename) for filename in filenames) for package_name_or_class in package_names_or_classes: if package_name_or_class == ':all:': From c02c03c8465a2515676b25f0686c1485083bb757 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Fri, 7 Jan 2022 10:02:00 -0800 Subject: [PATCH 110/115] tox.ini (gitpod-standard): Exclude some packages --- tox.ini | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index f5bc67f537d..8c19229c529 100644 --- a/tox.ini +++ b/tox.ini @@ -433,7 +433,7 @@ setenv = ubuntu-bionic-nvidia-cuda: BASE_TAG=ubuntu18.04 ubuntu-xenial-nvidia-cuda: BASE_TAG=ubuntu16.04 # - # https://hub.docker.com/r/gitpod/workspace-full + # https://hub.docker.com/r/gitpod/ # gitpod: SYSTEM=debian gitpod: BASE_IMAGE=gitpod/workspace-base @@ -441,6 +441,13 @@ setenv = gitpod: CONFIG_CONFIGURE_ARGS_ROOT=--prefix=/home/gitpod/sage-local --with-sage-venv gitpod: __SUDO=--sudo gitpod: __CHOWN=--chown=gitpod:gitpod + # Play safe and make sure that the image build succeeds even + # if packages that do not exist on ubuntu-focal are added to debian.txt + gitpod: IGNORE_MISSING_SYSTEM_PACKAGES=yes + # As of 2022-01, gitpod/workspace-base is based on ubuntu-focal. + # To save space, we filter out some packages that are too old and + # will be rejected by our configure script. + gitpod-standard: SAGE_PACKAGE_LIST_ARGS=--has-file=spkg-configure.m4 :standard: --exclude pari --exclude eclib --exclude lcalc --exclude giac --exclude singular # # Resulting full image:tag name # From 6285cd9563c8d3a4dc3f8b97e128373a6021a668 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Fri, 7 Jan 2022 10:38:36 -0800 Subject: [PATCH 111/115] .gitpod.Dockerfile: Save space by removing apt lists --- docker/.gitpod.Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile index 83d0c20e728..34f88c510db 100644 --- a/docker/.gitpod.Dockerfile +++ b/docker/.gitpod.Dockerfile @@ -1,6 +1,7 @@ ARG BASE_GITHUB_REPOSITORY=sagemath/sage ARG BASE_TAG=dev FROM ghcr.io/${BASE_GITHUB_REPOSITORY}/sage-docker-gitpod-standard-with-targets:${BASE_TAG} as with-targets +RUN sudo rm -rf /var/lib/apt/lists/* # Fast doc rebuilds do not work because # "loading pickled environment... failed; source directory has changed" # So remove the intermediate output to save space (1GB), but keep the built HTML documentation. From 7e74607979224e0b8f7d2803fc8b68f3802a64f4 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Fri, 7 Jan 2022 10:38:54 -0800 Subject: [PATCH 112/115] .gitpod.Dockerfile: Remove built Sage documentation --- docker/.gitpod.Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile index 34f88c510db..92acfd6acf3 100644 --- a/docker/.gitpod.Dockerfile +++ b/docker/.gitpod.Dockerfile @@ -4,8 +4,8 @@ FROM ghcr.io/${BASE_GITHUB_REPOSITORY}/sage-docker-gitpod-standard-with-targets: RUN sudo rm -rf /var/lib/apt/lists/* # Fast doc rebuilds do not work because # "loading pickled environment... failed; source directory has changed" -# So remove the intermediate output to save space (1GB), but keep the built HTML documentation. -RUN (cd /home/gitpod/sage-local/share/doc/sage && rm -Rf doctrees inventory) +# Until this is fixed, we can as well remove the whole documentation, which saves a lot of space. +RUN rm -Rf /home/gitpod/sage-local/share/doc/sage FROM ghcr.io/${BASE_GITHUB_REPOSITORY}/sage-docker-gitpod-standard-with-system-packages:${BASE_TAG} COPY --chown=gitpod:gitpod --from=with-targets /home/gitpod/sage/logs /home/gitpod/sage/logs From f71c8b6c625017b0d37e956b0386111630ad6fe1 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Fri, 7 Jan 2022 12:13:56 -0800 Subject: [PATCH 113/115] docker/.gitpod.Dockerfile: Clean more --- docker/.gitpod.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/.gitpod.Dockerfile b/docker/.gitpod.Dockerfile index 92acfd6acf3..56f2370ea0d 100644 --- a/docker/.gitpod.Dockerfile +++ b/docker/.gitpod.Dockerfile @@ -1,7 +1,7 @@ ARG BASE_GITHUB_REPOSITORY=sagemath/sage ARG BASE_TAG=dev FROM ghcr.io/${BASE_GITHUB_REPOSITORY}/sage-docker-gitpod-standard-with-targets:${BASE_TAG} as with-targets -RUN sudo rm -rf /var/lib/apt/lists/* +RUN sudo rm -rf /var/cache/debconf/* /var/lib/apt/lists/* /tmp/* /var/tmp/* # Fast doc rebuilds do not work because # "loading pickled environment... failed; source directory has changed" # Until this is fixed, we can as well remove the whole documentation, which saves a lot of space. From 2d0902a1fee81d0749cc274fbec1d965cc522466 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 11 Jan 2022 12:02:58 -0800 Subject: [PATCH 114/115] src/doc/en/developer/workspace.rst: Fix markup --- src/doc/en/developer/workspace.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/doc/en/developer/workspace.rst b/src/doc/en/developer/workspace.rst index cbc2b12d87f..238f171fd82 100644 --- a/src/doc/en/developer/workspace.rst +++ b/src/doc/en/developer/workspace.rst @@ -25,10 +25,11 @@ You can now run and edit Sage's code. Contributing your changes follows the norm :ref:`Git workflow `. For this to work, you first have to authorize Gitpod with Trac: - 1. In the running Gitpod workspace, generate a new SSH key pair by `ssh-keygen -f tempkey`. - 2. Save the private key as a secure environment variable in Gitpod using `gp env PRIVATE_SSH_KEY="$(`_. 3. Register the public key with Trac following the instructions in :ref:`section-trac-ssh-key`. You can also `use your VS Code Desktop `_ to keep -your local IDE configuration while still benefiting from Gitpod’s high-spec servers and automated prebuilds. \ No newline at end of file +your local IDE configuration while still benefiting from Gitpod’s high-spec servers and automated prebuilds. From 078f247c9a5ea383178499e9b0e990777f497a6e Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 11 Jan 2022 22:25:13 -0800 Subject: [PATCH 115/115] .github/workflows/tox*.yml: Transform GITHUB_REF_NAME to a valid docker tag --- .github/workflows/tox-experimental.yml | 7 ++++++- .github/workflows/tox-optional.yml | 7 ++++++- .github/workflows/tox.yml | 7 ++++++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tox-experimental.yml b/.github/workflows/tox-experimental.yml index 2cc00730301..b5a5e325908 100644 --- a/.github/workflows/tox-experimental.yml +++ b/.github/workflows/tox-experimental.yml @@ -85,7 +85,12 @@ jobs: echo "DOCKER_PUSH_REPOSITORY=docker.pkg.github.com/${{ github.repository }}/" >> $GITHUB_ENV echo "DOCKER_CONFIG_FILE=$HOME/.docker/config.json" >> $GITHUB_ENV fi - EXTRA_DOCKER_TAGS="$GITHUB_REF_NAME" + # From the docker documentation via .ci/update-env.sh: + # "A tag name must be valid ASCII and may + # contain lowercase and uppercase letters, digits, underscores, periods and + # dashes. A tag name may not start with a period or a dash and may contain a + # maximum of 128 characters." + EXTRA_DOCKER_TAGS=`echo $GITHUB_REF_NAME | tr -d '[:space:]' | tr -c '[:alnum:]_.-' '-' | sed 's/^[-.]*//' | cut -c1-128` shopt -s extglob case "$GITHUB_REF_NAME" in +([0-9]).+([0-9])?(.+([0-9])) ) diff --git a/.github/workflows/tox-optional.yml b/.github/workflows/tox-optional.yml index f1d1e57cfcf..13e4b7fd26d 100644 --- a/.github/workflows/tox-optional.yml +++ b/.github/workflows/tox-optional.yml @@ -85,7 +85,12 @@ jobs: echo "DOCKER_PUSH_REPOSITORY=docker.pkg.github.com/${{ github.repository }}/" >> $GITHUB_ENV echo "DOCKER_CONFIG_FILE=$HOME/.docker/config.json" >> $GITHUB_ENV fi - EXTRA_DOCKER_TAGS="$GITHUB_REF_NAME" + # From the docker documentation via .ci/update-env.sh: + # "A tag name must be valid ASCII and may + # contain lowercase and uppercase letters, digits, underscores, periods and + # dashes. A tag name may not start with a period or a dash and may contain a + # maximum of 128 characters." + EXTRA_DOCKER_TAGS=`echo $GITHUB_REF_NAME | tr -d '[:space:]' | tr -c '[:alnum:]_.-' '-' | sed 's/^[-.]*//' | cut -c1-128` shopt -s extglob case "$GITHUB_REF_NAME" in +([0-9]).+([0-9])?(.+([0-9])) ) diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index d5a0fc9277c..3db38eaacba 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -78,7 +78,12 @@ jobs: echo "DOCKER_PUSH_REPOSITORY=docker.pkg.github.com/${{ github.repository }}/" >> $GITHUB_ENV echo "DOCKER_CONFIG_FILE=$HOME/.docker/config.json" >> $GITHUB_ENV fi - EXTRA_DOCKER_TAGS="$GITHUB_REF_NAME" + # From the docker documentation via .ci/update-env.sh: + # "A tag name must be valid ASCII and may + # contain lowercase and uppercase letters, digits, underscores, periods and + # dashes. A tag name may not start with a period or a dash and may contain a + # maximum of 128 characters." + EXTRA_DOCKER_TAGS=`echo $GITHUB_REF_NAME | tr -d '[:space:]' | tr -c '[:alnum:]_.-' '-' | sed 's/^[-.]*//' | cut -c1-128` shopt -s extglob case "$GITHUB_REF_NAME" in +([0-9]).+([0-9])?(.+([0-9])) )