Skip to content
This repository has been archived by the owner on Jan 30, 2023. It is now read-only.

Commit

Permalink
Merge #33103
Browse files Browse the repository at this point in the history
  • Loading branch information
mkoeppe committed Jan 30, 2022
2 parents 439907f + 078f247 commit 531cd14
Show file tree
Hide file tree
Showing 14 changed files with 281 additions and 23 deletions.
14 changes: 14 additions & 0 deletions .github/workflows/tox-experimental.yml
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,20 @@ jobs:
echo "DOCKER_PUSH_REPOSITORY=docker.pkg.github.com/${{ github.repository }}/" >> $GITHUB_ENV
echo "DOCKER_CONFIG_FILE=$HOME/.docker/config.json" >> $GITHUB_ENV
fi
# From the docker documentation via .ci/update-env.sh:
# "A tag name must be valid ASCII and may
# contain lowercase and uppercase letters, digits, underscores, periods and
# dashes. A tag name may not start with a period or a dash and may contain a
# maximum of 128 characters."
EXTRA_DOCKER_TAGS=`echo $GITHUB_REF_NAME | tr -d '[:space:]' | tr -c '[:alnum:]_.-' '-' | sed 's/^[-.]*//' | cut -c1-128`
shopt -s extglob
case "$GITHUB_REF_NAME" in
+([0-9]).+([0-9])?(.+([0-9])) )
EXTRA_DOCKER_TAGS="latest dev $EXTRA_DOCKER_TAGS";;
+([0-9]).+([0-9])?(.+([0-9])).@(beta|rc)+([0-9]) )
EXTRA_DOCKER_TAGS="dev $EXTRA_DOCKER_TAGS";;
esac
echo "EXTRA_DOCKER_TAGS=$EXTRA_DOCKER_TAGS" >> $GITHUB_ENV
- run: |
set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;"
- name: Copy logs from the docker image or build container
Expand Down
14 changes: 14 additions & 0 deletions .github/workflows/tox-optional.yml
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,20 @@ jobs:
echo "DOCKER_PUSH_REPOSITORY=docker.pkg.github.com/${{ github.repository }}/" >> $GITHUB_ENV
echo "DOCKER_CONFIG_FILE=$HOME/.docker/config.json" >> $GITHUB_ENV
fi
# From the docker documentation via .ci/update-env.sh:
# "A tag name must be valid ASCII and may
# contain lowercase and uppercase letters, digits, underscores, periods and
# dashes. A tag name may not start with a period or a dash and may contain a
# maximum of 128 characters."
EXTRA_DOCKER_TAGS=`echo $GITHUB_REF_NAME | tr -d '[:space:]' | tr -c '[:alnum:]_.-' '-' | sed 's/^[-.]*//' | cut -c1-128`
shopt -s extglob
case "$GITHUB_REF_NAME" in
+([0-9]).+([0-9])?(.+([0-9])) )
EXTRA_DOCKER_TAGS="latest dev $EXTRA_DOCKER_TAGS";;
+([0-9]).+([0-9])?(.+([0-9])).@(beta|rc)+([0-9]) )
EXTRA_DOCKER_TAGS="dev $EXTRA_DOCKER_TAGS";;
esac
echo "EXTRA_DOCKER_TAGS=$EXTRA_DOCKER_TAGS" >> $GITHUB_ENV
- run: |
set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;"
- name: Copy logs from the docker image or build container
Expand Down
18 changes: 16 additions & 2 deletions .github/workflows/tox.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,12 +38,12 @@ jobs:
fail-fast: false
max-parallel: 20
matrix:
tox_system_factor: [ubuntu-trusty, ubuntu-xenial, ubuntu-bionic, ubuntu-focal, ubuntu-hirsute, ubuntu-impish, ubuntu-jammy, debian-stretch, debian-buster, debian-bullseye, debian-bookworm, debian-sid, linuxmint-17, linuxmint-18, linuxmint-19, linuxmint-19.3, linuxmint-20.1, linuxmint-20.2, linuxmint-20.3, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, fedora-32, fedora-33, fedora-34, fedora-35, fedora-36, centos-7, centos-8, gentoo-python3.9, gentoo-python3.10, archlinux-latest, opensuse-15, opensuse-15.3, opensuse-tumbleweed, slackware-14.2, conda-forge, ubuntu-bionic-i386, manylinux-2_24-i686, debian-buster-i386, centos-7-i386]
tox_system_factor: [gitpod, ubuntu-trusty, ubuntu-xenial, ubuntu-bionic, ubuntu-focal, ubuntu-hirsute, ubuntu-impish, ubuntu-jammy, debian-stretch, debian-buster, debian-bullseye, debian-bookworm, debian-sid, linuxmint-17, linuxmint-18, linuxmint-19, linuxmint-19.3, linuxmint-20.1, linuxmint-20.2, linuxmint-20.3, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, fedora-32, fedora-33, fedora-34, fedora-35, fedora-36, centos-7, centos-8, gentoo-python3.9, gentoo-python3.10, archlinux-latest, opensuse-15, opensuse-15.3, opensuse-tumbleweed, slackware-14.2, conda-forge, ubuntu-bionic-i386, manylinux-2_24-i686, debian-buster-i386, centos-7-i386]
tox_packages_factor: [minimal, standard]
env:
TOX_ENV: docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
DOCKER_TARGETS: configured with-targets with-targets-optional
DOCKER_TARGETS: with-system-packages configured with-targets-pre with-targets with-targets-optional
steps:
- uses: actions/checkout@v2
with:
Expand Down Expand Up @@ -78,6 +78,20 @@ jobs:
echo "DOCKER_PUSH_REPOSITORY=docker.pkg.github.com/${{ github.repository }}/" >> $GITHUB_ENV
echo "DOCKER_CONFIG_FILE=$HOME/.docker/config.json" >> $GITHUB_ENV
fi
# From the docker documentation via .ci/update-env.sh:
# "A tag name must be valid ASCII and may
# contain lowercase and uppercase letters, digits, underscores, periods and
# dashes. A tag name may not start with a period or a dash and may contain a
# maximum of 128 characters."
EXTRA_DOCKER_TAGS=`echo $GITHUB_REF_NAME | tr -d '[:space:]' | tr -c '[:alnum:]_.-' '-' | sed 's/^[-.]*//' | cut -c1-128`
shopt -s extglob
case "$GITHUB_REF_NAME" in
+([0-9]).+([0-9])?(.+([0-9])) )
EXTRA_DOCKER_TAGS="latest dev $EXTRA_DOCKER_TAGS";;
+([0-9]).+([0-9])?(.+([0-9])).@(beta|rc)+([0-9]) )
EXTRA_DOCKER_TAGS="dev $EXTRA_DOCKER_TAGS";;
esac
echo "EXTRA_DOCKER_TAGS=$EXTRA_DOCKER_TAGS" >> $GITHUB_ENV
- run: |
set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;"
- name: Copy logs from the docker image or build container
Expand Down
117 changes: 117 additions & 0 deletions .gitpod.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
# Use custom docker image. https://www.gitpod.io/docs/config-docker
image:
# Each of these two options works:
# - Directly use the "-with-targets" image built by .github/workflows/tox.yml
# image: ghcr.io/sagemath/sage/sage-docker-gitpod-standard-with-targets:dev
# - or go through a custom Dockerfile, which builds a smaller image
# based on the "-with-system-packages" image built by .github/workflows/tox.yml
# with the built SAGE_LOCAL from the "-with-targets" image copied in.
file: docker/.gitpod.Dockerfile

# We use the following layout:
#
# $HOME/sage Source tree used by the Docker build; see SAGE_ROOT/tox.ini (gitpod).
# - We delete it in every invocation of the 'before' script
# and replace it by a symlink to /workspace/...
# (This symlink is needed because the package-removal scripts
# ({prefix,venv}/var/lib/sage/scripts/*/{spkg-piprm,spkg-prerm,spkg-postrm)
# hardcode SAGE_ROOT and SAGE_SRC from package installation time)
# $HOME/sage/logs Logs of the Docker build.
# - In the first invocation of the 'before' script, we move it
# to /workspace/.../logs
# $HOME/sage-local The configured prefix (SAGE_LOCAL) of the Sage installation.
# - During the Docker build, this is the physical location.
# - In the first invocation of the 'before' script, we move it
# to the new physical location /workspace/.../local
# (because gitpod only preserves the contents of /workspace)
# and replace it by a symlink to the new physical location.
# - In subsequent invocations of the 'before' script, we
# remove it and replace it by a symlink to the physical
# location /workspace/.../local
# /worktree/.../local The physical location of the Sage installation,
# established in the first run of the 'before' script and
# preserved by gitpod.

# Start up tasks. https://www.gitpod.io/docs/config-start-tasks/
tasks:
- name: Setup
before: |
# Setup ssh key for authentication with trac
## In order to use this, generate a new key with `ssh-keygen -f tempkey` and save the private key to gitpod `gp env PRIVATE_SSH_KEY="$(<tempkey)"` (or by following https://www.gitpod.io/docs/environment-variables#using-the-account-settings)
## then follow https://doc.sagemath.org/html/en/developer/trac.html#linking-your-public-key-to-your-trac-account to register the public key with trac.
mkdir -p ~/.ssh
echo $PRIVATE_SSH_KEY | sed 's/\(-----\(BEGIN\|END\) OPENSSH PRIVATE KEY-----\)/\n\1\n/g' > ~/.ssh/id_rsa
sed -i '/^$/d' ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
unset PRIVATE_SSH_KEY
ssh-keyscan -H trac.sagemath.org >> ~/.ssh/known_hosts
## No need for pyenv
pyenv shell --unset
pyenv global system
if [ -d local ]; then
mkdir -p logs && echo '### .gitpod.yml Setup.before: Prebuild init script has been run. Running "make" again in case the build had timed out.' >> logs/install.log
# The init script has already populated the SAGE_LOCAL,
# but only /workspace is preserved; and the $HOME/sage-local may contain a resurrected
# copy of sage-local. Replace it again by a symlink.
rm -Rf $HOME/sage-local
ln -sf $(pwd)/local $HOME/sage-local
# Now run make. No timeout here.
MAKE='make -j24' make build V=0
else
# Prebuild init script has not been run
# Only /workspace is preserved during build.
# If the Docker image contains a built SAGE_LOCAL, use it to populate the SAGE_LOCAL in the workspace.
if [ -d $HOME/sage-local ]; then
mv $HOME/sage-local local
fi
rm -Rf $HOME/sage-local
ln -sf $(pwd)/local $HOME/sage-local
# Save the logs of the source tree used by the Docker build
if [ -d $HOME/sage/logs ]; then
mv $HOME/sage/logs logs
fi
fi
# Remove the source tree used by the Docker build and replace it by a symlink
rm -Rf $HOME/sage
ln -s $(pwd) $HOME/sage
init: |
# Setup trac repo
git remote add trac [email protected]:sage.git -t master
git remote set-url --push trac [email protected]:sage.git
# Start build
mkdir -p logs && echo '### .gitpod.yml Setup.init: Starting build' >> logs/install.log
./bootstrap
./configure --enable-editable --enable-download-from-upstream-url --prefix=$HOME/sage-local --with-sage-venv
## Gitpod has a timeout of 1h, so make sure we are below this to ensure that the prebuild is always successful
MAKE='make -j24' timeout 51m make build V=0 || echo "(ignoring error)"
env:
SAGE_NUM_THREADS: 8

# Preinstalled VS Code extensions. https://www.gitpod.io/docs/vscode-extensions
vscode:
extensions:
- ms-pyright.pyright
- ms-python.python
- trond-snekvik.simple-rst
- lextudio.restructuredtext
- streetsidesoftware.code-spell-checker
- ms-toolsai.jupyter

# https://www.gitpod.io/docs/prebuilds#github-specific-configuration
github:
prebuilds:
# enable for the default branch (defaults to true)
master: true
# enable for all branches in this repo (defaults to false)
branches: true
# enable for pull requests coming from this repo (defaults to true)
pullRequests: true
# enable for pull requests coming from forks (defaults to false)
pullRequestsFromForks: true
# add a check to pull requests (defaults to true)
addCheck: true
# add a "Review in Gitpod" button as a comment to pull requests (defaults to false)
addComment: true
# add a "Review in Gitpod" button to the pull request's description (defaults to false)
addBadge: true
3 changes: 1 addition & 2 deletions .vscode/settings.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
{
"python.venvPath": "./venv/bin/python",
"python.defaultInterpreterPath": "./venv/bin/python",
"python.dataScience.jupyterServerURI": "local",
"python.pythonPath": "./venv/bin/python",
"files.exclude": {
"**/__pycache__": true,
"src/**/*.cpp": true,
Expand Down
2 changes: 1 addition & 1 deletion build/bin/sage-spkg
Original file line number Diff line number Diff line change
Expand Up @@ -628,7 +628,7 @@ export SAGE_DESTDIR_LOCAL="${SAGE_DESTDIR}${SAGE_INST_LOCAL}"

# First uninstall the previous version of this package, if any
if [ "$KEEP_EXISTING" != "yes" ]; then
sage-spkg-uninstall "$PKG_BASE"
sage-spkg-uninstall "$PKG_BASE" "$SAGE_INST_LOCAL"
fi

# To work around #26996: Create lib and set a symlink so that writes into lib64/ end up in lib/
Expand Down
32 changes: 19 additions & 13 deletions build/bin/write-dockerfile.sh
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ done
echo "# Automatically generated by SAGE_ROOT/build/bin/write-dockerfile.sh"
echo "# the :comments: separate the generated file into sections"
echo "# to simplify writing scripts that customize this file"
ADD="ADD $__CHOWN"
RUN=RUN
case $SYSTEM in
debian*|ubuntu*)
Expand All @@ -43,10 +44,15 @@ EOF
RUN sed -i.bak $DIST_UPGRADE /etc/apt/sources.list && apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y dist-upgrade
EOF
fi
if [ -n "$__SUDO" ]; then
SUDO="sudo"
else
unset SUDO
fi
EXISTS="2>/dev/null >/dev/null apt-cache show"
UPDATE="apt-get update &&"
INSTALL="DEBIAN_FRONTEND=noninteractive apt-get install -qqq --no-install-recommends --yes"
CLEAN="&& apt-get clean"
UPDATE="$SUDO apt-get update &&"
INSTALL="$SUDO DEBIAN_FRONTEND=noninteractive apt-get install -qqq --no-install-recommends --yes"
CLEAN="&& $SUDO apt-get clean"
;;
fedora*|redhat*|centos*)
cat <<EOF
Expand Down Expand Up @@ -185,15 +191,15 @@ cat <<EOF
FROM with-system-packages as bootstrapped
#:bootstrapping:
RUN mkdir -p /sage
WORKDIR /sage
ADD Makefile VERSION.txt COPYING.txt condarc.yml README.md bootstrap configure.ac sage .homebrew-build-env tox.ini Pipfile.m4 ./
ADD src/doc/bootstrap src/doc/bootstrap
ADD src/bin src/bin
ADD src/Pipfile.m4 src/pyproject.toml.m4 src/requirements.txt.m4 src/setup.cfg.m4 src/
ADD m4 ./m4
ADD pkgs pkgs
ADD build ./build
RUN mkdir -p sage
WORKDIR sage
$ADD Makefile VERSION.txt COPYING.txt condarc.yml README.md bootstrap configure.ac sage .homebrew-build-env tox.ini Pipfile.m4 ./
$ADD src/doc/bootstrap src/doc/bootstrap
$ADD src/bin src/bin
$ADD src/Pipfile.m4 src/pyproject.toml.m4 src/requirements.txt.m4 src/setup.cfg.m4 src/
$ADD m4 ./m4
$ADD pkgs pkgs
$ADD build ./build
ARG BOOTSTRAP=./bootstrap
$RUN sh -x -c "\${BOOTSTRAP}" $ENDRUN
Expand Down Expand Up @@ -239,7 +245,7 @@ ENV MAKE="make -j\${NUMPROC}"
ARG USE_MAKEFLAGS="-k V=0"
ENV SAGE_CHECK=warn
ENV SAGE_CHECK_PACKAGES="!gfan,!cython,!r,!python3,!gap,!cysignals,!linbox,!git,!ppl,!cmake,!rpy2,!sage_sws2rst"
ADD src src
$ADD src src
ARG TARGETS="build"
$RUN make SAGE_SPKG="sage-spkg -y -o" \${USE_MAKEFLAGS} \${TARGETS} $ENDRUN
Expand Down
4 changes: 4 additions & 0 deletions build/pkgs/sagelib/spkg-install
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,10 @@ else
time python3 -u setup.py --no-user-cfg build install || exit 1
fi

# Trac #33103: The temp.* directories are large after a full build.
# We remove them to save space; they are not needed for fast rebuilds.
rm -rf build/temp.*

if [ "$UNAME" = "CYGWIN" ]; then
sage-rebase.sh "$SAGE_LOCAL" 2>/dev/null;
fi
5 changes: 4 additions & 1 deletion build/sage_bootstrap/cmdline.py
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,9 @@ def make_parser():
'--has-file', action='append', default=[], metavar='FILENAME', dest='has_files',
help=('only include packages that have this file in their metadata directory'
'(examples: SPKG.rst, spkg-configure.m4, distros/debian.txt)'))
parser_list.add_argument(
'--exclude', action='append', default=[], metavar='PACKAGE_NAME',
help='exclude package from list')
parser_name = subparsers.add_parser(
'name', epilog=epilog_name,
formatter_class=argparse.RawDescriptionHelpFormatter,
Expand Down Expand Up @@ -321,7 +324,7 @@ def run():
if args.subcommand == 'config':
app.config()
elif args.subcommand == 'list':
app.list_cls(*args.package_class, has_files=args.has_files)
app.list_cls(*args.package_class, has_files=args.has_files, exclude=args.exclude)
elif args.subcommand == 'name':
app.name(args.tarball_filename)
elif args.subcommand == 'tarball':
Expand Down
3 changes: 3 additions & 0 deletions build/sage_bootstrap/expand_class.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,13 @@ class PackageClass(object):
def __init__(self, *package_names_or_classes, **filters):
self.names = []
filenames = filters.pop('has_files', [])
excluded = filters.pop('exclude', [])
if filters:
raise ValueError('filter not supported')

def included_in_filter(pkg):
if pkg.name in excluded:
return False
return all(pkg.has_file(filename) for filename in filenames)
for package_name_or_class in package_names_or_classes:
if package_name_or_class == ':all:':
Expand Down
12 changes: 12 additions & 0 deletions docker/.gitpod.Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
ARG BASE_GITHUB_REPOSITORY=sagemath/sage
ARG BASE_TAG=dev
FROM ghcr.io/${BASE_GITHUB_REPOSITORY}/sage-docker-gitpod-standard-with-targets:${BASE_TAG} as with-targets
RUN sudo rm -rf /var/cache/debconf/* /var/lib/apt/lists/* /tmp/* /var/tmp/*
# Fast doc rebuilds do not work because
# "loading pickled environment... failed; source directory has changed"
# Until this is fixed, we can as well remove the whole documentation, which saves a lot of space.
RUN rm -Rf /home/gitpod/sage-local/share/doc/sage

FROM ghcr.io/${BASE_GITHUB_REPOSITORY}/sage-docker-gitpod-standard-with-system-packages:${BASE_TAG}
COPY --chown=gitpod:gitpod --from=with-targets /home/gitpod/sage/logs /home/gitpod/sage/logs
COPY --chown=gitpod:gitpod --from=with-targets /home/gitpod/sage-local /home/gitpod/sage-local
9 changes: 9 additions & 0 deletions src/doc/en/developer/index.rst
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,14 @@ development!
<http://doc.sagemath.org/html/en/installation/source.html#prerequisites>`_ on your
system.

As an easy way to get started, you can run and edit Sage's code and contribute
your changes using `Gitpod <https://www.gitpod.io>`_,
a free online development environment based on VS Code.
It will launch a pre-made workspace with all dependencies and tools installed
so that you can start contributing straight away.
Start by `going to Gitpod <https://gitpod.io/#https://github.com/sagemath/sage>`_,
and read :ref:`our Gitpod guidelines <section-gitpod>` to learn more.

- **Conventions:** read our :ref:`conventions and guidelines
<section-writing-code-for-sage>` for code and documentation.

Expand Down Expand Up @@ -115,6 +123,7 @@ Writing Code for Sage
.. toctree::
:maxdepth: 3

workspace
coding_basics
reviewer_checklist

Expand Down
Loading

0 comments on commit 531cd14

Please sign in to comment.