roundup-ci #420
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # merged in python-package.yml workflow | |
| # reference docs: | |
| # https://blog.deepjyoti30.dev/tests-github-python | |
| # https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python | |
| # https://github.com/pypa/twine/blob/main/.github/workflows/main.yml | |
| name: roundup-ci | |
| on: | |
| push: | |
| # skip if github.ref is 'refs/heads/maint-1.6' | |
| # aka github.ref_name of 'maint-1.6' | |
| # see https://github.com/orgs/community/discussions/26253 | |
| # for mechanism to control matrix based on branch | |
| branches: [ "*", '!maint-1.6' ] | |
| # pull_request: | |
| # branches: [ "master" ] | |
| schedule: | |
| # monthly build/check | |
| - cron: '23 17 1 * *' | |
| workflow_dispatch: | |
| inputs: | |
| debug_enabled: | |
| type: boolean | |
| description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)' | |
| required: false | |
| default: false | |
| # GITHUB_TOKEN only has read repo context. | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| test: | |
| name: CI build test | |
| #runs-on: ubuntu-latest | |
| # use below if running on multiple OS's. | |
| runs-on: ${{ matrix.os }} | |
| if: "!contains(github.event.head_commit.message, 'no-github-ci')" | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 4 | |
| matrix: | |
| # Run in all these versions of Python | |
| python-version: | |
| # - "2.7" | |
| - "3.12" | |
| # - 3.6 run via include on ubunto 20.04 | |
| # - "3.7" | |
| - "3.8" | |
| # - "3.9" | |
| - "3.10" | |
| - "3.11" | |
| # use for multiple os or ubuntu versions | |
| #os: [ubuntu-latest, macos-latest, windows-latest] | |
| # ubuntu latest 22.04 12/2022 | |
| os: [ubuntu-latest, ubuntu-20.04] | |
| # if the ones above fail. fail the build | |
| experimental: [ false ] | |
| include: | |
| # example: if 3.13 fails the jobs still succeeds | |
| # allow-prereleases in setup-python allows alpha/beta | |
| # releases to run | |
| - python-version: 3.13 | |
| os: ubuntu-22.04 | |
| experimental: true | |
| # 3.6 not available on new 22.04 runners, so run on 20.04 ubuntu | |
| - python-version: 3.6 | |
| os: ubuntu-20.04 | |
| exclude: | |
| # skip all python versions on 20.04 except explicitly included | |
| - os: ubuntu-20.04 | |
| # run the finalizer for coveralls even if one or more | |
| # experimental matrix runs fail. | |
| # moving it above strategy produces unexpected value false | |
| # moving it below (here) produces unexpected value ''. | |
| # continue-on-error: ${{ matrix.experimental }} | |
| env: | |
| # get colorized pytest output even without a controlling tty | |
| PYTEST_ADDOPTS: "--color=yes" | |
| # OS: ${{ matrix.os }} | |
| PYTHON_VERSION: ${{ matrix.python-version }} | |
| steps: | |
| # Checkout the latest code from the repo | |
| - name: Checkout source | |
| # example directives: | |
| # disable step | |
| # if: {{ false }} | |
| # continue running if step fails | |
| # continue-on-error: true | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 | |
| # Setup version of Python to use | |
| - name: Set Up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| allow-prereleases: true | |
| cache: 'pip' | |
| - name: Install build tools - setuptools | |
| run: pip install setuptools | |
| # Display the Python version being used | |
| - name: Display Python and key module versions | |
| run: | | |
| python -c "import sys; print('python version: ', sys.version)" | |
| python -c "import sqlite3; print('sqlite version: ', sqlite3.sqlite_version)" | |
| python -c "import setuptools; print('setuptools version: ', setuptools.__version__);" | |
| - name: Update pip | |
| run: python -m pip install --upgrade pip | |
| # note pytest-env is not needed for redis password as there is | |
| # no password on CI's redis. | |
| - name: Install pytest and other packages needed for running tests | |
| run: pip install flake8 hypothesis mock pytest pytest-cov requests sphinx-tabs | |
| # https://github.com/mxschmitt/action-tmate | |
| # allow remote ssh into the CI container. I need this to debug | |
| # some xfail cases | |
| - name: Setup tmate session | |
| uses: mxschmitt/action-tmate@v3 | |
| if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} | |
| timeout-minutes: 10 | |
| with: | |
| limit-access-to-actor: true | |
| - name: run flake8 - abort for syntax error, otherwise warn only | |
| run: | | |
| # stop the build for Python syntax errors or undefined names | |
| # talgettext is a utility function ignore it. | |
| flake8 roundup --count --select=E9,F63,F7,F82 --show-source --statistics --extend-exclude talgettext.py | |
| # exit-zero treats all errors as warnings. | |
| # The GitHub editor is 127 chars wide | |
| flake8 roundup --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics | |
| # Install the databases | |
| - name: Install mysql/mariadb | |
| run: | | |
| #set -xv | |
| # mysql is pre-installed and active but this is the install command | |
| # sudo apt-get install mysql-server mysql-client | |
| # set up mysql database | |
| sudo sed -i -e '/^\[mysqld\]/,/^\[mysql/s/^#* *max_allowed_packet.*/max_allowed_packet = 500M/' /etc/mysql/mysql.conf.d/mysqld.cnf; sleep 3 | |
| #tail -n +0 /etc/mysql/my.cnf /etc/mysql/mysql.conf.d/mysqld.cnf | |
| #grep max_allowed /etc/mysql/mysql.conf.d/mysqld.cnf | |
| #ls /etc/mysql/conf.d/ /etc/mysql/mysql.conf.d/ | |
| #sleep 5 | |
| # try to improve performance speed by disabling some ACID | |
| # settings and change some layout defaults. | |
| sudo sed -i -e '$a\innodb_flush_log_at_trx_commit = 2' /etc/mysql/mysql.conf.d/mysqld.cnf | |
| sudo sed -i -e '$a\innodb_file_per_table = OFF' /etc/mysql/mysql.conf.d/mysqld.cnf | |
| sudo sed -i -e '$a\innodb_doublewrite=OFF' /etc/mysql/mysql.conf.d/mysqld.cnf | |
| sudo sed -i -e '$a\innodb_fast_shutdown=2' /etc/mysql/mysql.conf.d/mysqld.cnf | |
| sudo sed -i -e '$a\innodb_log_file_size=1048576' /etc/mysql/mysql.conf.d/mysqld.cnf | |
| sudo sed -i -e '$a\innodb_flush_method=O_DIRECT' /etc/mysql/mysql.conf.d/mysqld.cnf | |
| sudo sed -i -e '$a\innodb_log_buffer_size=3M' /etc/mysql/mysql.conf.d/mysqld.cnf | |
| sudo sed -i -e '$a\innodb_buffer_pool_size=180M' /etc/mysql/mysql.conf.d/mysqld.cnf | |
| sleep 3 | |
| sudo service mysql restart | |
| #sleep 10 | |
| #ps -ef | grep mysqld | |
| #sudo netstat -anp | grep mysqld | |
| sudo mysql -u root -proot -e 'CREATE USER "rounduptest"@"localhost" IDENTIFIED WITH mysql_native_password BY "rounduptest"; GRANT ALL on rounduptest.* TO "rounduptest"@"localhost";' | |
| - name: Install postgres | |
| run: | | |
| sudo apt-get update && sudo apt-get install postgresql | |
| # Disable fsync, full page writes for speed, | |
| # don't care about data durability when testing | |
| sudo sed -i -e '$a\fsync = off' /etc/postgresql/*/*/postgresql.conf | |
| sudo sed -i -e '$a\full_page_writes = off' /etc/postgresql/*/*/postgresql.conf | |
| sudo sed -i -e '$a\synchronous_commit = off' /etc/postgresql/*/*/postgresql.conf | |
| sudo service postgresql restart; sleep 10 | |
| # set up postgresql database | |
| sudo -u postgres psql -c "CREATE ROLE rounduptest WITH CREATEDB LOGIN PASSWORD 'rounduptest';" -U postgres | |
| sudo -u postgres psql -c "CREATE ROLE rounduptest_schema LOGIN PASSWORD 'rounduptest';" -U postgres | |
| sudo -u postgres psql -c "CREATE DATABASE rounduptest_schema;" -U postgres | |
| sudo -u postgres psql -c "GRANT CREATE ON DATABASE rounduptest_schema TO rounduptest_schema;" -U postgres | |
| - name: install redis | |
| run: | | |
| sudo apt-get install redis | |
| pip install redis | |
| - name: Install python db libraries | |
| run: | | |
| pip install mysqlclient | |
| if [[ "$PYTHON_VERSION" != "3.13" ]]; then | |
| pip install psycopg2; fi | |
| - name: Install auxiliary packages | |
| run: | | |
| sudo apt-get install swig gpgsm libgpgme-dev | |
| # pygments for markdown2 to highlight code blocks | |
| pip install markdown2 pygments | |
| # docutils for ReStructuredText | |
| pip install beautifulsoup4 brotli docutils gpg jinja2 \ | |
| mistune==0.8.4 pyjwt pytz whoosh | |
| - name: Install aux packages that need versions differences | |
| # if zstd fails install, keep going with test, don't abort | |
| run: | | |
| set -xv | |
| pip install zstd || true | |
| if [[ "$PYTHON_VERSION" != "2."* ]]; then | |
| pip install Markdown; fi | |
| - name: Install xapian | |
| if: matrix.python-version != '3.13' | |
| run: | | |
| set -xv | |
| sudo apt-get install libxapian-dev | |
| # Sphinx required to build the xapian python bindings. Use 1.8.5 on | |
| # older python and newest on newer. | |
| if [[ $PYTHON_VERSION == "2."* ]]; then pip install sphinx==1.8.5; fi | |
| if [[ $PYTHON_VERSION == '3.'* ]] ; then pip install sphinx; fi | |
| if [[ $PYTHON_VERSION != "3.13" ]]; then | |
| XAPIAN_VER=$(dpkg -l libxapian-dev | tail -n 1 | awk '{print $3}' | cut -d '-' -f 1); echo $XAPIAN_VER; | |
| else | |
| XAPIAN_VER="1.4.25"; echo $XAPIAN_VER; | |
| fi | |
| cd /tmp | |
| curl -s -O https://oligarchy.co.uk/xapian/$XAPIAN_VER/xapian-bindings-$XAPIAN_VER.tar.xz | |
| tar -Jxvf xapian-bindings-$XAPIAN_VER.tar.xz | |
| cd xapian-bindings-$XAPIAN_VER/ | |
| if [[ $PYTHON_VERSION == "2."* ]]; then ./configure --prefix=$VIRTUAL_ENV --with-python --disable-documentation; fi | |
| # edit the configure script. | |
| # distutils.sysconfig.get_config_vars('SO') doesn't work for | |
| # 3.11 or newer. | |
| # Change distutils.sysconfig... to just sysconfig and SO | |
| # to EXT_SUFFIX to get valid value. | |
| if [[ $PYTHON_VERSION == "3."* ]]; then \ | |
| cp configure configure.FCS; \ | |
| sed -i \ | |
| -e '/PYTHON3_SO=/s/distutils\.//g' \ | |
| -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' \ | |
| -e '/PYTHON3_CACHE_TAG=/s/imp;print(imp.get_tag())/sys;print(sys.implementation.cache_tag)/' \ | |
| -e '/PYTHON3_CACHE_OPT1_EXT=/s/imp\.get_tag()/sys.implementation.cache_tag/g' \ | |
| -e '/PYTHON3_CACHE_OPT1_EXT=/s/imp\b/importlib/g' \ | |
| configure; \ | |
| diff -u configure.FCS configure || true; \ | |
| ./configure --prefix=$VIRTUAL_ENV --with-python3 --disable-documentation; \ | |
| fi | |
| case "$PYTHON_VERSION" in "3.13") echo skipping xapian build;; *) make && sudo make install; esac | |
| - name: Test build roundup and install locale so lang tests work. | |
| run: | | |
| sudo apt-get install gettext | |
| python setup.py build | |
| (cd locale; make local_install; ls -lR locale/de/LC_MESSAGES) | |
| # Run the tests using pytest with test files in tests directory. | |
| - name: Run tests | |
| run: | | |
| if [[ "$PYTHON_VERSION" != "2."* ]]; then | |
| pytest -r a \ | |
| --durations=20 \ | |
| -W default \ | |
| -W "ignore:SelectableGroups:DeprecationWarning" \ | |
| -W "ignore:the imp module:DeprecationWarning:gpg.gpgme:15" \ | |
| -W "ignore:'U' mode::docutils.io" \ | |
| -W "ignore:unclosed:ResourceWarning:roundup.roundup.demo" \ | |
| -W "ignore:unclosed file:ResourceWarning:enum" \ | |
| -v test/ --cov=roundup | |
| if [[ "$PYTHON_VERSION" != "3.6" ]]; then | |
| # coverage before 3.6 doesn't support lcov output | |
| coverage lcov | |
| fi | |
| else | |
| # python2 case | |
| pytest -v -r a --durations=20 test/ --cov=roundup | |
| fi | |
| - name: Build docs | |
| run: python ./setup.py build_doc | |
| - name: Upload coverage to Codecov | |
| # see: https://github.com/codecov/codecov-action#usage | |
| uses: codecov/codecov-action@4fe8c5f003fae66aa5ebb77cfd3e7bfbbda0b6b0 # v3.1.5 | |
| with: | |
| verbose: true | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| - name: Upload coverage to Coveralls | |
| # python 2.7 and 3.6 versions of coverage can't produce lcov files. | |
| if: matrix.python-version != '2.7' && matrix.python-version != '3.6' | |
| uses: coverallsapp/github-action@3dfc5567390f6fa9267c0ee9c251e4c8c3f18949 # v2.2.3 | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| path-to-lcov: coverage.lcov | |
| parallel: run-{{ matrix.python-version }}-{{ matrix.os }} | |
| #- name: test docker build current version ubuntu-latest | |
| # if: matrix.python-version == '3.10' && matrix.os == 'ubuntu-latest' | |
| # run: | | |
| # docker build -t roundup-app-dev -f scripts/Docker/Dockerfile . | |
| # mkdir tracker; chmod 777 tracker | |
| # docker run -d --rm -p 9017:8080 \ | |
| # -v $PWD/tracker:/usr/src/app/tracker \ | |
| # roundup-app-dev:latest demo | |
| # expect 200 | |
| # curl --fail http://localhost:9017/demo/ > /dev/null | |
| #- name: test docker build released pip version | |
| # run: | | |
| # docker build -t roundup-app-rel --build-arg="source=pypi" \ | |
| # -f scripts/Docker/Dockerfile . | |
| - name: run benchmarks | |
| if: "contains(github.event.head_commit.message, 'benchmark')" | |
| run: INCI=1 python test/benchmark.py | |
| # in parallel build coveralls requires a finish step | |
| finish: | |
| needs: test | |
| runs-on: ubuntu-latest | |
| if: "!contains(github.event.head_commit.message, 'no-github-ci')" | |
| steps: | |
| - name: Coveralls Finished | |
| uses: coverallsapp/github-action@3dfc5567390f6fa9267c0ee9c251e4c8c3f18949 # v2.2.3 | |
| with: | |
| github-token: ${{ secrets.github_token }} | |
| parallel-finished: true |