From b5dd5034c03d487b0ba9fcb01060620c59319ec2 Mon Sep 17 00:00:00 2001 From: William Dutton Date: Fri, 2 Dec 2022 08:22:42 +1000 Subject: [PATCH] Cicd upgrades (#99) * update cicd pipeline to scripts to allow local runs * add qld-qov ckan version * wrap controller for fs_download with IOError or OSError --- .flake8 | 1 + .github/workflows/ci.yml | 47 ++++++++++++++++++++------- ckanext/s3filestore/views/__init__.py | 6 +++- scripts/build.sh | 38 ++++++++++++++++++++++ scripts/init.sh | 4 +++ scripts/test.sh | 3 ++ 6 files changed, 86 insertions(+), 13 deletions(-) create mode 100644 scripts/build.sh create mode 100644 scripts/init.sh create mode 100644 scripts/test.sh diff --git a/.flake8 b/.flake8 index 0a914dd7..7ef64c7f 100644 --- a/.flake8 +++ b/.flake8 @@ -10,6 +10,7 @@ format = pylint # Show the source of errors. show_source = True +statistics = True max-complexity = 12 max-line-length = 127 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 53dda271..618a6e5c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,27 +4,32 @@ name: CI on: push: pull_request: - branches: master + branches: + - master jobs: code_quality: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + timeout-minutes: 2 - name: Setup Python uses: actions/setup-python@v2 with: python-version: '3.x' + timeout-minutes: 5 - name: Install flake8 run: | python -m pip install --upgrade pip pip install flake8 + timeout-minutes: 5 - name: Lint with flake8 run: | - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan + flake8 + timeout-minutes: 5 test: runs-on: ubuntu-latest @@ -32,11 +37,24 @@ jobs: strategy: matrix: ckan-version: ['2.9', '2.9-py2', '2.8', '2.7'] + ckan-git-org: ['ckan'] + include: + - ckan-version: '2.8' + ckan-git-version: 'ckan-2.8.8-qgov.5' + ckan-git-org: 'qld-gov-au' + - ckan-version: '2.9' + ckan-git-version: 'ckan-2.9.5-qgov.9' + ckan-git-org: 'qld-gov-au' + - ckan-version: '2.9' + ckan-git-version: 'ckan-2.9.7-qgov' + ckan-git-org: 'qld-gov-au' + fail-fast: false - name: CKAN ${{ matrix.ckan-version }} + name: CKAN ${{ matrix.ckan-version }} ${{ matrix.ckan-git-org }} ${{ matrix.ckan-git-version }} container: image: openknowledge/ckan-dev:${{ matrix.ckan-version }} + services: postgresql: image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }} @@ -76,22 +94,27 @@ jobs: CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgresql/datastore_test CKAN_SOLR_URL: http://solr:8983/solr/ckan CKAN_REDIS_URL: redis://redis:6379/1 + CKAN_VERSION: ${{ matrix.ckan-version }} + CKAN_GIT_ORG: ${{ matrix.ckan-git-org }} + CKAN_GIT_VERSION: ${{ matrix.ckan-git-version }} + steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + timeout-minutes: 2 - name: Install requirements run: | - pip install -r requirements.txt - pip install -r dev-requirements.txt - pip install -e . - # Replace default path to CKAN core config file with the one on the container - sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini + chmod u+x ./scripts/* + ./scripts/build.sh + timeout-minutes: 15 - name: Setup CKAN run: | - chmod u+x ./scripts/ckan_cli - CKAN_INI=test.ini ./scripts/ckan_cli db init + ./scripts/init.sh + timeout-minutes: 15 - name: Run tests - run: pytest --ckan-ini=test.ini --cov=ckanext.s3filestore + run: | + ./scripts/test.sh + timeout-minutes: 30 diff --git a/ckanext/s3filestore/views/__init__.py b/ckanext/s3filestore/views/__init__.py index 7922bbb7..588a09d1 100644 --- a/ckanext/s3filestore/views/__init__.py +++ b/ckanext/s3filestore/views/__init__.py @@ -85,7 +85,11 @@ def filesystem_resource_download(id, resource_id, filename=None): except NotAuthorized: return abort(401, _('Unauthorised to read resource %s') % resource_id) upload = DefaultResourceUpload(rsc) - return upload.download(rsc['id'], filename) + try: + return upload.download(rsc['id'], filename) + except (IOError, OSError): + # probably file not found + return abort(404, _('Resource data not found')) else: try: from ckan.views.resource import download diff --git a/scripts/build.sh b/scripts/build.sh new file mode 100644 index 00000000..bf137576 --- /dev/null +++ b/scripts/build.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +## +# Build site in CI. +# +set -ex + + + +if [ "$CKAN_GIT_ORG" != "ckan" ]; then + SRC_DIR=/srv/app/src + APP_DIR=/srv/app +# PIP_SRC=${SRC_DIR} + pushd ${APP_DIR} + echo "pip install -e git+https://github.com/${CKAN_GIT_ORG}/ckan.git@${CKAN_GIT_VERSION}#egg=ckan" + echo "update manually as its already a git pip installed module" + sudo git config --global --add safe.directory "$SRC_DIR/ckan" + popd + + pushd ${SRC_DIR}/ckan + + sudo git remote set-url origin "https://github.com/${CKAN_GIT_ORG}/ckan.git" + time sudo git fetch origin #could be the tag/branch only if download time is slow + sudo git clean -f + sudo git reset --hard + sudo find . -name '*.pyc' -delete + sudo git checkout "${CKAN_GIT_VERSION}" + pip install --no-binary :all: -r requirements.txt + + popd + +fi + +pip install -r requirements.txt +pip install -r dev-requirements.txt +pip install -e . +# Replace default path to CKAN core config file with the one on the container +sed -i -e "s|use = config:.*|use = config:${SRC_DIR}/ckan/test-core.ini|" test.ini + diff --git a/scripts/init.sh b/scripts/init.sh new file mode 100644 index 00000000..620488ea --- /dev/null +++ b/scripts/init.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +CLICK_ARGS="--yes" ./scripts/ckan_cli db clean +CKAN_INI=test.ini ./scripts/ckan_cli db init \ No newline at end of file diff --git a/scripts/test.sh b/scripts/test.sh new file mode 100644 index 00000000..78f6d454 --- /dev/null +++ b/scripts/test.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +pytest --ckan-ini=test.ini --cov=ckanext.s3filestore \ No newline at end of file