Cicd upgrades (#99)

* update cicd pipeline to scripts to allow local runs
* add qld-qov ckan version
* wrap controller for fs_download with IOError or OSError

.flake8

@@ -10,6 +10,7 @@ format = pylint
 
 # Show the source of errors.
 show_source = True
+statistics = True
 
 max-complexity = 12
 max-line-length = 127

.github/workflows/ci.yml

@@ -4,39 +4,57 @@ name: CI
 on:
   push:
   pull_request:
-    branches: master
+    branches:
+    - master
 
 jobs:
   code_quality:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+        timeout-minutes: 2
 
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
           python-version: '3.x'
+        timeout-minutes: 5
 
       - name: Install flake8
         run: |
           python -m pip install --upgrade pip
           pip install flake8
+        timeout-minutes: 5
 
       - name: Lint with flake8
         run: |
-          flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan
+          flake8
+        timeout-minutes: 5
 
   test:
     runs-on: ubuntu-latest
     needs: code_quality
     strategy:
       matrix:
         ckan-version: ['2.9', '2.9-py2', '2.8', '2.7']
+        ckan-git-org: ['ckan']
+        include:
+          - ckan-version: '2.8'
+            ckan-git-version: 'ckan-2.8.8-qgov.5'
+            ckan-git-org: 'qld-gov-au'
+          - ckan-version: '2.9'
+            ckan-git-version: 'ckan-2.9.5-qgov.9'
+            ckan-git-org: 'qld-gov-au'
+          - ckan-version: '2.9'
+            ckan-git-version: 'ckan-2.9.7-qgov'
+            ckan-git-org: 'qld-gov-au'
+
       fail-fast: false
 
-    name: CKAN ${{ matrix.ckan-version }}
+    name: CKAN ${{ matrix.ckan-version }} ${{ matrix.ckan-git-org }} ${{ matrix.ckan-git-version }}
     container:
       image: openknowledge/ckan-dev:${{ matrix.ckan-version }}
+
     services:
       postgresql:
         image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }}
@@ -76,22 +94,27 @@ jobs:
       CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgresql/datastore_test
       CKAN_SOLR_URL: http://solr:8983/solr/ckan
       CKAN_REDIS_URL: redis://redis:6379/1
+      CKAN_VERSION: ${{ matrix.ckan-version }}
+      CKAN_GIT_ORG: ${{ matrix.ckan-git-org }}
+      CKAN_GIT_VERSION: ${{ matrix.ckan-git-version }}
+
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
+        timeout-minutes: 2
 
       - name: Install requirements
         run: |
-          pip install -r requirements.txt
-          pip install -r dev-requirements.txt
-          pip install -e .
-          # Replace default path to CKAN core config file with the one on the container
-          sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini
+          chmod u+x ./scripts/*
+          ./scripts/build.sh
+        timeout-minutes: 15
 
       - name: Setup CKAN
         run: |
-          chmod u+x ./scripts/ckan_cli
-          CKAN_INI=test.ini ./scripts/ckan_cli db init
+          ./scripts/init.sh
+        timeout-minutes: 15
 
       - name: Run tests
-        run: pytest --ckan-ini=test.ini --cov=ckanext.s3filestore
+        run: |
+          ./scripts/test.sh
+        timeout-minutes: 30

ckanext/s3filestore/views/__init__.py

@@ -85,7 +85,11 @@ def filesystem_resource_download(id, resource_id, filename=None):
         except NotAuthorized:
             return abort(401, _('Unauthorised to read resource %s') % resource_id)
         upload = DefaultResourceUpload(rsc)
-        return upload.download(rsc['id'], filename)
+        try:
+            return upload.download(rsc['id'], filename)
+        except (IOError, OSError):
+            # probably file not found
+            return abort(404, _('Resource data not found'))
     else:
         try:
             from ckan.views.resource import download

scripts/build.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+##
+# Build site in CI.
+#
+set -ex
+
+
+
+if [ "$CKAN_GIT_ORG" != "ckan" ]; then
+  SRC_DIR=/srv/app/src
+  APP_DIR=/srv/app
+#  PIP_SRC=${SRC_DIR}
+  pushd ${APP_DIR}
+    echo "pip install -e git+https://github.com/${CKAN_GIT_ORG}/ckan.git@${CKAN_GIT_VERSION}#egg=ckan"
+    echo "update manually as its already a git pip installed module"
+    sudo git config --global --add safe.directory "$SRC_DIR/ckan"
+  popd
+
+  pushd ${SRC_DIR}/ckan
+
+  sudo git remote set-url origin "https://github.com/${CKAN_GIT_ORG}/ckan.git"
+  time sudo git fetch origin #could be the tag/branch only if download time is slow
+  sudo git clean -f
+  sudo git reset --hard
+  sudo find . -name '*.pyc' -delete
+  sudo git checkout "${CKAN_GIT_VERSION}"
+  pip install --no-binary :all: -r requirements.txt
+
+  popd
+
+fi
+
+pip install -r requirements.txt
+pip install -r dev-requirements.txt
+pip install -e .
+# Replace default path to CKAN core config file with the one on the container
+sed -i -e "s|use = config:.*|use = config:${SRC_DIR}/ckan/test-core.ini|" test.ini
+

scripts/init.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+CLICK_ARGS="--yes" ./scripts/ckan_cli db clean
+CKAN_INI=test.ini ./scripts/ckan_cli db init

scripts/test.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+pytest --ckan-ini=test.ini --cov=ckanext.s3filestore