Merge pull request #73 from qld-gov-au/develop

Develop to master

.coveragerc

@@ -1,5 +1,8 @@
+[run]
+relative_files = True
+
 [report]
 omit =
     */site-packages/*
     */python?.?/*
-    ckan/*
+    ckan/*

.flake8

@@ -11,7 +11,8 @@ format = pylint
 # Show the source of errors.
 show_source = True
 
-max-complexity = 10
+max-complexity = 12
+max-line-length = 127
 
 # List ignore rules one per line.
 ignore =

.github/workflows/ci.yml

@@ -0,0 +1,97 @@
+
+name: CI
+
+on:
+  push:
+  pull_request:
+    branches: master
+
+jobs:
+  code_quality:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
+
+      - name: Install flake8
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8
+
+      - name: Lint with flake8
+        run: |
+          flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan
+
+  test:
+    runs-on: ubuntu-latest
+    needs: code_quality
+    strategy:
+      matrix:
+        ckan-version: ['2.9', '2.9-py2', '2.8', '2.7']
+      fail-fast: false
+
+    name: CKAN ${{ matrix.ckan-version }}
+    container:
+      image: openknowledge/ckan-dev:${{ matrix.ckan-version }}
+    services:
+      postgresql:
+        image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }}
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+      redis:
+        image: redis
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 6379:6379
+
+      moto:
+        image: motoserver/moto
+        ports:
+          - "5000"
+
+      solr:
+        image: ckan/ckan-solr-dev:${{ matrix.ckan-version }}
+
+    env:
+      CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgresql/ckan_test
+      CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgresql/datastore_test
+      CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgresql/datastore_test
+      CKAN_SOLR_URL: http://solr:8983/solr/ckan
+      CKAN_REDIS_URL: redis://redis:6379/1
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Install requirements
+        run: |
+          pip install -r requirements.txt
+          pip install -r dev-requirements.txt
+          pip install -e .
+          # Replace default path to CKAN core config file with the one on the container
+          sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini
+
+      - name: Setup CKAN
+        run: |
+          chmod u+x ./scripts/ckan_cli
+          CKAN_INI=test.ini ./scripts/ckan_cli db init
+
+      - name: Run tests
+        run: pytest --ckan-ini=test.ini --cov=ckanext.s3filestore

.gitignore

@@ -38,8 +38,9 @@ htmlcov/
 nosetests.xml
 coverage.xml
 
+.idea
+
 # Sphinx documentation
 docs/_build/
 
 subdir/
-*.idea

README.rst

@@ -1,10 +1,5 @@
-.. You should enable this project on travis-ci.org and coveralls.io to make
-   these badges work. The necessary Travis and Coverage config files have been
-   generated for you.
-
-.. image:: https://travis-ci.org/okfn/ckanext-s3filestore.svg?branch=master
-    :target: https://travis-ci.org/okfn/ckanext-s3filestore
-
+.. You should enable this project on coveralls.io to make these badges
+   work. The necessary Coverage config file has been generated for you.
 
 .. image:: https://coveralls.io/repos/okfn/ckanext-s3filestore/badge.svg
   :target: https://coveralls.io/r/okfn/ckanext-s3filestore
@@ -59,7 +54,6 @@ Config Settings
 
 Required::
 
-
     ckanext.s3filestore.aws_bucket_name = a-bucket-to-store-your-stuff
     ckanext.s3filestore.region_name = region-name
     ckanext.s3filestore.signature_version = s3v4
@@ -107,10 +101,11 @@ Optional::
     # To use user provided filepath and not use internal url basename
     # on download. This affects behaviour when using a URL that points
     # to an earlier version of a resource, with a different file name.
-    # If True, the older file will be served; if False, the newest file
-    # will be served, but the older file will be available at
+    # If True, the older file will be served.
+    # If False, the newest file will be served (similarly to the default
+    # CKAN filestore behaviour), but the older file will be available at
     # /dataset/{id}/resource/{resource_id}/orig_download/{filename}
-    # Set to False to be backwards compatible to ckan file store.
+    # Defaults to False.
     ckanext.s3filestore.use_filename = True
 
     # To mask the S3 endpoint with your own domain/endpoint when serving URLs to end users.
@@ -129,9 +124,18 @@ Optional::
     ckanext.s3filestore.signed_url_cache_window = 1800
 
     # Queue used by s3 plugin, if not set, default queue is used
-    # i.e.
     ckanext.s3filestore.queue = bulk
 
+
+-----------------
+CLI
+-----------------
+
+To upload all local resources located in `ckan.storage_path` location dir to the configured S3 bucket use::
+
+    ckan -c /etc/ckan/default/production.ini s3 upload all
+
+
 ------------------------
 Development Installation
 ------------------------
@@ -152,12 +156,12 @@ Running the Tests
 
 To run the tests, do::
 
-    nosetests --ckan --nologcapture --with-pylons=test.ini
+    pytest --ckan-ini=test.ini
 
 To run the tests and produce a coverage report, first make sure you have
 coverage installed in your virtualenv (``pip install coverage``) then run::
 
-    nosetests --ckan --nologcapture --with-pylons=test.ini --with-coverage --cover-package=ckanext.s3filestore --cover-inclusive --cover-erase --cover-tests
+    pytest --ckan-ini=test.ini --cov=ckanext.s3filestore
 
 ------------------------
 Docker environment setup

ckanext/__init__.py

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 # this is a namespace package
 try:
     import pkg_resources