From ae834536d0743e2206b4e2b3fa12bb39bb6be026 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Thu, 13 Jan 2022 16:23:14 +1000 Subject: [PATCH 01/11] [QOL-8518] fix missing background task argument --- README.rst | 1 - ckanext/s3filestore/plugin.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 425caa57..880e0815 100644 --- a/README.rst +++ b/README.rst @@ -129,7 +129,6 @@ Optional:: ckanext.s3filestore.signed_url_cache_window = 1800 # Queue used by s3 plugin, if not set, default queue is used - # i.e. ckanext.s3filestore.queue = bulk ------------------------ diff --git a/ckanext/s3filestore/plugin.py b/ckanext/s3filestore/plugin.py index df977890..901e9772 100644 --- a/ckanext/s3filestore/plugin.py +++ b/ckanext/s3filestore/plugin.py @@ -116,7 +116,7 @@ def after_update_resource_list_update(self, visibility_level, pkg_id, pkg_dict): def enqueue_resource_visibility_update_job(self, visibility_level, pkg_id, pkg_dict): ckan_ini_filepath = os.path.abspath(toolkit.config['__file__']) resources = pkg_dict - args = [ckan_ini_filepath, visibility_level, resources] + args = [ckan_ini_filepath, visibility_level, pkg_id, resources] kwargs = { 'args': args, 'title': "s3_afterUpdatePackage: setting " + visibility_level + " on " + pkg_id From 657b2113eea01d1f87eac7d9e1b1d31da09b092c Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Thu, 13 Jan 2022 16:26:24 +1000 Subject: [PATCH 02/11] [QOL-8518] only run GitHub Actions on pull requests if merging to master --- .github/workflows/test.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d2a3bced..f5f7c774 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,5 +1,10 @@ name: Tests -on: [push, pull_request] +on: + push: + pull_request: + branches: + - master + jobs: lint: runs-on: ubuntu-latest From ac7d5800ae22e99a85b19c66957c38fd57a4246c Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 19 Jan 2022 15:57:05 +1000 Subject: [PATCH 03/11] [QOL-8518] drop obsolete Travis config --- .travis.yml | 23 --------------- bin/travis-build.bash | 67 ------------------------------------------- bin/travis-run.sh | 3 -- 3 files changed, 93 deletions(-) delete mode 100644 .travis.yml delete mode 100644 bin/travis-build.bash delete mode 100644 bin/travis-run.sh diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 090dcdd6..00000000 --- a/.travis.yml +++ /dev/null @@ -1,23 +0,0 @@ -os: linux -language: python -dist: bionic -python: - - "2.7" -env: - - PGVERSION=11 CKAN_GIT_REPO=ckan/ckan CKAN_BRANCH=2.8 - - PGVERSION=11 CKAN_GIT_REPO=qld-gov-au/ckan CKAN_BRANCH=qgov-master -jobs: - include: - - name: "python 3.6 ckan master" - python: "3.6" - env: PGVERSION=11 CKAN_GIT_REPO=ckan/ckan CKAN_BRANCH=master - allow_failures: - - python: 3.6 - env: PGVERSION=11 CKAN_BRANCH=master CKAN_GIT_REPO=ckan/ckan - -install: - - bash bin/travis-build.bash - - pip install coveralls -U -script: sh bin/travis-run.sh -after_success: - - coveralls diff --git a/bin/travis-build.bash b/bin/travis-build.bash deleted file mode 100644 index da5c9148..00000000 --- a/bin/travis-build.bash +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash -set -e - -echo "This is travis-build.bash..." - -echo "Installing the packages that CKAN requires..." -sudo apt-get update -qq -sudo apt-get install systemd git python-pip postgresql-$PGVERSION solr-jetty openjdk-8-jdk libcommons-fileupload-java:amd64 redis-server -echo "Start postgres" -sudo service postgresql start - -echo "Start redis" -sudo service redis-server start - -echo "Start s3 mock" -# Run s3 moto local client just in case we can't mock direclty via tests -pip install "moto[server]" -moto_server s3 & - -echo "Installing CKAN and its Python dependencies..." -if [ ! -d ckan ]; then - git clone https://github.com/$CKAN_GIT_REPO -fi -cd ckan -git checkout $CKAN_BRANCH -python setup.py develop -pip install -r requirements.txt -pip install -r dev-requirements.txt -cd - - -echo "start solr" -# Fix solr-jetty starting issues https://stackoverflow.com/a/56007895 -# https://github.com/Zharktas/ckanext-report/blob/py3/bin/travis-run.bash -sudo mkdir -p /etc/systemd/system/jetty9.service.d -printf "[Service]\nReadWritePaths=/var/lib/solr" | sudo tee /etc/systemd/system/jetty9.service.d/solr.conf -sed '16,21d' /etc/solr/solr-jetty.xml | sudo tee /etc/solr/solr-jetty.xml -sudo systemctl daemon-reload - -printf "NO_START=0\nJETTY_HOST=127.0.0.1\nJETTY_ARGS=\"jetty.http.port=8983\"\nJAVA_HOME=$JAVA_HOME" | sudo tee /etc/default/jetty9 -sudo cp ckan/ckan/config/solr/schema.xml /etc/solr/conf/schema.xml -sudo service jetty9 restart - -# Wait for jetty9 to start -timeout 20 bash -c 'while [[ "$(curl -s -o /dev/null -I -w %{http_code} http://localhost:8983)" != "200" ]]; do sleep 2;done' - -echo "Creating the PostgreSQL user and database..." -sudo -u postgres psql -c "CREATE USER ckan_default WITH PASSWORD 'pass';" -sudo -u postgres psql -c 'CREATE DATABASE ckan_test WITH OWNER ckan_default;' - -echo "Initialising the database..." -cd ckan -paster db init -c test-core.ini -cd - - -echo "Installing ckanext-s3filestore and its requirements..." -python setup.py develop -pip install -r requirements.txt -pip install -r dev-requirements.txt - -echo "Moving test.ini into a subdir..." -mkdir -p subdir -cp test.ini subdir - - - -echo "travis-build.bash is done." - diff --git a/bin/travis-run.sh b/bin/travis-run.sh deleted file mode 100644 index 868761b8..00000000 --- a/bin/travis-run.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -e - -nosetests --ckan --nologcapture --with-pylons=subdir/test.ini --with-coverage --cover-package=ckanext.s3filestore --cover-inclusive --cover-erase --cover-tests From cb5e1dbd6b7356a3040c8c12da573dbf4134ba7e Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 19 Jan 2022 15:58:11 +1000 Subject: [PATCH 04/11] [QOL-8518] split Paster commands from Click --- ckanext/s3filestore/cli_commands.py | 208 ++++++++++++++++++++++++++ ckanext/s3filestore/click_commands.py | 29 ++++ ckanext/s3filestore/commands.py | 207 +------------------------ ckanext/s3filestore/plugin.py | 16 +- 4 files changed, 253 insertions(+), 207 deletions(-) create mode 100644 ckanext/s3filestore/cli_commands.py create mode 100644 ckanext/s3filestore/click_commands.py diff --git a/ckanext/s3filestore/cli_commands.py b/ckanext/s3filestore/cli_commands.py new file mode 100644 index 00000000..964d4ddf --- /dev/null +++ b/ckanext/s3filestore/cli_commands.py @@ -0,0 +1,208 @@ +# encoding: utf-8 + +from botocore.exceptions import ClientError +import os +import sys + +from sqlalchemy import create_engine +from sqlalchemy.sql import text +from ckan.plugins.toolkit import config +from ckanext.s3filestore import uploader +from ckan.logic import get_action, ValidationError +from ckanext.s3filestore.uploader import get_s3_session, S3FileStoreException + + +class DBConnection: + + def __init__(self, config): + self.SQLALCHEMY_URL = config.get('sqlalchemy.url', 'postgresql://user:pass@localhost/db') + + def __enter__(self): + self.engine = create_engine(self.SQLALCHEMY_URL) + self.connection = self.engine.connect() + return self.connection + + def __exit__(self, exc_type, exc_value, traceback): + self.connection.close() + self.engine.dispose() + + +class S3FilestoreCommands(): + + def check_config(self): + exit = False + required_keys = ('ckanext.s3filestore.aws_bucket_name', + 'ckanext.s3filestore.region_name', + 'ckanext.s3filestore.signature_version') + if not config.get('ckanext.s3filestore.aws_use_ami_role'): + required_keys += ('ckanext.s3filestore.aws_access_key_id', + 'ckanext.s3filestore.aws_secret_access_key') + for key in required_keys: + if not config.get(key): + print('You must set the "{0}" option in your ini file'.format(key)) + exit = True + if exit: + sys.exit(1) + + print('All configuration options defined') + bucket_name = config.get('ckanext.s3filestore.aws_bucket_name') + + try: + uploader.BaseS3Uploader().get_s3_bucket(bucket_name) + except S3FileStoreException as ex: + print('An error was found while finding or creating the bucket:') + print(str(ex)) + sys.exit(1) + + print('Configuration OK!') + + def upload_all(self): + BASE_PATH = config.get('ckan.storage_path', '/var/lib/ckan/default/resources') + resource_ids_and_paths = {} + + for root, dirs, files in os.walk(BASE_PATH): + if files: + resource_id = root.split('/')[-2] + root.split('/')[-1] + files[0] + resource_ids_and_paths[resource_id] = os.path.join(root, files[0]) + + print('Found {0} resource files in the file system'.format( + len(resource_ids_and_paths.keys()))) + + with DBConnection(config) as connection: + resource_ids_and_names = {} + + for resource_id, file_path in resource_ids_and_paths.iteritems(): + resource = connection.execute(text(''' + SELECT id, url + FROM resource + WHERE id = :id + AND state = 'active' + AND url IS NOT NULL + AND url <> '' + AND url_type = 'upload' + '''), id=resource_id) + if resource.rowcount: + _id, url = resource.first() + file_name = url.split('/')[-1] if '/' in url else url + resource_ids_and_names[_id] = file_name.lower() + else: + print("{} is an orphan; no resource points to it".format(file_path)) + + print('{0} resources matched on the database'.format( + len(resource_ids_and_names.keys()))) + + _upload_files_to_s3(resource_ids_and_names, resource_ids_and_paths) + + def upload_single(self, id): + with DBConnection(config) as connection: + resource_ids_and_names = {} + for resource in connection.execute(text(''' + SELECT id, url + FROM resource + WHERE (id = :id or package_id = :id) + AND state = 'active' + AND url IS NOT NULL + AND url <> '' + AND url_type = 'upload' + '''), id=id): + _id, url = resource + file_name = url.split('/')[-1] if '/' in url else url + resource_ids_and_names[_id] = file_name.lower() + + print('{0} resources matched on the database'.format( + len(resource_ids_and_names.keys()))) + + BASE_PATH = config.get('ckan.storage_path', '/var/lib/ckan/default/resources') + resource_ids_and_paths = {} + for resource_id in resource_ids_and_names.keys(): + path = '{}/{}/{}/{}'.format(BASE_PATH, resource_id[0:2], resource_id[3:5], resource[6:]) + if os.path.isfile(path): + resource_ids_and_paths[resource_id] = path + + print('Found {0} resource files in the file system'.format( + len(resource_ids_and_paths.keys()))) + + _upload_files_to_s3(resource_ids_and_names, resource_ids_and_paths) + + def upload_pairtree(self): + def _to_pairtree_path(path): + return os.path.join(*[path[i:i + 2] for i in range(0, len(path), 2)]) + + BASE_PATH = os.path.join( + config.get('ckan.storage_path', config.get('ofs.storage_dir', '/var/lib/ckan/default')), + 'pairtree_root', + _to_pairtree_path(config.get('ckan.storage.key_prefix', 'ckan-file')), + 'obj' + ) + print("Uploading pairtree files from {}".format(BASE_PATH)) + resource_paths = [] + resource_ids_and_paths = {} + + # identify files on disk + for root, dirs, files in os.walk(BASE_PATH): + if files: + path = root.split('/')[-1] + resource_paths.append(path + '/' + files[0]) + + print('Found {0} resource files in the file system'.format( + len(resource_paths))) + + # match files to resource URLs + with DBConnection(config) as connection: + resource_ids_and_names = {} + + SITE_URL = config.get('ckan.site_url') + BASE_URL = SITE_URL + '/storage/f/' + for file_path in resource_paths: + pairtree_url = BASE_URL + file_path.replace(':', '%3A') + resource = connection.execute(text(''' + SELECT id, url + FROM resource + WHERE url = :url + AND state = 'active' + AND url IS NOT NULL + AND url <> '' + '''), url=pairtree_url) + if resource.rowcount: + _id, url = resource.first() + if url: + file_name = url.split('/')[-1] if '/' in url else url + resource_ids_and_names[_id] = file_name.lower() + resource_ids_and_paths[_id] = BASE_PATH + '/' + file_path + else: + print("{} is an orphan; no resource points to it".format(file_path)) + + resource_count = len(resource_ids_and_names.keys()) + print('{0} resources matched on the database'.format(resource_count)) + if resource_count == 0: + return + + _upload_files_to_s3(resource_ids_and_names, resource_ids_and_paths) + + +def _upload_files_to_s3(resource_ids_and_names, resource_ids_and_paths): + AWS_BUCKET_NAME = config.get('ckanext.s3filestore.aws_bucket_name') + AWS_S3_ACL = config.get('ckanext.s3filestore.acl', 'public-read') + s3_connection = get_s3_session(config).client('s3') + + context = {'ignore_auth': True} + uploaded_resources = [] + for resource_id, file_name in resource_ids_and_names.iteritems(): + key = 'resources/{resource_id}/{file_name}'.format( + resource_id=resource_id, file_name=file_name) + + try: + s3_connection.head_object(Bucket=AWS_BUCKET_NAME, Key=key) + print("{} is already in S3, skipping".format(key)) + continue + except ClientError: + s3_connection.put_object(Bucket=AWS_BUCKET_NAME, Key=key, Body=open(resource_ids_and_paths[resource_id]), ACL=AWS_S3_ACL) + uploaded_resources.append(resource_id) + print('Uploaded resource {0} ({1}) to S3'.format(resource_id, file_name)) + try: + get_action('resource_patch')(context, {'id': resource_id, 'url': file_name}) + except ValidationError: + print("{} failed to validate; file is in S3 but might not be used".format(resource_id)) + pass + + print('Done, uploaded {0} resources to S3'.format(len(uploaded_resources))) diff --git a/ckanext/s3filestore/click_commands.py b/ckanext/s3filestore/click_commands.py new file mode 100644 index 00000000..1f48e8cb --- /dev/null +++ b/ckanext/s3filestore/click_commands.py @@ -0,0 +1,29 @@ +# encoding: utf-8 + +import click + +from ckanext.s3filestore.cli_commands import S3FilestoreCommands + + +@click.group() +def s3(): + """ S3 Filestore commands + """ + pass + + +@s3.command(short_help=u'CKAN S3 FileStore utilities') +def check_config(): + S3FilestoreCommands().check_config() + + +@s3.command() +@click.argument(u'identifier', default='all') +def upload(identifier): + commands = S3FilestoreCommands() + if identifier == 'all': + commands.upload_all() + elif identifier == 'pairtree': + commands.upload_pairtree() + else: + commands.upload_single(identifier) diff --git a/ckanext/s3filestore/commands.py b/ckanext/s3filestore/commands.py index b52a936d..292df633 100644 --- a/ckanext/s3filestore/commands.py +++ b/ckanext/s3filestore/commands.py @@ -1,34 +1,13 @@ -from __future__ import print_function +# encoding: utf-8 -from botocore.exceptions import ClientError -import os import sys -from sqlalchemy import create_engine -from sqlalchemy.sql import text -import ckantoolkit as toolkit -from ckantoolkit import config -import ckanext.s3filestore.uploader -from ckan.logic import get_action, ValidationError -from uploader import get_s3_session +from ckan.plugins import toolkit +from cli_commands import S3FilestoreCommands -class DBConnection: - def __init__(self, config): - self.SQLALCHEMY_URL = config.get('sqlalchemy.url', 'postgresql://user:pass@localhost/db') - - def __enter__(self): - self.engine = create_engine(self.SQLALCHEMY_URL) - self.connection = self.engine.connect() - return self.connection - - def __exit__(self, exc_type, exc_value, traceback): - self.connection.close() - self.engine.dispose() - - -class TestConnection(toolkit.CkanCommand): +class TestConnection(toolkit.CkanCommand, S3FilestoreCommands): '''CKAN S3 FileStore utilities Usage: @@ -73,181 +52,3 @@ def command(self): self.upload_single(self.args[1]) else: self.parser.error('Unrecognized command') - - def check_config(self): - exit = False - required_keys = ('ckanext.s3filestore.aws_bucket_name', - 'ckanext.s3filestore.region_name', - 'ckanext.s3filestore.signature_version') - if not config.get('ckanext.s3filestore.aws_use_ami_role'): - required_keys += ('ckanext.s3filestore.aws_access_key_id', - 'ckanext.s3filestore.aws_secret_access_key') - for key in required_keys: - if not config.get(key): - print('You must set the "{0}" option in your ini file'.format(key)) - exit = True - if exit: - sys.exit(1) - - print('All configuration options defined') - bucket_name = config.get('ckanext.s3filestore.aws_bucket_name') - - try: - ckanext.s3filestore.uploader.BaseS3Uploader().get_s3_bucket(bucket_name) - except ckanext.S3FileStoreException as ex: - print('An error was found while finding or creating the bucket:') - print(str(ex)) - sys.exit(1) - - print('Configuration OK!') - - def upload_all(self): - BASE_PATH = config.get('ckan.storage_path', '/var/lib/ckan/default/resources') - resource_ids_and_paths = {} - - for root, dirs, files in os.walk(BASE_PATH): - if files: - resource_id = root.split('/')[-2] + root.split('/')[-1] + files[0] - resource_ids_and_paths[resource_id] = os.path.join(root, files[0]) - - print('Found {0} resource files in the file system'.format( - len(resource_ids_and_paths.keys()))) - - with DBConnection(config) as connection: - resource_ids_and_names = {} - - for resource_id, file_path in resource_ids_and_paths.iteritems(): - resource = connection.execute(text(''' - SELECT id, url - FROM resource - WHERE id = :id - AND state = 'active' - AND url IS NOT NULL - AND url <> '' - AND url_type = 'upload' - '''), id=resource_id) - if resource.rowcount: - _id, url = resource.first() - file_name = url.split('/')[-1] if '/' in url else url - resource_ids_and_names[_id] = file_name.lower() - else: - print("{} is an orphan; no resource points to it".format(file_path)) - - print('{0} resources matched on the database'.format( - len(resource_ids_and_names.keys()))) - - _upload_files_to_s3(resource_ids_and_names, resource_ids_and_paths) - - def upload_single(self, id): - with DBConnection(config) as connection: - resource_ids_and_names = {} - for resource in connection.execute(text(''' - SELECT id, url - FROM resource - WHERE (id = :id or package_id = :id) - AND state = 'active' - AND url IS NOT NULL - AND url <> '' - AND url_type = 'upload' - '''), id=id): - _id, url = resource - file_name = url.split('/')[-1] if '/' in url else url - resource_ids_and_names[_id] = file_name.lower() - - print('{0} resources matched on the database'.format( - len(resource_ids_and_names.keys()))) - - BASE_PATH = config.get('ckan.storage_path', '/var/lib/ckan/default/resources') - resource_ids_and_paths = {} - for resource_id in resource_ids_and_names.keys(): - path = '{}/{}/{}/{}'.format(BASE_PATH, resource_id[0:2], resource_id[3:5], resource[6:]) - if os.path.isfile(path): - resource_ids_and_paths[resource_id] = path - - print('Found {0} resource files in the file system'.format( - len(resource_ids_and_paths.keys()))) - - _upload_files_to_s3(resource_ids_and_names, resource_ids_and_paths) - - def upload_pairtree(self): - def _to_pairtree_path(path): - return os.path.join(*[path[i:i + 2] for i in range(0, len(path), 2)]) - - BASE_PATH = os.path.join( - config.get('ckan.storage_path', config.get('ofs.storage_dir', '/var/lib/ckan/default')), - 'pairtree_root', - _to_pairtree_path(config.get('ckan.storage.key_prefix', 'ckan-file')), - 'obj' - ) - print("Uploading pairtree files from {}".format(BASE_PATH)) - resource_paths = [] - resource_ids_and_paths = {} - - # identify files on disk - for root, dirs, files in os.walk(BASE_PATH): - if files: - path = root.split('/')[-1] - resource_paths.append(path + '/' + files[0]) - - print('Found {0} resource files in the file system'.format( - len(resource_paths))) - - # match files to resource URLs - with DBConnection(config) as connection: - resource_ids_and_names = {} - - SITE_URL = config.get('ckan.site_url') - BASE_URL = SITE_URL + '/storage/f/' - for file_path in resource_paths: - pairtree_url = BASE_URL + file_path.replace(':', '%3A') - resource = connection.execute(text(''' - SELECT id, url - FROM resource - WHERE url = :url - AND state = 'active' - AND url IS NOT NULL - AND url <> '' - '''), url=pairtree_url) - if resource.rowcount: - _id, url = resource.first() - if url: - file_name = url.split('/')[-1] if '/' in url else url - resource_ids_and_names[_id] = file_name.lower() - resource_ids_and_paths[_id] = BASE_PATH + '/' + file_path - else: - print("{} is an orphan; no resource points to it".format(file_path)) - - resource_count = len(resource_ids_and_names.keys()) - print('{0} resources matched on the database'.format(resource_count)) - if resource_count == 0: - return - - _upload_files_to_s3(resource_ids_and_names, resource_ids_and_paths) - - -def _upload_files_to_s3(resource_ids_and_names, resource_ids_and_paths): - AWS_BUCKET_NAME = config.get('ckanext.s3filestore.aws_bucket_name') - AWS_S3_ACL = config.get('ckanext.s3filestore.acl', 'public-read') - s3_connection = get_s3_session(config).client('s3') - - context = {'ignore_auth': True} - uploaded_resources = [] - for resource_id, file_name in resource_ids_and_names.iteritems(): - key = 'resources/{resource_id}/{file_name}'.format( - resource_id=resource_id, file_name=file_name) - - try: - s3_connection.head_object(Bucket=AWS_BUCKET_NAME, Key=key) - print("{} is already in S3, skipping".format(key)) - continue - except ClientError: - s3_connection.put_object(Bucket=AWS_BUCKET_NAME, Key=key, Body=open(resource_ids_and_paths[resource_id]), ACL=AWS_S3_ACL) - uploaded_resources.append(resource_id) - print('Uploaded resource {0} ({1}) to S3'.format(resource_id, file_name)) - try: - get_action('resource_patch')(context, {'id': resource_id, 'url': file_name}) - except ValidationError: - print("{} failed to validate; file is in S3 but might not be used".format(resource_id)) - pass - - print('Done, uploaded {0} resources to S3'.format(len(uploaded_resources))) diff --git a/ckanext/s3filestore/plugin.py b/ckanext/s3filestore/plugin.py index 901e9772..e4d8030a 100644 --- a/ckanext/s3filestore/plugin.py +++ b/ckanext/s3filestore/plugin.py @@ -6,8 +6,6 @@ from ckan import plugins from ckanext.s3filestore import uploader as s3_uploader -from ckanext.s3filestore.views import\ - resource as resource_view, uploads as uploads_view from ckan.lib.uploader import ResourceUpload as DefaultResourceUpload,\ get_resource_uploader @@ -24,8 +22,9 @@ class S3FileStorePlugin(plugins.SingletonPlugin): plugins.implements(plugins.IUploader) plugins.implements(plugins.IPackageController, inherit=True) - if toolkit.check_ckan_version(min_version='2.8.0'): + if toolkit.check_ckan_version(min_version='2.9.0'): plugins.implements(plugins.IBlueprint) + plugins.implements(plugins.IClick) else: plugins.implements(plugins.IRoutes, inherit=True) @@ -166,4 +165,13 @@ def before_map(self, map): # Ignored on CKAN < 2.8 def get_blueprint(self): - return resource_view.get_blueprints() + uploads_view.get_blueprints() + from ckanext.s3filestore.views import\ + resource, uploads + return resource.get_blueprints() + uploads.get_blueprints() + + # IClick + # Ignored on CKAN < 2.9 + + def get_commands(self): + from ckanext.s3filestore import click_commands + return [click_commands.s3] From 5b1f2efdae1e8ad8159903e175889b19f6b3d03c Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 19 Jan 2022 16:05:32 +1000 Subject: [PATCH 05/11] [QOL-8518] fix Flask support and clean up supporting tests - extract common routing logic to shared functions - drop test features specific to one CKAN version - prepare for pytest --- ckanext/s3filestore/controller.py | 154 +------ ckanext/s3filestore/plugin.py | 2 +- ckanext/s3filestore/tests/__init__.py | 26 +- ckanext/s3filestore/tests/fixtures.py | 57 --- ckanext/s3filestore/tests/test_controller.py | 442 +++++++++++-------- ckanext/s3filestore/tests/test_uploader.py | 145 +++--- ckanext/s3filestore/uploader.py | 43 +- ckanext/s3filestore/views/__init__.py | 116 +++++ ckanext/s3filestore/views/resource.py | 149 +------ ckanext/s3filestore/views/uploads.py | 31 +- conftest.py | 7 - 11 files changed, 501 insertions(+), 671 deletions(-) delete mode 100644 ckanext/s3filestore/tests/fixtures.py delete mode 100644 conftest.py diff --git a/ckanext/s3filestore/controller.py b/ckanext/s3filestore/controller.py index a51f2dfe..8077fd96 100644 --- a/ckanext/s3filestore/controller.py +++ b/ckanext/s3filestore/controller.py @@ -1,159 +1,17 @@ # encoding: utf-8 -import os -import mimetypes -from ckantoolkit import config +from ckan.plugins.toolkit import BaseController -import ckantoolkit as toolkit -import ckan.logic as logic -import ckan.lib.base as base -import ckan.model as model -import ckan.lib.uploader as uploader -from ckan.common import _, c, request, response -from botocore.exceptions import ClientError +from .views import resource_download, filesystem_resource_download, uploaded_file_redirect -from ckan.lib.uploader import ResourceUpload as DefaultResourceUpload -from ckanext.s3filestore.uploader import S3Uploader, is_path_addressing -import paste.fileapp -import logging -log = logging.getLogger(__name__) - -NotFound = logic.NotFound -NotAuthorized = logic.NotAuthorized -get_action = logic.get_action -abort = base.abort -redirect = toolkit.redirect_to - - -class S3Controller(base.BaseController): +class S3Controller(BaseController): def resource_download(self, id, resource_id, filename=None): - ''' - Provide a download by either redirecting the user to the url stored or - downloading the uploaded file from S3. - ''' - context = {'model': model, 'session': model.Session, - 'user': c.user or c.author, 'auth_user_obj': c.userobj} - - try: - rsc = get_action('resource_show')(context, {'id': resource_id}) - get_action('package_show')(context, {'id': id}) - except NotFound: - abort(404, _('Resource not found')) - except NotAuthorized: - abort(401, _('Unauthorized to read resource %s') % id) - - if 'url' not in rsc: - abort(404, _('No download is available')) - elif rsc.get('url_type') == 'upload': - upload = uploader.get_resource_uploader(rsc) - bucket_name = config.get('ckanext.s3filestore.aws_bucket_name') - - if filename is None: - filename = os.path.basename(rsc['url']) - key_path = upload.get_path(rsc['id'], filename) - - if filename is None: - log.warn("Key '%s' not found in bucket '%s'", - key_path, bucket_name) - - try: - url = upload.get_signed_url_to_key(key_path) - redirect(url) - except ClientError as ex: - if ex.response['Error']['Code'] in ['NoSuchKey', '404']: - # attempt fallback - if config.get( - 'ckanext.s3filestore.filesystem_download_fallback', - False): - log.info('Attempting filesystem fallback for resource %s', - resource_id) - url = toolkit.url_for( - controller='ckanext.s3filestore.controller:S3Controller', - action='filesystem_resource_download', - id=id, - resource_id=resource_id, - filename=filename) - redirect(url) - - abort(404, _('Resource data not found')) - else: - raise ex - redirect(rsc['url']) + return resource_download(id, resource_id, filename) def filesystem_resource_download(self, id, resource_id, filename=None): - """ - A fallback controller action to download resources from the - filesystem. A copy of the action from - `ckan.controllers.package:PackageController.resource_download`. - - Provide a direct download by either redirecting the user to the url - stored or downloading an uploaded file directly. - """ - context = {'model': model, 'session': model.Session, - 'user': c.user or c.author, 'auth_user_obj': c.userobj} - - try: - rsc = get_action('resource_show')(context, {'id': resource_id}) - get_action('package_show')(context, {'id': id}) - except NotFound: - abort(404, _('Resource not found')) - except NotAuthorized: - abort(401, _('Unauthorised to read resource %s') % resource_id) - - if rsc.get('url_type') == 'upload': - upload = DefaultResourceUpload(rsc) - try: - if hasattr(upload, 'download'): - return upload.download(rsc['id'], filename) - else: - # this is a copy of the original logic - # in case CKAN doesn't provide the 'download' function - # TODO get the 'download' function into upstream CKAN - filepath = upload.get_path(rsc['id']) - fileapp = paste.fileapp.FileApp(filepath) - status, headers, app_iter = request.call_application(fileapp) - response.headers.update(dict(headers)) - content_type, content_enc = mimetypes.guess_type( - rsc.get('url', '')) - if content_type: - response.headers['Content-Type'] = content_type - response.status = status - return app_iter - except (OSError, IOError): - # includes FileNotFoundError - abort(404, _('Resource data not found')) - except Exception as e: - log.warning("Unhandled exception %s of type %s", e, type(e)) - raise e - elif 'url' not in rsc: - abort(404, _('No download is available')) - redirect(rsc['url']) + return filesystem_resource_download(id, resource_id, filename) def uploaded_file_redirect(self, upload_to, filename): - '''Redirect static file requests to their location on S3.''' - bucket_name = config.get('ckanext.s3filestore.aws_bucket_name') - region_name = config.get('ckanext.s3filestore.region_name') - if is_path_addressing(): - host_name = config.get('ckanext.s3filestore.host_name', - 'https://s3-{region_name}.amazonaws.com'.format( - region_name=region_name - )) - # ensure trailing slash - if host_name[-1] != '/': - host_name += '/' - host_name += bucket_name - else: - host_name = config.get('ckanext.s3filestore.download_proxy', - 'https://{bucket_name}.s3.{region_name}.amazonaws.com'.format( - bucket_name=bucket_name, - region_name=region_name - )) - storage_path = S3Uploader.get_storage_path(upload_to) - filepath = os.path.join(storage_path, filename) - - redirect_url = '{host_name}/{filepath}'\ - .format(filepath=filepath, - host_name=host_name) - redirect(redirect_url) + return uploaded_file_redirect(upload_to, filename) diff --git a/ckanext/s3filestore/plugin.py b/ckanext/s3filestore/plugin.py index e4d8030a..f4253adc 100644 --- a/ckanext/s3filestore/plugin.py +++ b/ckanext/s3filestore/plugin.py @@ -144,7 +144,7 @@ def before_map(self, map): action='resource_download') # fallback controller action to download from the filesystem - m.connect('filesystem_resource_download', + m.connect('s3_resource.filesystem_resource_download', '/dataset/{id}/resource/{resource_id}/fs_download/{filename}', action='filesystem_resource_download') diff --git a/ckanext/s3filestore/tests/__init__.py b/ckanext/s3filestore/tests/__init__.py index c693f220..45b1ae19 100644 --- a/ckanext/s3filestore/tests/__init__.py +++ b/ckanext/s3filestore/tests/__init__.py @@ -1,15 +1,19 @@ # encoding: utf-8 -from ckantoolkit import config -import boto3 +def _get_status_code(response): + """ Get the status code from a HTTP response. + Supports both Pylons/WebOb and Flask. + """ + if hasattr(response, 'status_code'): + return response.status_code + elif hasattr(response, 'status_int'): + return response.status_int + else: + raise Exception("No status code found on %s" % response) -# moto AWS mock is started externally on port 5000 -endpoint_url = config.get('ckanext.s3filestore.host_name', 'http://localhost:5000') -botoSession = boto3.Session(region_name='ap-southeast-2', aws_access_key_id='a', aws_secret_access_key='b') -s3 = botoSession.client('s3', endpoint_url=endpoint_url) -BUCKET_NAME = 'my-bucket' - -def setup_package(self): - # We need to create the bucket since this is all in Moto's 'virtual' AWS account - s3.create_bucket(Bucket=BUCKET_NAME) +def _get_response_body(response): + if hasattr(response, 'text'): + return response.text + else: + return response.body diff --git a/ckanext/s3filestore/tests/fixtures.py b/ckanext/s3filestore/tests/fixtures.py deleted file mode 100644 index d9fdcd01..00000000 --- a/ckanext/s3filestore/tests/fixtures.py +++ /dev/null @@ -1,57 +0,0 @@ -# encoding: utf-8 -import pytest - -import ckan.tests.factories as factories - -from ckanext.s3filestore.uploader import BaseS3Uploader - - -@pytest.fixture -def s3_session(ckan_config): - base_uploader = BaseS3Uploader() - return base_uploader.get_s3_session() - - -@pytest.fixture -def s3_resource(ckan_config, s3_session): - base_uploader = BaseS3Uploader() - return base_uploader.get_s3_resource() - - -@pytest.fixture -def s3_client(ckan_config, s3_session): - base_uploader = BaseS3Uploader() - return base_uploader.get_s3_client() - - -@pytest.fixture -def resource_with_upload(create_with_upload): - content = u""" - Snow Course Name, Number, Elev. metres, Date of Survey, Snow Depth cm,\ - Water Equiv. mm, Survey Code, % of Normal, Density %, Survey Period, \ - Normal mm - SKINS LAKE,1B05,890,2015/12/30,34,53,,98,16,JAN-01,54 - MCGILLIVRAY PASS,1C05,1725,2015/12/31,88,239,,87,27,JAN-01,274 - NAZKO,1C08,1070,2016/01/05,20,31,,76,16,JAN-01,41 - """ - resource = create_with_upload( - content, u'test.csv', - package_id=factories.Dataset()[u"id"] - ) - return resource - - -@pytest.fixture -def organization_with_image(create_with_upload): - user = factories.Sysadmin() - context = { - u"user": user["name"] - } - org = create_with_upload( - b"\0\0\0", u"image.png", - context=context, - action=u"organization_create", - upload_field_name=u"image_upload", - name=u"test-org" - ) - return org diff --git a/ckanext/s3filestore/tests/test_controller.py b/ckanext/s3filestore/tests/test_controller.py index 7924db3e..227dffb3 100644 --- a/ckanext/s3filestore/tests/test_controller.py +++ b/ckanext/s3filestore/tests/test_controller.py @@ -1,225 +1,279 @@ +# encoding: utf-8 + +import io +import logging import os +import requests +import six from nose.tools import (assert_equal, - assert_true, + assert_in, with_setup) -import requests - -import ckan.plugins.toolkit as toolkit -from ckantoolkit import config -import ckan.tests.helpers as helpers -import ckan.tests.factories as factories - -import ckanapi - -import logging -log = logging.getLogger(__name__) - +from werkzeug.datastructures import FileStorage as FlaskFileStorage -if toolkit.check_ckan_version('2.9'): +from ckan.plugins import toolkit +from ckan.plugins.toolkit import config +from ckan.lib.helpers import url_for +from ckan.tests import helpers +from ckan.tests import factories - def setup_function(self): - helpers.reset_db() +from ckanext.s3filestore import uploader - @with_setup(setup_function) - class TestS3ControllerResourceDownload(): +from . import _get_status_code, _get_response_body - def _upload_resource(self): - factories.Dataset(name="my-dataset") - - file_path = os.path.join(os.path.dirname(__file__), 'data.csv') - resource = helpers.call_action('resource_create', context={'ignore_auth': True}, package_id='my-dataset', - upload=open(file_path)) - return resource - - @helpers.change_config('ckan.site_url', 'http://mytest.ckan.net') - def test_resource_show_url(self): - '''The resource_show url is expected for uploaded resource file.''' - - assert_equal(config.get('ckan.site_url'), 'http://mytest.ckan.net') - resource = self._upload_resource() - - # does resource_show have the expected resource file url? - resource_show = helpers.call_action('resource_show', id=resource['id']) - - expected_url = 'http://mytest.ckan.net/dataset/{0}/resource/{1}/download/data.csv' \ - .format(resource['package_id'], resource['id']) +log = logging.getLogger(__name__) - assert_equal(resource_show['url'], expected_url) - def test_resource_download_s3(self): - '''A resource uploaded to S3 can be downloaded.''' +def setup_function(self): + self.sysadmin = factories.Sysadmin(apikey="my-test-key") - resource = self._upload_resource() - resource_show = helpers.call_action('resource_show', id=resource['id']) - resource_file_url = resource_show['url'] + assert_equal(config.get('ckanext.s3filestore.signature_version'), 's3v4') + self.bucket_name = config.get(u'ckanext.s3filestore.aws_bucket_name') + uploader.BaseS3Uploader().get_s3_bucket(self.bucket_name) - file_response = requests.get(resource_file_url) - location = file_response.headers['Location'] - log.info("ckanext.s3filestore.tests: response is: %s, %s", location, file_response) - assert_equal(file_response.status_int, 302) - file_response = requests.get(location) - if hasattr(file_response, 'content_type'): - content_type = file_response.content_type - else: - content_type = file_response.headers.get('Content-Type') - assert_equal(content_type, "text/csv") - if hasattr(file_response, 'text'): - body = file_response.text - else: - body = file_response.body - assert_true('date,price' in body) - - def test_resource_download_wrong_filename(self): - '''A resource downloaded with the wrong filename gives 404.''' - - resource, demo, app = self._upload_resource() - resource_file_url = '/dataset/{0}/resource/{1}/fs_download/foo.txt' \ - .format(resource['package_id'], resource['id']) - - file_response = app.get(resource_file_url, expect_errors=True) - log.info("ckanext.s3filestore.tests: response is: %s", file_response) - assert_equal(404, file_response.status_int) - - def test_resource_download_s3_no_filename(self): - '''A resource uploaded to S3 can be downloaded when no filename in - url.''' - - resource = self._upload_resource() - - resource_file_url = '/dataset/{0}/resource/{1}/download' \ - .format(resource['package_id'], resource['id']) - - file_response = requests.get(resource_file_url) - location = file_response.headers['Location'] - assert_equal(file_response.status_code, 302) - file_response = requests.get(location) - log.info("ckanext.s3filestore.tests: response is: {0}, {1}".format(location, file_response)) - if hasattr(file_response, 'text'): - body = file_response.text - else: - body = file_response.body - assert_true('date,price' in body) +def teardown_function(self): + helpers.reset_db() - def test_resource_download_url_link(self): - '''A resource with a url (not file) is redirected correctly.''' - dataset = factories.Dataset() - resource = helpers.call_action('resource_create', context={'ignore_auth': True}, package_id=dataset['id'], - url='http://example') - resource_show = helpers.call_action('resource_show', id=resource['id']) - resource_file_url = '/dataset/{0}/resource/{1}/download' \ - .format(resource['package_id'], resource['id']) - assert_equal(resource_show['url'], 'http://example') +def _test_org(): + try: + return helpers.call_action('organization_show', id='test-org') + except toolkit.ObjectNotFound: + user = factories.Sysadmin() + context = { + u"user": user["name"] + } + return helpers.call_action( + 'organization_create', + context=context, + name=u"test-org", + upload_field_name="image_upload", + image_upload=FlaskFileStorage(six.BytesIO(b"\0\0\0"), u"image.png")) - # attempt redirect to linked url - r = requests.get(resource_file_url) - assert_true(r.status_code in [302, 301]) - assert_equal(r.headers['Location'], 'http://example') -else: +@with_setup(setup_function, teardown_function) +class TestS3Controller(object): - class TestS3ControllerResourceDownload(helpers.FunctionalTestBase): + def _upload_resource(self): + dataset = factories.Dataset(name="my-dataset") - def _upload_resource(self): - factories.Sysadmin(apikey="my-test-key") + file_path = os.path.join(os.path.dirname(__file__), 'data.csv') + resource = helpers.call_action( + 'resource_create', + package_id=dataset['id'], + upload=FlaskFileStorage(io.open(file_path, 'rb'))) + return resource - app = self._get_test_app() - demo = ckanapi.TestAppCKAN(app, apikey='my-test-key') - factories.Dataset(name="my-dataset") + def test_resource_show_url(self): + '''The resource_show url is expected for uploaded resource file.''' - file_path = os.path.join(os.path.dirname(__file__), 'data.csv') - resource = demo.action.resource_create(package_id='my-dataset', - upload=open(file_path)) - return resource, demo, app + site_url = config.get('ckan.site_url') + resource = self._upload_resource() + assert_in('url', resource) - @helpers.change_config('ckan.site_url', 'http://mytest.ckan.net') - def test_resource_show_url(self): - '''The resource_show url is expected for uploaded resource file.''' + # does resource_show have the expected resource file url? + resource_show = helpers.call_action('resource_show', id=resource['id']) + assert_in('url', resource_show) - assert_equal(config.get('ckan.site_url'), 'http://mytest.ckan.net') - resource, demo, _ = self._upload_resource() + expected_url = site_url + '/dataset/{0}/resource/{1}/download/data.csv' \ + .format(resource['package_id'], resource['id']) - # does resource_show have the expected resource file url? - resource_show = demo.action.resource_show(id=resource['id']) + assert_equal(resource['url'], expected_url) + assert_equal(resource_show['url'], expected_url) - expected_url = 'http://mytest.ckan.net/dataset/{0}/resource/{1}/download/data.csv' \ - .format(resource['package_id'], resource['id']) + def test_resource_download_s3(self): + '''A resource uploaded to S3 can be downloaded.''' - assert_equal(resource_show['url'], expected_url) + resource = self._upload_resource() + resource_show = helpers.call_action('resource_show', id=resource['id']) + assert_in('url', resource_show) + location = resource_show['url'] - def test_resource_download_s3(self): - '''A resource uploaded to S3 can be downloaded.''' + status_code, location = self._get_expecting_redirect(location) + file_response = requests.get(location) + log.info("ckanext.s3filestore.tests: response is: %s, %s", location, file_response) - resource, demo, app = self._upload_resource() - resource_show = demo.action.resource_show(id=resource['id']) - resource_file_url = resource_show['url'] + if hasattr(file_response, 'content_type'): + content_type = file_response.content_type + else: + content_type = file_response.headers.get('Content-Type') + assert_equal(content_type, "text/csv") + assert_in('date,price', _get_response_body(file_response)) - file_response = app.get(resource_file_url) - location = file_response.headers['Location'] - log.info("ckanext.s3filestore.tests: response is: %s, %s", location, file_response) - assert_equal(302, file_response.status_int) - file_response = requests.get(location) - if hasattr(file_response, 'content_type'): - content_type = file_response.content_type - else: - content_type = file_response.headers.get('Content-Type') - assert_equal("text/csv", content_type) - if hasattr(file_response, 'text'): - body = file_response.text - else: - body = file_response.body - assert_true('date,price' in body) - - def test_resource_download_wrong_filename(self): - '''A resource downloaded with the wrong filename gives 404.''' - - resource, demo, app = self._upload_resource() - resource_file_url = '/dataset/{0}/resource/{1}/fs_download/foo.txt' \ - .format(resource['package_id'], resource['id']) - - file_response = app.get(resource_file_url, expect_errors=True) - log.info("ckanext.s3filestore.tests: response is: %s", file_response) - assert_equal(404, file_response.status_int) - - def test_resource_download_s3_no_filename(self): - '''A resource uploaded to S3 can be downloaded when no filename in - url.''' - - resource, demo, app = self._upload_resource() - - resource_file_url = '/dataset/{0}/resource/{1}/download' \ - .format(resource['package_id'], resource['id']) - - file_response = app.get(resource_file_url) - location = file_response.headers['Location'] - assert_equal(302, file_response.status_int) + def test_resource_download_wrong_filename(self): + '''A resource downloaded with the wrong filename gives 404.''' + + resource = self._upload_resource() + resource_file_url = '/dataset/{0}/resource/{1}/fs_download/foo.txt' \ + .format(resource['package_id'], resource['id']) + + app = helpers._get_test_app() + file_response = app.get(resource_file_url, expect_errors=True) + log.info("ckanext.s3filestore.tests: response is: %s", file_response) + assert_equal(_get_status_code(file_response), 404) + + def test_resource_download_s3_no_filename(self): + '''A resource uploaded to S3 can be downloaded when no filename in + url.''' + + resource = self._upload_resource() + + location = '/dataset/{0}/resource/{1}/download' \ + .format(resource['package_id'], resource['id']) + + status_code, location = self._get_expecting_redirect(location) + file_response = requests.get(location) + log.info("ckanext.s3filestore.tests: response is: {0}, {1}".format(location, file_response)) + + assert_in('date,price', _get_response_body(file_response)) + + def test_resource_download_url_link(self): + '''A resource with a url (not file) is redirected correctly.''' + dataset = factories.Dataset() + + resource = helpers.call_action( + 'resource_create', + package_id=dataset['id'], + url='http://example') + resource_show = helpers.call_action('resource_show', id=resource['id']) + assert_equal(resource_show['url'], 'http://example') + + resource_file_url = '/dataset/{0}/resource/{1}/download' \ + .format(resource['package_id'], resource['id']) + # attempt redirect to linked url + status_code, location = self._get_expecting_redirect(resource_file_url) + assert_equal(location, 'http://example') + + def test_resource_download_url(self): + u'''The resource url is expected for uploaded resource file.''' + resource_with_upload = self._upload_resource() + + site_url = config.get('ckan.site_url') + expected_url = site_url + u'/dataset/{0}/resource/{1}/download/data.csv'.\ + format(resource_with_upload[u'package_id'], + resource_with_upload[u'id']) + + assert resource_with_upload['url'] == expected_url + + def test_resource_download_no_filename(self): + '''A resource uploaded to S3 can be downloaded + when no filename in url.''' + resource_with_upload = self._upload_resource() + + resource_file_url = u'/dataset/{0}/resource/{1}/download' \ + .format(resource_with_upload[u'package_id'], + resource_with_upload[u'id']) + + status_code, location = self._get_expecting_redirect(resource_file_url) + + assert 302 == status_code + + def test_s3_resource_mimetype(self): + u'''A resource mimetype test.''' + resource_with_upload = self._upload_resource() + + assert u'text/csv' == resource_with_upload[u'mimetype'] + + def test_organization_image_redirects_to_s3(self): + organization_with_image = _test_org() + url = u'/uploads/group/{0}'\ + .format(organization_with_image[u'image_url']) + status_code, location = self._get_expecting_redirect(url) + assert 302 == status_code + + def test_organization_image_download_from_s3(self): + organization_with_image = _test_org() + url = u'/uploads/group/{0}'\ + .format(organization_with_image[u'image_url']) + status_code, location = self._get_expecting_redirect(url) + assert 302 == status_code + assert location + image = requests.get(location) + assert image.content == b"\0\0\0" + + if toolkit.check_ckan_version('2.9'): + + def _get_expecting_redirect(self, url, app=None): + if url.startswith('http:') or url.startswith('https:'): + site_url = config.get('ckan.site_url') + url = url.replace(site_url, '') + if not app: + app = helpers._get_test_app() + response = app.get(url, follow_redirects=False) + status_code = _get_status_code(response) + assert_in(status_code, [301, 302], + "%s resulted in %s instead of a redirect" % (url, response.status)) + return status_code, response.location + + def test_resource_download(self): + u'''When trying to download resource + from CKAN it should redirect to S3.''' + resource_with_upload = self._upload_resource() + + status_code, location = self._get_expecting_redirect( + url_for( + u'dataset_resource.download', + id=resource_with_upload[u'package_id'], + resource_id=resource_with_upload[u'id'], + ) + ) + assert 302 == status_code + + def test_resource_download_not_found(self): + u'''Downloading a nonexistent resource gives HTTP 404.''' + + app = helpers._get_test_app() + response = app.get( + url_for( + u'dataset_resource.download', + id=u'package_id', + resource_id=u'resource_id', + ) + ) + assert 404 == _get_status_code(response) + + def test_s3_download_link(self): + u'''A resource download from s3 test.''' + resource_with_upload = self._upload_resource() + + status_code, location = self._get_expecting_redirect( + url_for( + u'dataset_resource.download', + id=resource_with_upload[u'package_id'], + resource_id=resource_with_upload[u'id'], + ) + ) file_response = requests.get(location) - log.info("ckanext.s3filestore.tests: response is: {0}, {1}".format(location, file_response)) - - if hasattr(file_response, 'text'): - body = file_response.text - else: - body = file_response.body - assert_true('date,price' in body) - - def test_resource_download_url_link(self): - '''A resource with a url (not file) is redirected correctly.''' - factories.Sysadmin(apikey="my-test-key") - - app = self._get_test_app() - demo = ckanapi.TestAppCKAN(app, apikey='my-test-key') - dataset = factories.Dataset() - - resource = demo.action.resource_create(package_id=dataset['id'], - url='http://example') - resource_show = demo.action.resource_show(id=resource['id']) - resource_file_url = '/dataset/{0}/resource/{1}/download' \ - .format(resource['package_id'], resource['id']) - assert_equal(resource_show['url'], 'http://example') - - # attempt redirect to linked url - r = app.get(resource_file_url, status=[302, 301]) - assert_equal(r.location, 'http://example') + assert 'date,price' in _get_response_body(file_response) + + else: + + def _get_expecting_redirect(self, url, app=None): + if url.startswith('http:') or url.startswith('https:'): + site_url = config.get('ckan.site_url') + url = url.replace(site_url, '') + if not app: + app = helpers._get_test_app() + response = app.get(url) + status_code = _get_status_code(response) + assert_in(status_code, [301, 302], + "%s resulted in %s instead of a redirect" % (url, response.status)) + return status_code, response.headers['Location'] + + def test_resource_upload_with_url_and_clear(self): + '''Test that clearing an upload and using a URL does not crash''' + dataset = factories.Dataset(name="my-dataset") + + url = toolkit.url_for(controller='package', action='new_resource', + id=dataset['id']) + env = {'REMOTE_USER': self.sysadmin['name'].encode('ascii')} + + helpers._get_test_app().post( + url, + {'clear_upload': True, + 'id': '', # Empty id from the form + 'url': 'http://asdf', 'save': 'save'}, + headers={'Authorization': 'my-test-key'}, + extra_environ=env) diff --git a/ckanext/s3filestore/tests/test_uploader.py b/ckanext/s3filestore/tests/test_uploader.py index 76256836..cd858155 100644 --- a/ckanext/s3filestore/tests/test_uploader.py +++ b/ckanext/s3filestore/tests/test_uploader.py @@ -1,5 +1,6 @@ # encoding: utf-8 import datetime +import io import os import mock @@ -9,21 +10,19 @@ assert_in, with_setup) -import ckanapi -from ckantoolkit import config from botocore.exceptions import ClientError from werkzeug.datastructures import FileStorage as FlaskFileStorage -import ckantoolkit as toolkit +from ckan.plugins import toolkit +from ckan.plugins.toolkit import config from ckan.tests import helpers import ckan.tests.factories as factories -from ckanext.s3filestore.uploader import (S3Uploader, - S3ResourceUploader, - _is_presigned_url) +from ckanext.s3filestore.uploader import ( + BaseS3Uploader, S3Uploader, S3ResourceUploader, _is_presigned_url) -from . import BUCKET_NAME, endpoint_url, s3 +from . import _get_status_code DIRECT_DOWNLOAD_URL_FORMAT = '/dataset/{0}/resource/{1}/orig_download/{2}' @@ -34,11 +33,16 @@ def _setup_function(self): self.app = helpers._get_test_app() self.sysadmin = factories.Sysadmin(apikey="my-test-key") self.organisation = factories.Organization(name='my-organisation') + self.endpoint_url = config.get('ckanext.s3filestore.host_name') + uploader = BaseS3Uploader() + self.s3 = uploader.get_s3_client() + # ensure the bucket exists, create if needed + self.bucket_name = config.get('ckanext.s3filestore.aws_bucket_name') + uploader.get_s3_bucket(self.bucket_name) def _resource_setup_function(self): _setup_function(self) - self.demo = ckanapi.TestAppCKAN(self.app, apikey='my-test-key') def _get_object_key(resource): @@ -55,7 +59,7 @@ class TestS3Uploader(): def test_get_bucket(self): '''S3Uploader retrieves bucket as expected''' uploader = S3Uploader('') - assert_true(uploader.get_s3_bucket(BUCKET_NAME)) + assert_true(uploader.get_s3_bucket(self.bucket_name)) def test_clean_dict(self): '''S3Uploader retrieves bucket as expected''' @@ -75,7 +79,7 @@ def test_group_image_upload(self): file_path = os.path.join(os.path.dirname(__file__), 'data.csv') file_name = 'somename.png' - img_uploader = FlaskFileStorage(filename=file_name, stream=open(file_path), content_type='image/png') + img_uploader = FlaskFileStorage(filename=file_name, stream=io.open(file_path, 'rb'), content_type='image/png') with mock.patch('ckanext.s3filestore.uploader.datetime') as mock_date: mock_date.datetime.utcnow.return_value = \ @@ -92,15 +96,15 @@ def test_group_image_upload(self): # check whether the object exists in S3 # will throw exception if not existing - s3.head_object(Bucket=BUCKET_NAME, Key=key) + self.s3.head_object(Bucket=self.bucket_name, Key=key) # requesting image redirects to s3 # attempt redirect to linked url image_file_url = '/uploads/group/2001-01-29-000000{0}'.format(file_name) - r = self.app.get(image_file_url, status=[302, 301]) - assert_equal(r.location.split('?')[0], + status_code, location = self._get_expecting_redirect(self.app, image_file_url) + assert_equal(location.split('?')[0], '{0}/my-bucket/my-path/storage/uploads/group/2001-01-29-000000{1}' - .format(endpoint_url, file_name)) + .format(self.endpoint_url, file_name)) def test_group_image_upload_then_clear(self): '''Test that clearing an upload removes the S3 key''' @@ -108,7 +112,7 @@ def test_group_image_upload_then_clear(self): file_path = os.path.join(os.path.dirname(__file__), 'data.csv') file_name = "somename.png" - img_uploader = FlaskFileStorage(filename=file_name, stream=open(file_path), content_type='image/png') + img_uploader = FlaskFileStorage(filename=file_name, stream=io.open(file_path, 'rb'), content_type='image/png') with mock.patch('ckanext.s3filestore.uploader.datetime') as mock_date: mock_date.datetime.utcnow.return_value = \ @@ -124,7 +128,7 @@ def test_group_image_upload_then_clear(self): # check whether the object exists in S3 # will throw exception if not existing - s3.head_object(Bucket=BUCKET_NAME, Key=key) + self.s3.head_object(Bucket=self.bucket_name, Key=key) # clear upload helpers.call_action('group_update', context=context, @@ -133,13 +137,31 @@ def test_group_image_upload_then_clear(self): # key shouldn't exist try: - s3.head_object(Bucket=BUCKET_NAME, Key=key) + self.s3.head_object(Bucket=self.bucket_name, Key=key) # broken by https://github.com/ckan/ckan/commit/48afb9da4d # assert_false(True, "file '{}' should not exist".format(key)) except ClientError: # passed assert_true(True, "passed") + if toolkit.check_ckan_version('2.9'): + + def _get_expecting_redirect(self, app, url): + response = app.get(url, follow_redirects=False) + status_code = _get_status_code(response) + assert_in(status_code, [301, 302], + "%s resulted in %s instead of a redirect" % (url, status_code)) + return status_code, response.location + + else: + + def _get_expecting_redirect(self, app, url): + response = app.get(url) + status_code = _get_status_code(response) + assert_in(status_code, [301, 302], + "%s resulted in %s instead of a redirect" % (url, status_code)) + return status_code, response.headers['Location'] + @with_setup(_resource_setup_function) class TestS3ResourceUploader(): @@ -161,8 +183,11 @@ def _upload_test_resource(self, dataset=None): if not dataset: dataset = self._test_dataset() file_path = os.path.join(os.path.dirname(__file__), 'data.csv') - return self.demo.action.resource_create( - package_id=dataset['id'], upload=open(file_path), url='file.txt') + return helpers.call_action( + 'resource_create', + package_id=dataset['id'], + upload=FlaskFileStorage(io.open(file_path, 'rb')), + url='file.txt') def test_resource_upload(self): '''Test a basic resource file upload''' @@ -173,41 +198,12 @@ def test_resource_upload(self): # check whether the object exists in S3 # will throw exception if not existing - s3.head_object(Bucket=BUCKET_NAME, Key=key) + self.s3.head_object(Bucket=self.bucket_name, Key=key) # test the file contains what's expected - obj = s3.get_object(Bucket=BUCKET_NAME, Key=key) + obj = self.s3.get_object(Bucket=self.bucket_name, Key=key) data = obj['Body'].read() - assert_equal(data, open(file_path).read()) - - def test_resource_upload_then_clear(self): - '''Test that clearing an upload removes the S3 key''' - - dataset = self._test_dataset() - resource = self._upload_test_resource(dataset) - key = _get_object_key(resource) - - # check whether the object exists in S3 - # will throw exception if not existing - s3.head_object(Bucket=BUCKET_NAME, Key=key) - - # clear upload - url = toolkit.url_for(controller='package', action='resource_edit', - id=dataset['id'], resource_id=resource['id']) - env = {'REMOTE_USER': self.sysadmin['name'].encode('ascii')} - self.app.post( - url, - {'clear_upload': True, - 'url': 'http://asdf', 'save': 'save'}, - extra_environ=env) - - # key shouldn't exist - try: - s3.head_object(Bucket=BUCKET_NAME, Key=key) - assert_false(True, "file should not exist") - except ClientError: - # passed - assert_true(True, "passed") + assert_equal(data, io.open(file_path, 'rb').read()) def test_uploader_get_path(self): '''Uploader get_path returns as expected''' @@ -219,22 +215,6 @@ def test_uploader_get_path(self): assert_equal(returned_path, 'my-path/resources/{0}/myfile.txt'.format(resource['id'])) - def test_resource_upload_with_url_and_clear(self): - '''Test that clearing an upload and using a URL does not crash''' - - dataset = factories.Dataset(name="my-dataset") - - url = toolkit.url_for(controller='package', action='new_resource', - id=dataset['id']) - env = {'REMOTE_USER': self.sysadmin['name'].encode('ascii')} - - self.app.post( - url, - {'clear_upload': True, - 'id': '', # Empty id from the form - 'url': 'http://asdf', 'save': 'save'}, - extra_environ=env) - def test_is_presigned_url(self): ''' Tests that presigned URLs are correctly recognised.''' assert_true(_is_presigned_url('https://example.s3.amazonaws.com/resources/foo?AWSAccessKeyId=SomeKey&Expires=9999999999Signature=hb7%2F%2Bz1H%2B8wdEy0pCsX7bZG%2BuPU%3D')) @@ -331,3 +311,34 @@ def test_encoding_object_metadata_headers(self): object_metadata = uploader._get_resource_metadata() assert_equal(object_metadata['package_title'], 'Test Dataset—with em dash') assert_equal(object_metadata['package_author'], '擬製 暗影') + + if toolkit.check_ckan_version(max_version='2.8.99'): + + def test_resource_upload_then_clear(self): + '''Test that clearing an upload removes the S3 key''' + + dataset = self._test_dataset() + resource = self._upload_test_resource(dataset) + key = _get_object_key(resource) + + # check whether the object exists in S3 + # will throw exception if not existing + self.s3.head_object(Bucket=self.bucket_name, Key=key) + + # clear upload + url = toolkit.url_for(controller='package', action='resource_edit', + id=dataset['id'], resource_id=resource['id']) + env = {'REMOTE_USER': self.sysadmin['name'].encode('ascii')} + self.app.post( + url, + {'clear_upload': True, + 'url': 'http://asdf', 'save': 'save'}, + extra_environ=env) + + # key shouldn't exist + try: + self.s3.head_object(Bucket=self.bucket_name, Key=key) + assert_false(True, "file should not exist") + except ClientError: + # passed + assert_true(True, "passed") diff --git a/ckanext/s3filestore/uploader.py b/ckanext/s3filestore/uploader.py index bf232389..bedf6460 100644 --- a/ckanext/s3filestore/uploader.py +++ b/ckanext/s3filestore/uploader.py @@ -150,25 +150,32 @@ def get_directory(self, id, storage_path): directory = os.path.join(storage_path, id) return directory - def get_s3_resource(self): - return get_s3_session(config).resource('s3', - endpoint_url=self.host_name, - config=Config( - signature_version=self.signature, - s3={'addressing_style': self.addressing_style} - )) - - def get_s3_client(self): - return get_s3_session(config).client('s3', - endpoint_url=self.host_name, - config=Config( - signature_version=self.signature, - s3={'addressing_style': self.addressing_style} - )) - - def get_s3_bucket(self, bucket_name): + def _get_s3_config(self): + return Config( + signature_version=self.signature, + s3={'addressing_style': self.addressing_style} + ) + + def get_s3_resource(self, session=None): + if not session: + session = get_s3_session(config) + return session.resource('s3', + endpoint_url=self.host_name, + config=self._get_s3_config()) + + def get_s3_client(self, session=None): + if not session: + session = get_s3_session(config) + return session.client('s3', + endpoint_url=self.host_name, + config=self._get_s3_config()) + + def get_s3_bucket(self, bucket_name=None): '''Return a boto bucket, creating it if it doesn't exist.''' + if not bucket_name: + bucket_name = self.bucket_name + # make s3 connection using boto3 s3 = self.get_s3_resource() @@ -204,7 +211,7 @@ def upload_to_key(self, filepath, upload_file, acl, extra_metadata=None): upload_file.seek(0) mime_type = getattr(self, 'mimetype', '') or 'application/octet-stream' log.debug( - "ckanext.s3filestore.uploader: going to upload [%s] to bucket [%s]" + "ckanext.s3filestore.uploader: going to upload [%s] to bucket [%s] " "with access [%s] and mimetype [%s]", filepath, self.bucket_name, acl, mime_type) diff --git a/ckanext/s3filestore/views/__init__.py b/ckanext/s3filestore/views/__init__.py index e69de29b..7922bbb7 100644 --- a/ckanext/s3filestore/views/__init__.py +++ b/ckanext/s3filestore/views/__init__.py @@ -0,0 +1,116 @@ +# encoding: utf-8 + +import logging +import os + +from botocore.exceptions import ClientError + +from ckan import model +from ckan.lib import uploader +from ckan.lib.uploader import ResourceUpload as DefaultResourceUpload +from ckan.plugins.toolkit import abort, config, _, g, get_action,\ + NotAuthorized, ObjectNotFound, url_for, redirect_to + +from ckanext.s3filestore.uploader import S3Uploader, BaseS3Uploader + +log = logging.getLogger(__name__) + + +def resource_download(id, resource_id, filename=None): + ''' + Provide a download by either redirecting the user to the url stored or + downloading the uploaded file from S3. + ''' + context = {'model': model, 'session': model.Session, + 'user': g.user, 'auth_user_obj': g.userobj} + + try: + rsc = get_action('resource_show')(context, {'id': resource_id}) + except ObjectNotFound: + return abort(404, _('Resource not found')) + except NotAuthorized: + return abort(401, _('Unauthorized to read resource %s') % id) + + if 'url' not in rsc: + return abort(404, _('No download is available')) + elif rsc.get('url_type') == 'upload': + upload = uploader.get_resource_uploader(rsc) + + if filename is None: + filename = os.path.basename(rsc['url']) + key_path = upload.get_path(rsc['id'], filename) + + if filename is None: + log.warn("Key '%s' not found in bucket '%s'", + key_path, upload.bucket_name) + + try: + url = upload.get_signed_url_to_key(key_path) + return redirect_to(url) + except ClientError as ex: + if ex.response['Error']['Code'] in ['NoSuchKey', '404']: + # attempt fallback + if config.get( + 'ckanext.s3filestore.filesystem_download_fallback', + False): + log.info('Attempting filesystem fallback for resource %s', + resource_id) + url = url_for( + u's3_resource.filesystem_resource_download', + id=id, + resource_id=resource_id, + filename=filename) + return redirect_to(url) + + return abort(404, _('Resource data not found')) + else: + raise ex + # if we're trying to download a link resource, just redirect to it + return redirect_to(rsc['url']) + + +def filesystem_resource_download(id, resource_id, filename=None): + """ + Provide a direct download by either redirecting the user to the url + stored or downloading an uploaded file directly. + """ + if hasattr(DefaultResourceUpload, 'download'): + context = {'model': model, 'session': model.Session, + 'user': g.user, 'auth_user_obj': g.userobj} + + try: + rsc = get_action('resource_show')(context, {'id': resource_id}) + except ObjectNotFound: + return abort(404, _('Resource not found')) + except NotAuthorized: + return abort(401, _('Unauthorised to read resource %s') % resource_id) + upload = DefaultResourceUpload(rsc) + return upload.download(rsc['id'], filename) + else: + try: + from ckan.views.resource import download + return download('dataset', id, resource_id, filename) + except ImportError: + # pre-Flask + from ckan.controllers.package import PackageController + return PackageController().resource_download(id, resource_id, filename) + except (IOError, OSError): + # probably file not found + return abort(404, _('Resource data not found')) + + +def uploaded_file_redirect(upload_to, filename): + '''Redirect static file requests to their location on S3.''' + storage_path = S3Uploader.get_storage_path(upload_to) + filepath = os.path.join(storage_path, filename) + base_uploader = BaseS3Uploader() + + try: + url = base_uploader.get_signed_url_to_key(filepath) + except ClientError as ex: + if ex.response['Error']['Code'] in ['NoSuchKey', '404']: + return abort(404, _('Keys not found on S3')) + else: + raise ex + + return redirect_to(url) diff --git a/ckanext/s3filestore/views/resource.py b/ckanext/s3filestore/views/resource.py index 4d411dd3..ab9af3d9 100644 --- a/ckanext/s3filestore/views/resource.py +++ b/ckanext/s3filestore/views/resource.py @@ -1,163 +1,36 @@ # encoding: utf-8 -import os + import logging -import mimetypes import flask -from botocore.exceptions import ClientError - -from ckantoolkit import config as ckan_config -from ckantoolkit import _, request, c, g -import ckantoolkit as toolkit -import ckan.logic as logic -import ckan.lib.base as base -import ckan.lib.uploader as uploader -from ckan.lib.uploader import get_storage_path, ResourceUpload as DefaultResourceUpload +from ckan.plugins.toolkit import config -import ckan.model as model +from . import resource_download, filesystem_resource_download log = logging.getLogger(__name__) Blueprint = flask.Blueprint -NotFound = logic.NotFound -NotAuthorized = logic.NotAuthorized -get_action = logic.get_action -abort = base.abort -redirect = toolkit.redirect_to - s3_resource = Blueprint( u's3_resource', __name__, - url_prefix=u'/dataset//resource', - url_defaults={u'package_type': u'dataset'} + url_prefix=u'/dataset//resource' ) -def resource_download(package_type, id, resource_id, filename=None): - ''' - Provide a download by either redirecting the user to the url stored or - downloading the uploaded file from S3. - ''' - context = {'model': model, 'session': model.Session, - 'user': c.user or c.author, 'auth_user_obj': c.userobj} - - try: - rsc = get_action('resource_show')(context, {'id': resource_id}) - get_action('package_show')(context, {'id': id}) - except NotFound: - return abort(404, _('Resource not found')) - except NotAuthorized: - return abort(401, _('Unauthorized to read resource %s') % id) - - if 'url' not in rsc: - return abort(404, _('No download is available')) - elif rsc.get('url_type') == 'upload': - upload = uploader.get_resource_uploader(rsc) - bucket_name = ckan_config.get('ckanext.s3filestore.aws_bucket_name') - - if filename is None: - filename = os.path.basename(rsc['url']) - key_path = upload.get_path(rsc['id'], filename) - - if filename is None: - log.warn("Key '%s' not found in bucket '%s'", - key_path, bucket_name) - - try: - url = upload.get_signed_url_to_key(key_path) - return redirect(url) - except ClientError as ex: - if ex.response['Error']['Code'] in ['NoSuchKey', '404']: - # attempt fallback - if ckan_config.get( - 'ckanext.s3filestore.filesystem_download_fallback', - False): - log.info('Attempting filesystem fallback for resource %s', - resource_id) - url = toolkit.url_for( - u's3_resource.filesystem_resource_download', - id=id, - resource_id=resource_id, - filename=filename) - return redirect(url) - - return abort(404, _('Resource data not found')) - else: - raise ex - # if we're trying to download a link resource, just redirect to it - return redirect(rsc['url']) - - -def filesystem_resource_download(package_type, id, resource_id, filename=None): - """ - A fallback view action to download resources from the - filesystem. A copy of the action from - `ckan.views.resource:download`. +s3_resource.add_url_rule(u'//download', + view_func=resource_download) +s3_resource.add_url_rule(u'//download/', + view_func=resource_download) +s3_resource.add_url_rule(u'//fs_download/', + view_func=filesystem_resource_download) - Provide a direct download by either redirecting the user to the url - stored or downloading an uploaded file directly. - """ - context = { - u'model': model, - u'session': model.Session, - u'user': g.user, - u'auth_user_obj': g.userobj - } - preview = request.args.get(u'preview', False) - - try: - rsc = get_action(u'resource_show')(context, {u'id': resource_id}) - get_action(u'package_show')(context, {u'id': id}) - except NotFound: - return abort(404, _(u'Resource not found')) - except NotAuthorized: - return abort(401, _('Unauthorised to read resource %s') % resource_id) - - mimetype, enc = mimetypes.guess_type(rsc.get('url', '')) - - if rsc.get(u'url_type') == u'upload': - try: - if hasattr(DefaultResourceUpload, 'download'): - upload = DefaultResourceUpload(rsc) - return upload.download(rsc['id'], filename) - - path = get_storage_path() - storage_path = os.path.join(path, 'resources') - directory = os.path.join(storage_path, - resource_id[0:3], resource_id[3:6]) - filepath = os.path.join(directory, resource_id[6:]) - if preview: - return flask.send_file(filepath, mimetype=mimetype) - else: - return flask.send_file(filepath) - except (OSError, IOError): - # includes FileNotFoundError - return abort(404, _('Resource data not found')) - except Exception as e: - log.warning("Unhandled exception %s of type %s", e, type(e)) - raise e - elif u'url' not in rsc: - return abort(404, _(u'No download is available')) - return redirect(rsc[u'url']) - - -if not hasattr(DefaultResourceUpload, "download"): - s3_resource.add_url_rule(u'//download', view_func=resource_download) - s3_resource.add_url_rule( - u'//download/', view_func=resource_download - ) - -if not ckan_config.get('ckanext.s3filestore.use_filename', False): +if not config.get('ckanext.s3filestore.use_filename', False): s3_resource.add_url_rule( u'//orig_download/', view_func=resource_download ) -s3_resource.add_url_rule( - u'//fs_download/', view_func=filesystem_resource_download -) - def get_blueprints(): return [s3_resource] diff --git a/ckanext/s3filestore/views/uploads.py b/ckanext/s3filestore/views/uploads.py index 7c19cba2..537db582 100644 --- a/ckanext/s3filestore/views/uploads.py +++ b/ckanext/s3filestore/views/uploads.py @@ -1,21 +1,10 @@ # encoding: utf-8 -import os -import logging import flask -from botocore.exceptions import ClientError - -import ckantoolkit as toolkit -from ckantoolkit import _ -import ckan.lib.base as base - -from ckanext.s3filestore.uploader import S3Uploader, BaseS3Uploader +from . import uploaded_file_redirect Blueprint = flask.Blueprint -redirect = toolkit.redirect_to -abort = base.abort -log = logging.getLogger(__name__) s3_uploads = Blueprint( u's3_uploads', @@ -23,24 +12,6 @@ ) -def uploaded_file_redirect(upload_to, filename): - '''Redirect static file requests to their location on S3.''' - - storage_path = S3Uploader.get_storage_path(upload_to) - filepath = os.path.join(storage_path, filename) - base_uploader = BaseS3Uploader() - - try: - url = base_uploader.get_signed_url_to_key(filepath) - except ClientError as ex: - if ex.response['Error']['Code'] in ['NoSuchKey', '404']: - return abort(404, _('Keys not found on S3')) - else: - raise ex - - return redirect(url) - - s3_uploads.add_url_rule(u'/uploads//', view_func=uploaded_file_redirect) diff --git a/conftest.py b/conftest.py deleted file mode 100644 index 10cba657..00000000 --- a/conftest.py +++ /dev/null @@ -1,7 +0,0 @@ -# -*- coding: utf-8 -*- - -pytest_plugins = [ - u'ckanext.s3filestore.tests.fixtures', - u'ckan.tests.pytest_ckan.ckan_setup', - u'ckan.tests.pytest_ckan.fixtures', -] From 37ff5085c8d8b59fd2733a1faf0e2538e703217e Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 19 Jan 2022 16:06:22 +1000 Subject: [PATCH 06/11] [QOL-8518] clean up documentation --- .coveragerc | 5 ++++- .gitignore | 3 ++- README.rst | 26 ++++++++++++++++---------- ckanext/__init__.py | 2 ++ test.ini | 7 ++++++- 5 files changed, 30 insertions(+), 13 deletions(-) diff --git a/.coveragerc b/.coveragerc index 3138593c..7fec6e6a 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,5 +1,8 @@ +[run] +relative_files = True + [report] omit = */site-packages/* */python?.?/* - ckan/* \ No newline at end of file + ckan/* diff --git a/.gitignore b/.gitignore index 12f1f3c0..6bb23e29 100644 --- a/.gitignore +++ b/.gitignore @@ -38,8 +38,9 @@ htmlcov/ nosetests.xml coverage.xml +.idea + # Sphinx documentation docs/_build/ subdir/ -*.idea diff --git a/README.rst b/README.rst index 880e0815..b7e22533 100644 --- a/README.rst +++ b/README.rst @@ -1,10 +1,5 @@ -.. You should enable this project on travis-ci.org and coveralls.io to make - these badges work. The necessary Travis and Coverage config files have been - generated for you. - -.. image:: https://travis-ci.org/okfn/ckanext-s3filestore.svg?branch=master - :target: https://travis-ci.org/okfn/ckanext-s3filestore - +.. You should enable this project on coveralls.io to make these badges + work. The necessary Coverage config file has been generated for you. .. image:: https://coveralls.io/repos/okfn/ckanext-s3filestore/badge.svg :target: https://coveralls.io/r/okfn/ckanext-s3filestore @@ -107,10 +102,11 @@ Optional:: # To use user provided filepath and not use internal url basename # on download. This affects behaviour when using a URL that points # to an earlier version of a resource, with a different file name. - # If True, the older file will be served; if False, the newest file - # will be served, but the older file will be available at + # If True, the older file will be served. + # If False, the newest file will be served (similarly to the default + # CKAN filestore behaviour), but the older file will be available at # /dataset/{id}/resource/{resource_id}/orig_download/{filename} - # Set to False to be backwards compatible to ckan file store. + # Defaults to False. ckanext.s3filestore.use_filename = True # To mask the S3 endpoint with your own domain/endpoint when serving URLs to end users. @@ -131,6 +127,16 @@ Optional:: # Queue used by s3 plugin, if not set, default queue is used ckanext.s3filestore.queue = bulk + +----------------- +CLI +----------------- + +To upload all local resources located in `ckan.storage_path` location dir to the configured S3 bucket use:: + + ckan -c /etc/ckan/default/production.ini s3 upload all + + ------------------------ Development Installation ------------------------ diff --git a/ckanext/__init__.py b/ckanext/__init__.py index 2e2033b3..ed48ed01 100644 --- a/ckanext/__init__.py +++ b/ckanext/__init__.py @@ -1,3 +1,5 @@ +# encoding: utf-8 + # this is a namespace package try: import pkg_resources diff --git a/test.ini b/test.ini index 99809776..79ec5ee8 100644 --- a/test.ini +++ b/test.ini @@ -39,7 +39,7 @@ ckanext.s3filestore.acl.async_update = False # Logging configuration [loggers] -keys = root, ckan, sqlalchemy +keys = root, ckan, ckanext, sqlalchemy [handlers] keys = console @@ -56,6 +56,11 @@ qualname = ckan handlers = level = INFO +[logger_ckanext] +qualname = ckanext +handlers = +level = DEBUG + [logger_sqlalchemy] handlers = qualname = sqlalchemy.engine From 7f4fdc604aa3a0f346ffa34b34304cfdd7bf1513 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 19 Jan 2022 16:07:25 +1000 Subject: [PATCH 07/11] [QOL-8518] clean up continuous integration workflow - rename to better reflect its purpose - unify setup steps using ckan_cli script --- .github/workflows/ci.yml | 96 ++++++++++++++++++++++++++++++++++++++ .github/workflows/test.yml | 78 ------------------------------- scripts/ckan_cli | 75 +++++++++++++++++++++++++++++ 3 files changed, 171 insertions(+), 78 deletions(-) create mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/test.yml create mode 100644 scripts/ckan_cli diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..043d746d --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,96 @@ +name: CI + +on: + push: + pull_request: + branches: master + +jobs: + code_quality: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: '3.9' + + - name: Install requirements + run: | + python -m pip install --upgrade pip + pip install flake8 pycodestyle + + - name: Check syntax + run: | + flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan + + test: + runs-on: ubuntu-latest + needs: code_quality + strategy: + matrix: + ckan-version: ['2.9', '2.9-py2', '2.8', '2.7'] + fail-fast: false + + name: CKAN ${{ matrix.ckan-version }} + container: + image: openknowledge/ckan-dev:${{ matrix.ckan-version }} + services: + postgresql: + image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }} + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + + redis: + image: redis + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 6379:6379 + + moto: + image: motoserver/moto + ports: + - "5000" + + solr: + image: ckan/ckan-solr-dev:${{ matrix.ckan-version }} + + env: + CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgresql/ckan_test + CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgresql/datastore_test + CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgresql/datastore_test + CKAN_SOLR_URL: http://solr:8983/solr/ckan + CKAN_REDIS_URL: redis://redis:6379/1 + + steps: + - uses: actions/checkout@v2 + + - name: Install requirements + run: | + pip install -r requirements.txt + pip install -r dev-requirements.txt + pip install -e . + # Replace default path to CKAN core config file with the one on the container + sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini + + - name: Setup CKAN + run: | + chmod u+x ./scripts/ckan_cli + CKAN_INI=test.ini ./scripts/ckan_cli db init + + - name: Run tests + run: pytest --ckan-ini=test.ini --cov=ckanext.s3filestore diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index f5f7c774..00000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,78 +0,0 @@ -name: Tests -on: - push: - pull_request: - branches: - - master - -jobs: - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: '2.7' - - name: Install requirements - run: pip install flake8 pycodestyle - - name: Check syntax - run: flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan - - test: - needs: lint - strategy: - matrix: - ckan-version: ['2.9', '2.9-py2', '2.8', '2.7'] - fail-fast: false - - name: CKAN ${{ matrix.ckan-version }} - runs-on: ubuntu-latest - container: - image: openknowledge/ckan-dev:${{ matrix.ckan-version }} - services: - solr: - image: ckan/ckan-solr-dev:${{ matrix.ckan-version }} - postgres: - image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }} - env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: postgres - options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 - redis: - image: redis:3 - moto: - image: motoserver/moto - ports: - - "5000" - env: - CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test - CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test - CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test - CKAN_SOLR_URL: http://solr:8983/solr/ckan - CKAN_REDIS_URL: redis://redis:6379/1 - - steps: - - uses: actions/checkout@v2 - - name: Install requirements - run: | - pip install -r requirements.txt - pip install -r dev-requirements.txt - pip install -e . - # Replace default path to CKAN core config file with the one on the container - sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini - - name: Setup extension (CKAN >= 2.9) - if: ${{ matrix.ckan-version != '2.7' && matrix.ckan-version != '2.8' }} - run: | - ckan -c test.ini db init - - name: Setup extension (CKAN < 2.9) - if: ${{ matrix.ckan-version == '2.7' || matrix.ckan-version == '2.8' }} - run: | - paster --plugin=ckan db init -c test.ini - - name: Run tests (CKAN >= 2.9) - if: ${{ matrix.ckan-version != '2.7' && matrix.ckan-version != '2.8' }} - continue-on-error: true - run: pytest --ckan-ini=test.ini --cov=ckanext.s3filestore - - name: Run tests (CKAN < 2.9) - if: ${{ matrix.ckan-version == '2.7' || matrix.ckan-version == '2.8' }} - run: nosetests --ckan --nologcapture --with-pylons=test.ini --with-coverage --cover-package=ckanext.s3filestore --cover-inclusive --cover-erase --cover-tests diff --git a/scripts/ckan_cli b/scripts/ckan_cli new file mode 100644 index 00000000..3cf0b4cc --- /dev/null +++ b/scripts/ckan_cli @@ -0,0 +1,75 @@ +#!/bin/sh + +# Call either 'ckan' (from CKAN >= 2.9) or 'paster' (from CKAN <= 2.8) +# with appropriate syntax, depending on what is present on the system. +# This is intended to smooth the upgrade process from 2.8 to 2.9. +# Eg: +# ckan_cli jobs list +# could become either: +# paster --plugin=ckan jobs list -c /etc/ckan/default/production.ini +# or: +# ckan -c /etc/ckan/default/production.ini jobs list + +# This script is aware of the VIRTUAL_ENV environment variable, and will +# attempt to respect it with similar behaviour to commands like 'pip'. +# Eg placing this script in a virtualenv 'bin' directory will cause it +# to call the 'ckan' or 'paster' command in that directory, while +# placing this script elsewhere will cause it to rely on the VIRTUAL_ENV +# variable, or if that is not set, the system PATH. + +# Since the positioning of the CKAN configuration file is central to the +# differences between 'paster' and 'ckan', this script needs to be aware +# of the config file location. It will use the CKAN_INI environment +# variable if it exists, or default to /etc/ckan/default/production.ini. + +# If 'paster' is being used, the default plugin is 'ckan'. A different +# plugin can be specified by setting the PASTER_PLUGIN environment +# variable. This variable is irrelevant if using the 'ckan' command. + +CKAN_INI="${CKAN_INI:-/etc/ckan/default/production.ini}" +PASTER_PLUGIN="${PASTER_PLUGIN:-ckan}" +# First, look for a command alongside this file +ENV_DIR=$(dirname "$0") +if [ -f "$ENV_DIR/ckan" ]; then + COMMAND=ckan +elif [ -f "$ENV_DIR/paster" ]; then + COMMAND=paster +elif [ "$VIRTUAL_ENV" != "" ]; then + # If command not found alongside this file, check the virtualenv + ENV_DIR="$VIRTUAL_ENV/bin" + if [ -f "$ENV_DIR/ckan" ]; then + COMMAND=ckan + elif [ -f "$ENV_DIR/paster" ]; then + COMMAND=paster + fi +else + # if no virtualenv is active, try the system path + if (which ckan > /dev/null 2>&1); then + ENV_DIR=$(dirname $(which ckan)) + COMMAND=ckan + elif (which paster > /dev/null 2>&1); then + ENV_DIR=$(dirname $(which paster)) + COMMAND=paster + else + echo "Unable to locate 'ckan' or 'paster' command" >&2 + exit 1 + fi +fi + +if [ "$COMMAND" = "ckan" ]; then + echo "Using 'ckan' command from $ENV_DIR with config ${CKAN_INI}..." >&2 + # adjust args to match ckan expectations + COMMAND=$(echo "$1" | sed -e 's/create-test-data/seed/') + shift + exec $ENV_DIR/ckan -c ${CKAN_INI} $COMMAND "$@" $CLICK_ARGS +elif [ "$COMMAND" = "paster" ]; then + echo "Using 'paster' command from $ENV_DIR with config ${CKAN_INI}..." >&2 + # adjust args to match paster expectations + COMMAND=$1 + shift + if [ "$1" = "show" ]; then shift; fi + exec $ENV_DIR/paster --plugin=$PASTER_PLUGIN $COMMAND "$@" -c ${CKAN_INI} +else + echo "Unable to locate 'ckan' or 'paster' command in $ENV_DIR" >&2 + exit 1 +fi From 25b4304876f599cf17dcd58f7d9e56c50e29ba5e Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Thu, 20 Jan 2022 12:35:39 +1000 Subject: [PATCH 08/11] [QOL-8518] add another test from Keitaro fork --- .flake8 | 3 +- .github/workflows/ci.yml | 7 +- README.rst | 5 +- ckanext/s3filestore/controller.py | 2 +- ckanext/s3filestore/plugin.py | 10 +-- ckanext/s3filestore/tests/__init__.py | 7 ++ ckanext/s3filestore/tests/test_controller.py | 6 +- .../tests/test_fix_for_webpageview_plugin.py | 82 +++++++++++++++++++ 8 files changed, 104 insertions(+), 18 deletions(-) create mode 100644 ckanext/s3filestore/tests/test_fix_for_webpageview_plugin.py diff --git a/.flake8 b/.flake8 index a89a787b..0a914dd7 100644 --- a/.flake8 +++ b/.flake8 @@ -11,7 +11,8 @@ format = pylint # Show the source of errors. show_source = True -max-complexity = 10 +max-complexity = 12 +max-line-length = 127 # List ignore rules one per line. ignore = diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 043d746d..94819bbb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,3 +1,4 @@ + name: CI on: @@ -16,12 +17,12 @@ jobs: with: python-version: '3.9' - - name: Install requirements + - name: Install flake8 run: | python -m pip install --upgrade pip - pip install flake8 pycodestyle + pip install flake8 - - name: Check syntax + - name: Lint with flake8 run: | flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan diff --git a/README.rst b/README.rst index b7e22533..cf5e4100 100644 --- a/README.rst +++ b/README.rst @@ -54,7 +54,6 @@ Config Settings Required:: - ckanext.s3filestore.aws_bucket_name = a-bucket-to-store-your-stuff ckanext.s3filestore.region_name = region-name ckanext.s3filestore.signature_version = s3v4 @@ -157,12 +156,12 @@ Running the Tests To run the tests, do:: - nosetests --ckan --nologcapture --with-pylons=test.ini + pytest --ckan-ini=test.ini To run the tests and produce a coverage report, first make sure you have coverage installed in your virtualenv (``pip install coverage``) then run:: - nosetests --ckan --nologcapture --with-pylons=test.ini --with-coverage --cover-package=ckanext.s3filestore --cover-inclusive --cover-erase --cover-tests + pytest --ckan-ini=test.ini --cov=ckanext.s3filestore ------------------------ Docker environment setup diff --git a/ckanext/s3filestore/controller.py b/ckanext/s3filestore/controller.py index 8077fd96..f2aa252f 100644 --- a/ckanext/s3filestore/controller.py +++ b/ckanext/s3filestore/controller.py @@ -1,6 +1,6 @@ # encoding: utf-8 -from ckan.plugins.toolkit import BaseController +from ckantoolkit import BaseController from .views import resource_download, filesystem_resource_download, uploaded_file_redirect diff --git a/ckanext/s3filestore/plugin.py b/ckanext/s3filestore/plugin.py index f4253adc..7bda03e0 100644 --- a/ckanext/s3filestore/plugin.py +++ b/ckanext/s3filestore/plugin.py @@ -2,9 +2,8 @@ import os import logging -from routes.mapper import SubMapper from ckan import plugins - +import ckantoolkit as toolkit from ckanext.s3filestore import uploader as s3_uploader from ckan.lib.uploader import ResourceUpload as DefaultResourceUpload,\ get_resource_uploader @@ -12,7 +11,6 @@ from ckanext.s3filestore.tasks import s3_afterUpdatePackage LOG = logging.getLogger(__name__) -toolkit = plugins.toolkit class S3FileStorePlugin(plugins.SingletonPlugin): @@ -130,9 +128,11 @@ def enqueue_resource_visibility_update_job(self, visibility_level, pkg_id, pkg_d pkg_id) # IRoutes - # Ignored on CKAN >= 2.8 + # Ignored on CKAN >= 2.9 def before_map(self, map): + from routes.mapper import SubMapper + with SubMapper(map, controller='ckanext.s3filestore.controller:S3Controller') as m: # Override the resource download links if not hasattr(DefaultResourceUpload, 'download'): @@ -162,7 +162,7 @@ def before_map(self, map): return map # IBlueprint - # Ignored on CKAN < 2.8 + # Ignored on CKAN < 2.9 def get_blueprint(self): from ckanext.s3filestore.views import\ diff --git a/ckanext/s3filestore/tests/__init__.py b/ckanext/s3filestore/tests/__init__.py index 45b1ae19..89b6da1b 100644 --- a/ckanext/s3filestore/tests/__init__.py +++ b/ckanext/s3filestore/tests/__init__.py @@ -1,5 +1,8 @@ # encoding: utf-8 +from ckan.tests import helpers + + def _get_status_code(response): """ Get the status code from a HTTP response. Supports both Pylons/WebOb and Flask. @@ -17,3 +20,7 @@ def _get_response_body(response): return response.text else: return response.body + + +def teardown_function(self): + helpers.reset_db() diff --git a/ckanext/s3filestore/tests/test_controller.py b/ckanext/s3filestore/tests/test_controller.py index 227dffb3..904fc19d 100644 --- a/ckanext/s3filestore/tests/test_controller.py +++ b/ckanext/s3filestore/tests/test_controller.py @@ -20,7 +20,7 @@ from ckanext.s3filestore import uploader -from . import _get_status_code, _get_response_body +from . import _get_status_code, _get_response_body, teardown_function log = logging.getLogger(__name__) @@ -33,10 +33,6 @@ def setup_function(self): uploader.BaseS3Uploader().get_s3_bucket(self.bucket_name) -def teardown_function(self): - helpers.reset_db() - - def _test_org(): try: return helpers.call_action('organization_show', id='test-org') diff --git a/ckanext/s3filestore/tests/test_fix_for_webpageview_plugin.py b/ckanext/s3filestore/tests/test_fix_for_webpageview_plugin.py new file mode 100644 index 00000000..1a8daf03 --- /dev/null +++ b/ckanext/s3filestore/tests/test_fix_for_webpageview_plugin.py @@ -0,0 +1,82 @@ +# encoding: utf-8 +import requests +import six + +from nose.tools import assert_raises, with_setup +from werkzeug.datastructures import FileStorage as FlaskFileStorage + +from ckan.lib.helpers import url_for +from ckantoolkit import config +from ckan.tests import helpers, factories + +from . import _get_status_code, _get_response_body, teardown_function + + +@with_setup(teardown=teardown_function) +@helpers.change_config("ckan.plugins", "webpage_view s3filestore") +@helpers.change_config("ckan.views.default_views", "webpage_view") +def test_view_shown_for_url_type_upload(app=None): + + if not app: + app = helpers._get_test_app() + + dataset = factories.Dataset() + context = {u'user': factories.Sysadmin()[u'name']} + + content = u""" + + + + + WebpageView + + + + + """ + resource = helpers.call_action( + 'resource_create', + package_id=dataset['id'], + upload=FlaskFileStorage(six.StringIO(content), u'test.html') + ) + + resource_view = helpers.call_action(u'resource_view_list', context, + id=resource[u'id'])[0] + + with assert_raises(KeyError): + assert resource_view[u'page_url'] + + resource_view_src_url = url_for( + u's3_resource.resource_download', + id=dataset[u'name'], + resource_id=resource[u'id'] + ) + + url = url_for( + u'resource.read', id=dataset[u'name'], resource_id=resource[u'id'] + ) + + response = app.get(url) + + assert (u'/dataset/{0}/resource/{1}/download?preview=True' + .format(dataset[u'id'], resource[u'id']) + in response) + + status_code, location = _get_expecting_redirect(resource_view_src_url) + + r = requests.get(location) + + assert u'WebpageView' in _get_response_body(r) + + +def _get_expecting_redirect(url, app=None): + if url.startswith('http:') or url.startswith('https:'): + site_url = config.get('ckan.site_url') + url = url.replace(site_url, '') + if not app: + app = helpers._get_test_app() + response = app.get(url, follow_redirects=False) + status_code = _get_status_code(response) + assert status_code in [301, 302], \ + "%s resulted in %s instead of a redirect" % (url, response.status) + return status_code, response.location From 6b22480c8cf782ca7655f093e60c6098b6fe5c98 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Thu, 20 Jan 2022 13:03:14 +1000 Subject: [PATCH 09/11] [QOL-8518] update Pylons route names to better match Flask --- ckanext/s3filestore/plugin.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ckanext/s3filestore/plugin.py b/ckanext/s3filestore/plugin.py index 7bda03e0..022b7f95 100644 --- a/ckanext/s3filestore/plugin.py +++ b/ckanext/s3filestore/plugin.py @@ -136,10 +136,10 @@ def before_map(self, map): with SubMapper(map, controller='ckanext.s3filestore.controller:S3Controller') as m: # Override the resource download links if not hasattr(DefaultResourceUpload, 'download'): - m.connect('resource_download', + m.connect('s3_resource.resource_download', '/dataset/{id}/resource/{resource_id}/download', action='resource_download') - m.connect('resource_download', + m.connect('s3_resource.resource_download', '/dataset/{id}/resource/{resource_id}/download/{filename}', action='resource_download') @@ -151,7 +151,7 @@ def before_map(self, map): # Allow fallback to access old files use_filename = toolkit.asbool(toolkit.config.get('ckanext.s3filestore.use_filename', False)) if not use_filename: - m.connect('resource_download', + m.connect('s3_resource.resource_download', '/dataset/{id}/resource/{resource_id}/orig_download/{filename}', action='resource_download') From cdcd1e4a0e5a04f86489b056fe4fbd999f953c45 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Thu, 20 Jan 2022 13:03:31 +1000 Subject: [PATCH 10/11] [QOL-8518] make test teardown argument optional as we don't need it --- ckanext/s3filestore/tests/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/s3filestore/tests/__init__.py b/ckanext/s3filestore/tests/__init__.py index 89b6da1b..872da3bd 100644 --- a/ckanext/s3filestore/tests/__init__.py +++ b/ckanext/s3filestore/tests/__init__.py @@ -22,5 +22,5 @@ def _get_response_body(response): return response.body -def teardown_function(self): +def teardown_function(self=None): helpers.reset_db() From d4d279079f03c690a1435787f12fc32ed0935273 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Thu, 20 Jan 2022 13:23:27 +1000 Subject: [PATCH 11/11] [QOL-8518] drop webpageview test as we don't provide special preview handling - basing inline display on a request parameter undermines the purpose of blocking inline display in the first place, which is to prevent HTML resources from potentially being used as XSS vectors. --- .../tests/test_fix_for_webpageview_plugin.py | 82 ------------------- 1 file changed, 82 deletions(-) delete mode 100644 ckanext/s3filestore/tests/test_fix_for_webpageview_plugin.py diff --git a/ckanext/s3filestore/tests/test_fix_for_webpageview_plugin.py b/ckanext/s3filestore/tests/test_fix_for_webpageview_plugin.py deleted file mode 100644 index 1a8daf03..00000000 --- a/ckanext/s3filestore/tests/test_fix_for_webpageview_plugin.py +++ /dev/null @@ -1,82 +0,0 @@ -# encoding: utf-8 -import requests -import six - -from nose.tools import assert_raises, with_setup -from werkzeug.datastructures import FileStorage as FlaskFileStorage - -from ckan.lib.helpers import url_for -from ckantoolkit import config -from ckan.tests import helpers, factories - -from . import _get_status_code, _get_response_body, teardown_function - - -@with_setup(teardown=teardown_function) -@helpers.change_config("ckan.plugins", "webpage_view s3filestore") -@helpers.change_config("ckan.views.default_views", "webpage_view") -def test_view_shown_for_url_type_upload(app=None): - - if not app: - app = helpers._get_test_app() - - dataset = factories.Dataset() - context = {u'user': factories.Sysadmin()[u'name']} - - content = u""" - - - - - WebpageView - - - - - """ - resource = helpers.call_action( - 'resource_create', - package_id=dataset['id'], - upload=FlaskFileStorage(six.StringIO(content), u'test.html') - ) - - resource_view = helpers.call_action(u'resource_view_list', context, - id=resource[u'id'])[0] - - with assert_raises(KeyError): - assert resource_view[u'page_url'] - - resource_view_src_url = url_for( - u's3_resource.resource_download', - id=dataset[u'name'], - resource_id=resource[u'id'] - ) - - url = url_for( - u'resource.read', id=dataset[u'name'], resource_id=resource[u'id'] - ) - - response = app.get(url) - - assert (u'/dataset/{0}/resource/{1}/download?preview=True' - .format(dataset[u'id'], resource[u'id']) - in response) - - status_code, location = _get_expecting_redirect(resource_view_src_url) - - r = requests.get(location) - - assert u'WebpageView' in _get_response_body(r) - - -def _get_expecting_redirect(url, app=None): - if url.startswith('http:') or url.startswith('https:'): - site_url = config.get('ckan.site_url') - url = url.replace(site_url, '') - if not app: - app = helpers._get_test_app() - response = app.get(url, follow_redirects=False) - status_code = _get_status_code(response) - assert status_code in [301, 302], \ - "%s resulted in %s instead of a redirect" % (url, response.status) - return status_code, response.location