Merge pull request #139 from kkopachev/nosetests

Switch from pyvows to nosetest

Makefile

@@ -20,7 +20,7 @@ docs: setup_docs build_docs
 	python -mwebbrowser file:///`pwd`/docs/_build/html/index.html
 
 test: setup
-	pyvows -c -l tc_aws
+	nosetests
 
 publish:
 	python setup.py register -r pypi

setup.py

@@ -5,7 +5,6 @@
 import re
 
 from setuptools import setup, find_packages
-from setuptools.command.install import install
 
 def version():
     """retrieve version from tag name"""
@@ -58,16 +57,14 @@ def readme():
         'python-dateutil',
         'thumbor>=6.0.0,<7',
         'tornado-botocore',
-        #'botocore',
     ],
     extras_require={
         'tests': [
-            'pyvows',
             'coverage',
-            'tornado_pyvows',
             'boto',
-            'moto<=1.3.3',
+            'moto[server]',
             'mock',
+            'nose',
         ],
     },
 )

tc_aws/aws/storage.py

@@ -87,7 +87,7 @@ def remove(self, path, callback=None):
         Deletes data at path
         :param string path: Path to delete
         """
-        self.storage.delete(path)
+        self.storage.delete(path, callback)
 
     @return_future
     def exists(self, path, callback):
@@ -109,7 +109,7 @@ def return_data(file_key):
     def is_expired(self, key):
         """
         Tells whether key has expired
-        :param string key: Path to check
+        :param  key: Path to check
         :return: Whether it is expired or not
         :rtype: bool
         """
@@ -164,13 +164,15 @@ def return_data(file_key):
 
         self.storage.get(crypto_path, callback=return_data)
 
-    def put_crypto(self, path):
+    @return_future
+    def put_crypto(self, path, callback):
         """
         Stores crypto data at given path
         :param string path: Path to store the data at
         :return: Path where the crypto data is stored
         """
         if not self.context.config.STORES_CRYPTO_KEY_FOR_EACH_IMAGE:
+            callback(None)
             return
 
         if not self.context.server.security_key:
@@ -179,9 +181,11 @@ def put_crypto(self, path):
         file_abspath = self._normalize_path(path)
         crypto_path = '%s.txt' % splitext(file_abspath)[0]
 
-        self.set(self.context.server.security_key, crypto_path)
+        def cb(*args, **kwargs):
+            callback(crypto_path)
+
+        self.set(self.context.server.security_key, crypto_path, cb)
 
-        return crypto_path
 
     @return_future
     def get_detector_data(self, path, callback):
@@ -202,7 +206,8 @@ def return_data(file_key):
 
         self.storage.get(path, callback=return_data)
 
-    def put_detector_data(self, path, data):
+    @return_future
+    def put_detector_data(self, path, data, callback):
         """
         Stores detector data at given path
         :param string path: Path to store the data at
@@ -214,9 +219,7 @@ def put_detector_data(self, path, data):
 
         path = '%s.detectors.txt' % splitext(file_abspath)[0]
 
-        self.set(dumps(data), path)
-
-        return path
+        self.set(dumps(data), path, callback)
 
     def _get_error(self, response):
         """

tests/__init__.py

@@ -0,0 +1,112 @@
+# coding: utf-8
+
+# Copyright (c) 2015, thumbor-community
+# Use of this source code is governed by the MIT license that can be
+# found in the LICENSE file.
+
+import logging
+import os
+import signal
+import subprocess as sp
+import sys
+import time
+
+import botocore.session
+import mock
+import requests
+from botocore.client import ClientCreator
+from tornado.testing import AsyncTestCase
+
+from tc_aws.aws.bucket import Bucket
+from tests.fixtures.storage_fixture import s3_bucket
+
+logging.basicConfig(level=logging.CRITICAL)
+
+os.environ["TEST_SERVER_MODE"] = "true"
+
+_proxy_bypass = {
+    "http": None,
+    "https": None,
+}
+
+
+def start_service(host, port):
+    args = [sys.executable, "-m", "moto.server", "-H", host,
+            "-p", str(port)]
+
+    process = sp.Popen(args, stderr=sp.PIPE)
+    url = "http://{host}:{port}".format(host=host, port=port)
+
+    for i in range(0, 30):
+        if process.poll() is not None:
+            process.communicate()
+            break
+
+        try:
+            # we need to bypass the proxies due to monkeypatches
+            requests.get(url, timeout=0.5)
+            break
+        except requests.exceptions.ConnectionError:
+            time.sleep(0.5)
+    else:
+        stop_process(process)
+
+    return process
+
+
+def stop_process(process):
+    try:
+        process.send_signal(signal.SIGTERM)
+        process.communicate()
+    except Exception:
+        process.kill()
+        outs, errors = process.communicate()
+        exit_code = process.returncode
+        msg = "Child process finished {} not in clean way: {} {}" \
+            .format(exit_code, outs, errors)
+        raise RuntimeError(msg)
+
+
+class FakeClientCreator(ClientCreator):
+    def create_client(self, *args, **kwargs):
+        if kwargs['endpoint_url'] is None:
+            kwargs['endpoint_url'] = "http://localhost:5000"
+        return super(FakeClientCreator, self).create_client(*args, **kwargs)
+
+
+class S3MockedAsyncTestCase(AsyncTestCase):
+    _process = None
+
+    @classmethod
+    def setUpClass(cls):
+        super(S3MockedAsyncTestCase, cls).setUpClass()
+        cls._process = start_service("localhost", 5000)
+
+        os.environ['AWS_SHARED_CREDENTIALS_FILE'] = ''
+        os.environ['AWS_ACCESS_KEY_ID'] = 'test-key'
+        os.environ['AWS_SECRET_ACCESS_KEY'] = 'test-secret-key'
+        os.environ['AWS_SESSION_TOKEN'] = 'test-session-token'
+
+    @classmethod
+    def tearDownClass(cls):
+        super(S3MockedAsyncTestCase, cls).tearDownClass()
+        stop_process(cls._process)
+
+    def setUp(self):
+        super(S3MockedAsyncTestCase, self).setUp()
+
+        requests.post("http://localhost:5000/moto-api/reset")
+
+        self._client_patcher = mock.patch('botocore.client.ClientCreator', FakeClientCreator)
+        self._client_patcher.start()
+
+        client = botocore.session.get_session().create_client('s3')
+        client.create_bucket(Bucket=s3_bucket)
+
+        self.addCleanup(self._client_patcher.stop)
+
+    def tearDown(self):
+        super(S3MockedAsyncTestCase, self).tearDown()
+        # singleton Bucket holds old IOLoop instance which closed after each test
+        # this cleans singleton
+        Bucket._instances = {}

tests/fixtures/storage_fixture.py

@@ -0,0 +1,23 @@
+# coding: utf-8
+
+# Copyright (c) 2015, thumbor-community
+# Use of this source code is governed by the MIT license that can be
+# found in the LICENSE file.
+
+from os.path import join, abspath, dirname
+
+from thumbor.context import ServerParameters
+
+s3_bucket = 'thumbor-images-test'
+
+IMAGE_URL = 's.glbimg.com/some/image_%s.jpg'
+IMAGE_PATH = join(abspath(dirname(__file__)), 'image.jpg')
+
+with open(IMAGE_PATH, 'r') as img:
+    IMAGE_BYTES = img.read()
+
+
+def get_server(key=None):
+    server_params = ServerParameters(8888, 'localhost', 'thumbor.conf', None, 'info', None)
+    server_params.security_key = key
+    return server_params

tests/test_loader.py

@@ -0,0 +1,42 @@
+# coding: utf-8
+
+# Copyright (c) 2015, thumbor-community
+# Use of this source code is governed by the MIT license that can be
+# found in the LICENSE file.
+
+from unittest import TestCase
+
+from thumbor.config import Config
+from thumbor.context import Context
+
+from tc_aws.loaders import _get_bucket, _get_bucket_and_key, _get_key
+from .fixtures.storage_fixture import IMAGE_PATH
+
+
+class LoaderTestCase(TestCase):
+    def test_can_get_bucket_and_key(self):
+        conf = Config(
+            TC_AWS_LOADER_BUCKET=None,
+            TC_AWS_LOADER_ROOT_PATH=''
+        )
+
+        ctx = Context(config=conf)
+
+        path = 'some-bucket/some/image/path.jpg'
+        bucket, key = _get_bucket_and_key(ctx, path)
+        self.assertEqual(bucket, 'some-bucket')
+        self.assertEqual(key, 'some/image/path.jpg')
+
+    def test_can_detect_bucket(self):
+        topic = _get_bucket('/'.join(['thumbor-images-test', IMAGE_PATH]))
+        self.assertEqual(topic, 'thumbor-images-test')
+
+    def test_can_detect_key(self):
+        conf = Config(
+            TC_AWS_LOADER_BUCKET=None,
+            TC_AWS_LOADER_ROOT_PATH='',
+        )
+        context = Context(config=conf)
+        key = _get_key(IMAGE_PATH, context)
+
+        self.assertEqual(key, IMAGE_PATH)

tests/test_presigning_loader.py

@@ -0,0 +1,78 @@
+# coding: utf-8
+
+# Copyright (c) 2015, thumbor-community
+# Use of this source code is governed by the MIT license that can be
+# found in the LICENSE file.
+
+from urlparse import urlparse, parse_qs
+
+import botocore.session
+from derpconf.config import Config
+from mock import patch
+from thumbor.context import Context
+from tornado.testing import gen_test
+
+from fixtures.storage_fixture import IMAGE_PATH, IMAGE_BYTES, s3_bucket
+from tc_aws.loaders import presigning_loader
+from tests import S3MockedAsyncTestCase
+
+
+class PreSigningLoaderTestCase(S3MockedAsyncTestCase):
+    @gen_test
+    def test_can_load_image(self):
+        client = botocore.session.get_session().create_client('s3')
+        client.create_bucket(Bucket=s3_bucket)
+
+        client.put_object(
+            Bucket=s3_bucket,
+            Key=''.join(['root_path', IMAGE_PATH]),
+            Body=IMAGE_BYTES,
+            ContentType='image/jpeg', )
+
+        conf = Config(
+            TC_AWS_LOADER_BUCKET=s3_bucket,
+            TC_AWS_LOADER_ROOT_PATH='root_path',
+        )
+
+        image = yield presigning_loader.load(Context(config=conf), IMAGE_PATH)
+        self.assertEqual(image.buffer, IMAGE_BYTES)
+
+    @patch('thumbor.loaders.http_loader.load_sync')
+    @gen_test
+    def test_should_use_http_loader(self, load_sync_patch):
+        def cb(a, b, callback, *args, **kwargs):
+            callback('foobar')
+            return None
+
+        load_sync_patch.side_effect = cb
+
+        conf = Config(TC_AWS_ENABLE_HTTP_LOADER=True)
+        presigning_loader.load(Context(config=conf), 'http://foo.bar')
+        self.assertTrue(load_sync_patch.called)
+
+    @gen_test
+    def test_can_validate_buckets(self):
+        conf = Config(
+            TC_AWS_ALLOWED_BUCKETS=['whitelist_bucket'],
+            TC_AWS_LOADER_BUCKET=None,
+        )
+
+        image = yield presigning_loader.load(Context(config=conf), '/'.join([s3_bucket, IMAGE_PATH]))
+        self.assertIsNone(image)
+
+    @gen_test
+    def test_can_build_presigned_url(self):
+        context = Context(config=(Config()))
+        url = yield presigning_loader._generate_presigned_url(context, "bucket-name", "some-s3-key")
+
+        url = urlparse(url)
+        self.assertEqual(url.scheme[0:4], 'http')
+        self.assertEqual(url.path, '/bucket-name/some-s3-key')
+
+        url_params = parse_qs(url.query)
+        # We can't test Expires & Signature values as they vary depending on the TZ
+        self.assertIn('Expires', url_params)
+        self.assertIn('Signature', url_params)
+
+        self.assertDictContainsSubset({'AWSAccessKeyId': ['test-key'], 'x-amz-security-token': ['test-session-token']},
+                                      url_params)